Skip to content
Jump to bottom

DBAL qualify table names (CFM-356) #176

Merged
merged 1 commit into from Mar 29, 2024
+208 −123
Merged

DBAL qualify table names (CFM-356) #176

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
120 changes: 37 additions & 83 deletions app/src/Command/TransmogrifyCommand.php
View file
Open in desktop
Expand Up @@ -39,8 +39,8 @@
use Cake\ORM\TableRegistry;
use Cake\Utility\Inflector;
use \App\Lib\Util\PaginatedSqlIterator;
use \App\Lib\Util\DBALConnection;

use Doctrine\DBAL\DriverManager;
use Doctrine\DBAL\Exception\ForeignKeyConstraintViolationException;

class TransmogrifyCommand extends Command {
Expand Down Expand Up @@ -356,8 +356,6 @@ class TransmogrifyCommand extends Command {
// Make some objects more easily accessible
protected $inconn = null;
protected $outconn = null;
// Cache the driver for ease of workarounds
protected $outdriver = null;

// Shell arguments, for easier access
protected $args = null;
Expand Down Expand Up @@ -454,7 +452,9 @@ protected function check_group_membership(array $origRow, array $row) {
'actor_identifier' => $origRow['actor_identifier']
];

$this->outconn->insert('group_members', $ownerRow);
$tableName = 'group_members';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $ownerRow);
} else {
$this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']);
}
Expand Down Expand Up @@ -522,74 +522,13 @@ public function execute(Arguments $args, ConsoleIo $io) {
// new database.

// First, open connections to both old and new databases.
// Use the Cake ConnectionManager to get the database configs to pass to DBAL.
$indb = ConnectionManager::get('transmogrify');
$incfg = $indb->config();

if(empty($incfg)) {
throw new \InvalidArgumentException(__d('error', 'db.config', ["transmogrify"]));
}

$outdb = ConnectionManager::get('default');
$outcfg = $outdb->config();

if(empty($outcfg)) {
throw new \InvalidArgumentException(__d('error', 'db.config', ["default"]));
}

$inconfig = new \Doctrine\DBAL\Configuration();

$cargs = [
'dbname' => $incfg['database'],
'user' => $incfg['username'],
'password' => $incfg['password'],
'host' => $incfg['host'],
'driver' => ($incfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($incfg['ssl_ca'])) {
// mysqli supports SSL configuration
$cargs['ssl_ca'] = $incfg['ssl_ca'];
}

$this->inconn = DriverManager::getConnection($cargs, $inconfig);

$outconfig = new \Doctrine\DBAL\Configuration();

$cargs = [
'dbname' => $outcfg['database'],
'user' => $outcfg['username'],
'password' => $outcfg['password'],
'host' => $outcfg['host'],
'driver' => ($outcfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($outcfg['ssl_ca'])) {
// mysqli supports SSL configuration
$cargs['ssl_ca'] = $outcfg['ssl_ca'];
}

$this->outconn = DriverManager::getConnection($cargs, $outconfig);
$this->outdriver = $cargs['driver'];

$this->inconn = DBALConnection::factory($io, 'transmogrify');
$this->outconn = DBALConnection::factory($io, 'default');

// We accept a list of table names, mostly for testing purposes
$atables = $args->getArguments();

$schemaPrefix = '';

if($this->outdriver == 'mysqli') {
// We prefix the database to the table to avoid having to quote table names
// that match (MySQL) reserved keywords (in particular "groups"). While
// theoretically Postgres supports the same notation, it seems to cause
// more problems than it solves.

$schemaPrefix = $outcfg['database'] . '.';
}

// Register the current version for future upgrade purposes

$targetVersion = rtrim(file_get_contents(CONFIG . DS . "VERSION"));

$metaTable = $this->getTableLocator()->get('Meta');
Expand All @@ -610,9 +549,10 @@ public function execute(Arguments $args, ConsoleIo $io) {
$this->$p();
}

$count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $this->tables[$t]['source']);
$qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']);
$count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $qualifiedTableName);

$insql = "SELECT * FROM " . $this->tables[$t]['source'] . " ORDER BY id ASC";
$insql = "SELECT * FROM " . $qualifiedTableName . " ORDER BY id ASC";
$stmt = $this->inconn->executeQuery($insql);

// Check if the table contains data
Expand Down Expand Up @@ -650,7 +590,8 @@ public function execute(Arguments $args, ConsoleIo $io) {

$this->mapFields($t, $row);

$this->outconn->insert($schemaPrefix.$t, $row);
$qualifiedTableName = $this->outconn->qualifyTableName($t);
$this->outconn->insert($qualifiedTableName, $row);

$this->cacheResults($t, $row);

Expand Down Expand Up @@ -695,17 +636,18 @@ public function execute(Arguments $args, ConsoleIo $io) {
$io->out("<error>(Errors: " . $err . ")</error>");

// Reset sequence to next value after current max.
$max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $t);
$qualifiedTableName = $this->outconn->qualifyTableName($t);
$max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $qualifiedTableName);
$max++;
$this->io->info("Resetting sequence for $t to $max");
$this->io->info("Resetting sequence for $qualifiedTableName to $max");

// Strictly speaking we should use prepared statements, but we control the
// data here, and also we're executing a maintenance operation (so query
// optimization is less important)
if($this->outdriver == 'mysqli') {
$outsql = "ALTER TABLE `" . $t . "` AUTO_INCREMENT = " . $max;
if($this->outconn->isMySQL()) {
$outsql = "ALTER TABLE $qualifiedTableName AUTO_INCREMENT = " . $max;
} else {
$outsql = "ALTER SEQUENCE " . $t . "_id_seq RESTART WITH " . $max;
$outsql = "ALTER SEQUENCE " . $qualifiedTableName . "_id_seq RESTART WITH " . $max;
}
$this->outconn->executeQuery($outsql);

Expand Down Expand Up @@ -812,7 +754,7 @@ protected function fixBooleans(string $table, array &$row) {
// this issue: https://github.com/doctrine/dbal/issues/1847
// We need to (more generically than this hack) convert from boolean to char
// to avoid errors on insert
if($this->outdriver == 'mysqli') {
if($this->outconn->isMySQL()) {
$row[$a] = ($row[$a] ? '1' : '0');
} else {
$row[$a] = ($row[$a] ? 't' : 'f');
Expand Down Expand Up @@ -1068,7 +1010,9 @@ protected function map_login_identifiers(array $origRow, array $row) {
$this->fixBooleans('identifiers', $copiedRow);

try {
$this->outconn->insert('identifiers', $copiedRow);
$tableName = 'identifiers';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $copiedRow);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->io->warning("record already exists: " . print_r($copiedRow, true));
}
Expand Down Expand Up @@ -1151,7 +1095,9 @@ protected function map_org_identity_co_person_id(array $row) {
$this->io->info('Populating org identity map...');

// We pull deleted rows because we might be migrating deleted rows
$mapsql = "SELECT * FROM cm_co_org_identity_links";
$tableName = "cm_co_org_identity_links";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$mapsql = "SELECT * FROM $qualifiedTableName";
$stmt = $this->inconn->query($mapsql);

while($r = $stmt->fetch()) {
Expand Down Expand Up @@ -1254,7 +1200,9 @@ protected function processExtendedAttributes() {
// First, pull the old Extended Attribute configuration.
$extendedAttrs = [];

$insql = "SELECT * FROM cm_co_extended_attributes ORDER BY id ASC";
$tableName = "cm_co_extended_attributes";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$insql = "SELECT * FROM $qualifiedTableName ORDER BY id ASC";
$stmt = $this->inconn->query($insql);

while($row = $stmt->fetch()) {
Expand All @@ -1267,7 +1215,9 @@ protected function processExtendedAttributes() {
}

foreach(array_keys($extendedAttrs) as $coId) {
$insql = "SELECT * FROM cm_co" . $coId . "_person_extended_attributes";
$tableName = "cm_co" . $coId . "_person_extended_attributes";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$insql = "SELECT * FROM $qualifiedTableName";
$stmt = $this->inconn->query($insql);

while($eaRow = $stmt->fetch()) {
Expand All @@ -1288,7 +1238,9 @@ protected function processExtendedAttributes() {
$this->fixBooleans('ad_hoc_attributes', $adhocRow);

try {
$this->outconn->insert('ad_hoc_attributes', $adhocRow);
$tableName = 'ad_hoc_attributes';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $adhocRow);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->io->warning("record already exists: " . print_r($adhocRow, true));
}
Expand Down Expand Up @@ -1346,6 +1298,8 @@ protected function split_external_identity(array $origRow, array $row) {
// Since we're creating a new row, we have to manually fix up booleans
$roleRow['deleted'] = ($roleRow['deleted'] ? 't' : 'f');

$this->outconn->insert('external_identity_roles', $roleRow);
$tableName = 'external_identity_roles';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $roleRow);
}
}
155 changes: 155 additions & 0 deletions app/src/Lib/Util/DBALConnection.php
View file
Open in desktop
@@ -0,0 +1,155 @@
<?php
/**
* COmanage Registry DBAL Connection
*
* Portions licensed to the University Corporation for Advanced Internet
* Development, Inc. ("UCAID") under one or more contributor license agreements.
* See the NOTICE file distributed with this work for additional information
* regarding copyright ownership.
*
* UCAID licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @link https://www.internet2.edu/comanage COmanage Project
* @package registry
* @since COmanage Registry v5.0.0
* @license Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
*/

declare(strict_types = 1);

namespace App\Lib\Util;

use Cake\Console\ConsoleIo;
use Cake\Datasource\ConnectionManager;

use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\DriverManager;

class DBALConnection extends Connection {

// Console for output
protected $io = null;

// The database driver in use
public $driver = null;

// PostgreSQL schema
protected $pgSchema = null;

/**
* Return DBALConnection instance constructed from CakePHP connection name
*
* @since COmanage Registry v5.0.0
* @param ConsoleIO $io Cake ConsoleIo object
* @param string $connection Database connection name
*/

public static function factory(?ConsoleIo $io=null, string $connection='default') {
// Use the CakePHP ConnectionManager to get the database config to pass to DBAL.
$db = ConnectionManager::get($connection);

// $db is a ConnectionInterface object.
$cfg = $db->config();

// DBAL Configuration instance to pass into DBAL connection factory
$config = new Configuration();

// Translate from CakePHP param names to DBAL param names.
$cfargs = [
'dbname' => $cfg['database'],
'user' => $cfg['username'],
'password' => $cfg['password'],
'host' => $cfg['host'],
'driver' => ($cfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($cfg['ssl_ca'])) {
$cfargs['ssl_ca'] = $cfg['ssl_ca'];
}

// Signal to DBAL factory to create an instance of this class.
$cfargs['wrapperClass'] = self::class;

if($io) {
$io->out("Connecting to database " . $cfg['database'] . " as "
. $cfg['username'] . "@" . $cfg['host']);
}

// Use the DBAL factory to open a connection but return instance
// of this class.
$conn = DriverManager::getConnection($cfargs, $config);

// The CakePHP database configuration supports using a PostgreSQL schema. If
// defined signal that we should prefix tables with the PostgreSQL schema and a dot '.'.
if(!empty($cfg['schema'])) {
$conn->pgSchema = $cfg['schema'];
}

if($io) {
$conn->io = $io;
}
$conn->driver = $cfg['driver'];

return $conn;
}

/**
* Is the database MySQL or MariaDB?
*
* @since COmanage Registry v5.0.0
* @return boolean
*/
public function isMySQL() {
return ($this->driver == 'Cake\Database\Driver\Mysql');
}

/**
* Is the database PostgreSQL
*
* @since COmanage Registry v5.0.0
* @return boolean
*/
public function isPostgreSQL() {
return ($this->driver == 'Cake\Database\Driver\Postgres');
}

/**
* Qualify database table name.
*
* @since COmanage Registry v5.0.0
* @param string $tableName Unqualified table name
* @return string Qualified table name
*/

public function qualifyTableName($tableName) {
switch ($this->driver) {
case 'Cake\Database\Driver\Postgres':
if($this->pgSchema) {
$qualifiedTableName = $this->pgSchema . '.' . $tableName;
}
break;

case 'Cake\Database\Driver\Mysql':
$qualifiedTableName = $this->getDatabase() . '.' . $tableName;
break;

default:
$qualifiedTableName = $tableName;

}

return $qualifiedTableName;
}
}