Skip to content

Commit

Permalink
Merge pull request COmanage#176 from skoranda/CFM-356-string_concat
Browse files Browse the repository at this point in the history
DBAL qualify table names (CFM-356)
  • Loading branch information
skoranda authored Mar 29, 2024
2 parents 46e4df3 + cab2db2 commit f254147
Show file tree
Hide file tree
Showing 3 changed files with 208 additions and 123 deletions.
120 changes: 37 additions & 83 deletions app/src/Command/TransmogrifyCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
use Cake\ORM\TableRegistry;
use Cake\Utility\Inflector;
use \App\Lib\Util\PaginatedSqlIterator;
use \App\Lib\Util\DBALConnection;

use Doctrine\DBAL\DriverManager;
use Doctrine\DBAL\Exception\ForeignKeyConstraintViolationException;

class TransmogrifyCommand extends Command {
Expand Down Expand Up @@ -356,8 +356,6 @@ class TransmogrifyCommand extends Command {
// Make some objects more easily accessible
protected $inconn = null;
protected $outconn = null;
// Cache the driver for ease of workarounds
protected $outdriver = null;

// Shell arguments, for easier access
protected $args = null;
Expand Down Expand Up @@ -454,7 +452,9 @@ protected function check_group_membership(array $origRow, array $row) {
'actor_identifier' => $origRow['actor_identifier']
];

$this->outconn->insert('group_members', $ownerRow);
$tableName = 'group_members';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $ownerRow);
} else {
$this->io->error("Could not find owners group for CoGroupMember " . $origRow['id']);
}
Expand Down Expand Up @@ -522,74 +522,13 @@ public function execute(Arguments $args, ConsoleIo $io) {
// new database.

// First, open connections to both old and new databases.
// Use the Cake ConnectionManager to get the database configs to pass to DBAL.
$indb = ConnectionManager::get('transmogrify');
$incfg = $indb->config();

if(empty($incfg)) {
throw new \InvalidArgumentException(__d('error', 'db.config', ["transmogrify"]));
}

$outdb = ConnectionManager::get('default');
$outcfg = $outdb->config();

if(empty($outcfg)) {
throw new \InvalidArgumentException(__d('error', 'db.config', ["default"]));
}

$inconfig = new \Doctrine\DBAL\Configuration();

$cargs = [
'dbname' => $incfg['database'],
'user' => $incfg['username'],
'password' => $incfg['password'],
'host' => $incfg['host'],
'driver' => ($incfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($incfg['ssl_ca'])) {
// mysqli supports SSL configuration
$cargs['ssl_ca'] = $incfg['ssl_ca'];
}

$this->inconn = DriverManager::getConnection($cargs, $inconfig);

$outconfig = new \Doctrine\DBAL\Configuration();

$cargs = [
'dbname' => $outcfg['database'],
'user' => $outcfg['username'],
'password' => $outcfg['password'],
'host' => $outcfg['host'],
'driver' => ($outcfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($outcfg['ssl_ca'])) {
// mysqli supports SSL configuration
$cargs['ssl_ca'] = $outcfg['ssl_ca'];
}

$this->outconn = DriverManager::getConnection($cargs, $outconfig);
$this->outdriver = $cargs['driver'];

$this->inconn = DBALConnection::factory($io, 'transmogrify');
$this->outconn = DBALConnection::factory($io, 'default');

// We accept a list of table names, mostly for testing purposes
$atables = $args->getArguments();

$schemaPrefix = '';

if($this->outdriver == 'mysqli') {
// We prefix the database to the table to avoid having to quote table names
// that match (MySQL) reserved keywords (in particular "groups"). While
// theoretically Postgres supports the same notation, it seems to cause
// more problems than it solves.

$schemaPrefix = $outcfg['database'] . '.';
}

// Register the current version for future upgrade purposes

$targetVersion = rtrim(file_get_contents(CONFIG . DS . "VERSION"));

$metaTable = $this->getTableLocator()->get('Meta');
Expand All @@ -610,9 +549,10 @@ public function execute(Arguments $args, ConsoleIo $io) {
$this->$p();
}

$count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $this->tables[$t]['source']);
$qualifiedTableName = $this->inconn->qualifyTableName($this->tables[$t]['source']);
$count = $this->inconn->fetchOne("SELECT COUNT(*) FROM " . $qualifiedTableName);

$insql = "SELECT * FROM " . $this->tables[$t]['source'] . " ORDER BY id ASC";
$insql = "SELECT * FROM " . $qualifiedTableName . " ORDER BY id ASC";
$stmt = $this->inconn->executeQuery($insql);

// Check if the table contains data
Expand Down Expand Up @@ -650,7 +590,8 @@ public function execute(Arguments $args, ConsoleIo $io) {

$this->mapFields($t, $row);

$this->outconn->insert($schemaPrefix.$t, $row);
$qualifiedTableName = $this->outconn->qualifyTableName($t);
$this->outconn->insert($qualifiedTableName, $row);

$this->cacheResults($t, $row);

Expand Down Expand Up @@ -695,17 +636,18 @@ public function execute(Arguments $args, ConsoleIo $io) {
$io->out("<error>(Errors: " . $err . ")</error>");

// Reset sequence to next value after current max.
$max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $t);
$qualifiedTableName = $this->outconn->qualifyTableName($t);
$max = $this->outconn->fetchOne('SELECT MAX(id) FROM ' . $qualifiedTableName);
$max++;
$this->io->info("Resetting sequence for $t to $max");
$this->io->info("Resetting sequence for $qualifiedTableName to $max");

// Strictly speaking we should use prepared statements, but we control the
// data here, and also we're executing a maintenance operation (so query
// optimization is less important)
if($this->outdriver == 'mysqli') {
$outsql = "ALTER TABLE `" . $t . "` AUTO_INCREMENT = " . $max;
if($this->outconn->isMySQL()) {
$outsql = "ALTER TABLE $qualifiedTableName AUTO_INCREMENT = " . $max;
} else {
$outsql = "ALTER SEQUENCE " . $t . "_id_seq RESTART WITH " . $max;
$outsql = "ALTER SEQUENCE " . $qualifiedTableName . "_id_seq RESTART WITH " . $max;
}
$this->outconn->executeQuery($outsql);

Expand Down Expand Up @@ -812,7 +754,7 @@ protected function fixBooleans(string $table, array &$row) {
// this issue: https://github.com/doctrine/dbal/issues/1847
// We need to (more generically than this hack) convert from boolean to char
// to avoid errors on insert
if($this->outdriver == 'mysqli') {
if($this->outconn->isMySQL()) {
$row[$a] = ($row[$a] ? '1' : '0');
} else {
$row[$a] = ($row[$a] ? 't' : 'f');
Expand Down Expand Up @@ -1068,7 +1010,9 @@ protected function map_login_identifiers(array $origRow, array $row) {
$this->fixBooleans('identifiers', $copiedRow);

try {
$this->outconn->insert('identifiers', $copiedRow);
$tableName = 'identifiers';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $copiedRow);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->io->warning("record already exists: " . print_r($copiedRow, true));
}
Expand Down Expand Up @@ -1151,7 +1095,9 @@ protected function map_org_identity_co_person_id(array $row) {
$this->io->info('Populating org identity map...');

// We pull deleted rows because we might be migrating deleted rows
$mapsql = "SELECT * FROM cm_co_org_identity_links";
$tableName = "cm_co_org_identity_links";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$mapsql = "SELECT * FROM $qualifiedTableName";
$stmt = $this->inconn->query($mapsql);

while($r = $stmt->fetch()) {
Expand Down Expand Up @@ -1254,7 +1200,9 @@ protected function processExtendedAttributes() {
// First, pull the old Extended Attribute configuration.
$extendedAttrs = [];

$insql = "SELECT * FROM cm_co_extended_attributes ORDER BY id ASC";
$tableName = "cm_co_extended_attributes";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$insql = "SELECT * FROM $qualifiedTableName ORDER BY id ASC";
$stmt = $this->inconn->query($insql);

while($row = $stmt->fetch()) {
Expand All @@ -1267,7 +1215,9 @@ protected function processExtendedAttributes() {
}

foreach(array_keys($extendedAttrs) as $coId) {
$insql = "SELECT * FROM cm_co" . $coId . "_person_extended_attributes";
$tableName = "cm_co" . $coId . "_person_extended_attributes";
$qualifiedTableName = $this->inconn->qualifyTableName($tableName);
$insql = "SELECT * FROM $qualifiedTableName";
$stmt = $this->inconn->query($insql);

while($eaRow = $stmt->fetch()) {
Expand All @@ -1288,7 +1238,9 @@ protected function processExtendedAttributes() {
$this->fixBooleans('ad_hoc_attributes', $adhocRow);

try {
$this->outconn->insert('ad_hoc_attributes', $adhocRow);
$tableName = 'ad_hoc_attributes';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $adhocRow);
} catch (\Doctrine\DBAL\Exception\UniqueConstraintViolationException $e) {
$this->io->warning("record already exists: " . print_r($adhocRow, true));
}
Expand Down Expand Up @@ -1346,6 +1298,8 @@ protected function split_external_identity(array $origRow, array $row) {
// Since we're creating a new row, we have to manually fix up booleans
$roleRow['deleted'] = ($roleRow['deleted'] ? 't' : 'f');

$this->outconn->insert('external_identity_roles', $roleRow);
$tableName = 'external_identity_roles';
$qualifiedTableName = $this->outconn->qualifyTableName($tableName);
$this->outconn->insert($qualifiedTableName, $roleRow);
}
}
155 changes: 155 additions & 0 deletions app/src/Lib/Util/DBALConnection.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
<?php
/**
* COmanage Registry DBAL Connection
*
* Portions licensed to the University Corporation for Advanced Internet
* Development, Inc. ("UCAID") under one or more contributor license agreements.
* See the NOTICE file distributed with this work for additional information
* regarding copyright ownership.
*
* UCAID licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @link https://www.internet2.edu/comanage COmanage Project
* @package registry
* @since COmanage Registry v5.0.0
* @license Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
*/

declare(strict_types = 1);

namespace App\Lib\Util;

use Cake\Console\ConsoleIo;
use Cake\Datasource\ConnectionManager;

use Doctrine\DBAL\Configuration;
use Doctrine\DBAL\Connection;
use Doctrine\DBAL\DriverManager;

class DBALConnection extends Connection {

// Console for output
protected $io = null;

// The database driver in use
public $driver = null;

// PostgreSQL schema
protected $pgSchema = null;

/**
* Return DBALConnection instance constructed from CakePHP connection name
*
* @since COmanage Registry v5.0.0
* @param ConsoleIO $io Cake ConsoleIo object
* @param string $connection Database connection name
*/

public static function factory(?ConsoleIo $io=null, string $connection='default') {
// Use the CakePHP ConnectionManager to get the database config to pass to DBAL.
$db = ConnectionManager::get($connection);

// $db is a ConnectionInterface object.
$cfg = $db->config();

// DBAL Configuration instance to pass into DBAL connection factory
$config = new Configuration();

// Translate from CakePHP param names to DBAL param names.
$cfargs = [
'dbname' => $cfg['database'],
'user' => $cfg['username'],
'password' => $cfg['password'],
'host' => $cfg['host'],
'driver' => ($cfg['driver'] == 'Cake\Database\Driver\Postgres' ? "pdo_pgsql" : "mysqli")
];

// For MySQL SSL
if(!empty($cfg['ssl_ca'])) {
$cfargs['ssl_ca'] = $cfg['ssl_ca'];
}

// Signal to DBAL factory to create an instance of this class.
$cfargs['wrapperClass'] = self::class;

if($io) {
$io->out("Connecting to database " . $cfg['database'] . " as "
. $cfg['username'] . "@" . $cfg['host']);
}

// Use the DBAL factory to open a connection but return instance
// of this class.
$conn = DriverManager::getConnection($cfargs, $config);

// The CakePHP database configuration supports using a PostgreSQL schema. If
// defined signal that we should prefix tables with the PostgreSQL schema and a dot '.'.
if(!empty($cfg['schema'])) {
$conn->pgSchema = $cfg['schema'];
}

if($io) {
$conn->io = $io;
}
$conn->driver = $cfg['driver'];

return $conn;
}

/**
* Is the database MySQL or MariaDB?
*
* @since COmanage Registry v5.0.0
* @return boolean
*/
public function isMySQL() {
return ($this->driver == 'Cake\Database\Driver\Mysql');
}

/**
* Is the database PostgreSQL
*
* @since COmanage Registry v5.0.0
* @return boolean
*/
public function isPostgreSQL() {
return ($this->driver == 'Cake\Database\Driver\Postgres');
}

/**
* Qualify database table name.
*
* @since COmanage Registry v5.0.0
* @param string $tableName Unqualified table name
* @return string Qualified table name
*/

public function qualifyTableName($tableName) {
switch ($this->driver) {
case 'Cake\Database\Driver\Postgres':
if($this->pgSchema) {
$qualifiedTableName = $this->pgSchema . '.' . $tableName;
}
break;

case 'Cake\Database\Driver\Mysql':
$qualifiedTableName = $this->getDatabase() . '.' . $tableName;
break;

default:
$qualifiedTableName = $tableName;

}

return $qualifiedTableName;
}
}
Loading

0 comments on commit f254147

Please sign in to comment.