From b3e1e0e626966e99e41ccf6793a420ee69757104 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 13 Jun 2026 17:20:46 +0300 Subject: [PATCH 1/4] Support MySQL prefix lengths for indexes in schema.json (e.g. TEXT source(255)).Create indexes last. --- app/config/schema/schema.json | 2 +- app/src/Lib/Util/SchemaManager.php | 122 ++++++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/app/config/schema/schema.json b/app/config/schema/schema.json index 150f09f1f..173eb1114 100644 --- a/app/config/schema/schema.json +++ b/app/config/schema/schema.json @@ -618,7 +618,7 @@ "notifications_i2": { "columns": [ "subject_group_id" ] }, "notifications_i3": { "columns": [ "recipient_person_id" ] }, "notifications_i4": { "columns": [ "recipient_group_id" ] }, - "notifications_i5": { "columns": [ "source" ] }, + "notifications_i5": { "columns": [ { "name": "source", "length": 255 } ] }, "notifications_i6": { "needed": false, "columns": [ "actor_person_id" ] }, "notifications_i7": { "needed": false, "columns": [ "resolver_person_id" ] } } diff --git a/app/src/Lib/Util/SchemaManager.php b/app/src/Lib/Util/SchemaManager.php index e6eb9f7ac..9a2fa0b14 100644 --- a/app/src/Lib/Util/SchemaManager.php +++ b/app/src/Lib/Util/SchemaManager.php @@ -165,6 +165,13 @@ protected function processSchema( $qualifiedTableName = $this->conn->qualifyTableName($tablePrefix.$tName); $table = $schema->createTable($qualifiedTableName); + // Defer index/unique creation until the very end of table definition + $pendingIndexes = []; + $pendingUniques = []; + + // Counter for auto-generated "_imN" index names (shared across MVEA + sourced) + $imIndexCounter = 1; + foreach($tCfg->columns as $cName => $cCfg) { // We allow "inherited" definitions from the fieldLibrary, so merge together // the configurations (if appropriate) @@ -258,11 +265,21 @@ protected function processSchema( // have UUIDs, and there may be cases where skeletal records are created // before an object is fully active. $table->addColumn("uuid", "guid", ['notnull' => false]); - $table->addIndex(["uuid"], $tablePrefix.$tName."_id1"); + $pendingIndexes[] = [ + 'columns' => ["uuid"], + 'name' => $tablePrefix.$tName."_id1", + 'flags' => [], + 'options' => [] + ]; // While cri consists of deployer specific values, we want it searchable $table->addColumn("cri", "string", ['notnull' => false]); - $table->addIndex(["cri"], $tablePrefix.$tName."_id2"); + $pendingIndexes[] = [ + 'columns' => ["cri"], + 'name' => $tablePrefix.$tName."_id2", + 'flags' => [], + 'options' => [] + ]; // Flag indicating $table->addColumn("do_not_clone", "boolean", ['notnull' => false]); @@ -276,7 +293,12 @@ protected function processSchema( // foreign key, since minimally it crosses a CO (which would violate AR-GMR-2) // and maybe crosses CMPs (in which case it's a foreign key to another database). $table->addColumn("originalid", "integer", ['notnull' => false]); - $table->addIndex(["originalid"], $tablePrefix.$tName."_icb", [], []); + $pendingIndexes[] = [ + 'columns' => ["originalid"], + 'name' => $tablePrefix.$tName."_icb", + 'flags' => [], + 'options' => [] + ]; } // (For Registry) If MVEA models are specified, emit the appropriate @@ -284,8 +306,6 @@ protected function processSchema( // case the table has composite indexes referencing MVEA columns. if(!empty($tCfg->mvea)) { - $i = 1; - foreach($tCfg->mvea as $m) { $mColumn = $m . "_id"; $fkTable = \Cake\Utility\Inflector::tableize($m); @@ -294,7 +314,13 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn($mColumn, "integer", ['notnull' => false]); $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $tablePrefix.$tName . "_" . $mColumn . "_fkey"); - $table->addIndex([$mColumn], $tablePrefix.$tName . "_im" . $i++); + + $pendingIndexes[] = [ + 'columns' => [$mColumn], + 'name' => $tablePrefix.$tName . "_im" . $imIndexCounter++, + 'flags' => [], + 'options' => [] + ]; } // MVEA tables also support frozen flags @@ -313,10 +339,50 @@ protected function processSchema( $flags = []; $options = []; + // Support schema.json formats: + // - columns: ["colA", "colB"] + // - columns: [{"name":"colA","length":255}, "colB"] + $indexColumns = []; + $lengths = []; + $hasLengths = false; + + foreach($iCfg->columns as $col) { + if(is_string($col)) { + $indexColumns[] = $col; + $lengths[] = null; + } elseif(is_object($col) && !empty($col->name) && is_string($col->name)) { + $indexColumns[] = $col->name; + + if(isset($col->length)) { + $lengths[] = (int)$col->length; + $hasLengths = true; + } else { + $lengths[] = null; + } + } else { + throw new \RuntimeException(__d('error', 'schema.parse', ['Invalid index column definition for ' . $tName . '.' . $iName])); + } + } + + // Doctrine DBAL uses the 'lengths' option for MySQL prefix indexes + if($hasLengths && method_exists($this->conn, 'isMySQL') && $this->conn->isMySQL()) { + $options['lengths'] = $lengths; + } + if(isset($iCfg->unique) && $iCfg->unique) { - $table->addUniqueConstraint($iCfg->columns, $tablePrefix.$iName, $flags, $options); + $pendingUniques[] = [ + 'columns' => $indexColumns, + 'name' => $tablePrefix.$iName, + 'flags' => $flags, + 'options' => $options + ]; } else { - $table->addIndex($iCfg->columns, $tablePrefix.$iName, $flags, $options); + $pendingIndexes[] = [ + 'columns' => $indexColumns, + 'name' => $tablePrefix.$iName, + 'flags' => $flags, + 'options' => $options + ]; } } } @@ -332,7 +398,13 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn($sColumn, "integer", ['notnull' => false]); $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $tablePrefix.$tName . "_" . $sColumn . "_fkey"); - $table->addIndex([$sColumn], $tablePrefix.$tName . "_im" . $i++); + + $pendingIndexes[] = [ + 'columns' => [$sColumn], + 'name' => $tablePrefix.$tName . "_im" . $imIndexCounter++, + 'flags' => [], + 'options' => [] + ]; } // If this table uses TreeBehavior, emit the appropriate columnsand indexes. @@ -347,11 +419,21 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn("parent_id", "integer", ['notnull' => false]); $table->addForeignKeyConstraint($foreignTableName, ["parent_id"], ['id'], [], $tablePrefix.$tName . "_parent_id_fkey"); - $table->addIndex(["parent_id"], $tablePrefix.$tName."_it1"); + $pendingIndexes[] = [ + 'columns' => ["parent_id"], + 'name' => $tablePrefix.$tName."_it1", + 'flags' => [], + 'options' => [] + ]; // Add the other columns $table->addColumn("lft", "integer", ['notnull' => false]); - $table->addIndex(["lft"], $tablePrefix.$tName."_it2"); + $pendingIndexes[] = [ + 'columns' => ["lft"], + 'name' => $tablePrefix.$tName."_it2", + 'flags' => [], + 'options' => [] + ]; $table->addColumn("rght", "integer", ['notnull' => false]); } @@ -373,7 +455,21 @@ protected function processSchema( $table->addColumn("actor_identifier", "string", ['length' => 256, 'notnull' => false]); $table->addForeignKeyConstraint($table, [$clColumn], ['id'], [], $tName . "_" . $clColumn . "_fkey"); - $table->addIndex([$clColumn], $tablePrefix.$tName . "_icl", [], []); + $pendingIndexes[] = [ + 'columns' => [$clColumn], + 'name' => $tablePrefix.$tName . "_icl", + 'flags' => [], + 'options' => [] + ]; + } + + // Create deferred unique constraints and indexes last (so they can refer to any columns) + foreach($pendingUniques as $u) { + $table->addUniqueConstraint($u['columns'], $u['name'], $u['flags'], $u['options']); + } + + foreach($pendingIndexes as $idx) { + $table->addIndex($idx['columns'], $idx['name'], $idx['flags'], $idx['options']); } } @@ -410,7 +506,7 @@ protected function processSchema( // Remove the DROP SEQUENCE statements in $fromSql because they're Postgres automagic // being misinterpreted. (Note toSaveSql might mask this now.) // XXX Maybe debug and file a PR to not emit DROP SEQUENCE on PG for autoincrementesque fields? - if($this->io) $io->out("Skipping sequence drop"); + if($this->io) $this->io->out("Skipping sequence drop"); } else { if(!$diffOnly) { $stmt = $this->conn->executeQuery($sql); From ef37a57f5c5a7405103d0cadee42f04ba0a00cff Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 13 Jun 2026 17:43:20 +0300 Subject: [PATCH 2/4] Fix constraint name length limitations for MySql:shorten constraint names --- app/src/Lib/Util/SchemaManager.php | 91 +++++++++++++++++++++++++++--- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/app/src/Lib/Util/SchemaManager.php b/app/src/Lib/Util/SchemaManager.php index 9a2fa0b14..4bcc192c4 100644 --- a/app/src/Lib/Util/SchemaManager.php +++ b/app/src/Lib/Util/SchemaManager.php @@ -246,13 +246,17 @@ protected function processSchema( if(isset($colCfg->foreignkey)) { $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$colCfg->foreignkey->table); + + $fkName = $tablePrefix.$tName . "_" . $cName . "_fkey"; + $fkName = $this->normalizeConstraintName($fkName); + $table->addForeignKeyConstraint($foreignTableName, [$cName], [$colCfg->foreignkey->column], [], // We name our foreign keys the same way they // were previously named by adodb - $tablePrefix.$tName . "_" . $cName . "_fkey"); + $fkName); } } @@ -313,7 +317,11 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn($mColumn, "integer", ['notnull' => false]); - $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $tablePrefix.$tName . "_" . $mColumn . "_fkey"); + + $fkName = $tablePrefix.$tName . "_" . $mColumn . "_fkey"; + $fkName = $this->normalizeConstraintName($fkName); + + $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $fkName); $pendingIndexes[] = [ 'columns' => [$mColumn], @@ -397,7 +405,11 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn($sColumn, "integer", ['notnull' => false]); - $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $tablePrefix.$tName . "_" . $sColumn . "_fkey"); + + $fkName = $tablePrefix.$tName . "_" . $sColumn . "_fkey"; + $fkName = $this->normalizeConstraintName($fkName); + + $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $fkName); $pendingIndexes[] = [ 'columns' => [$sColumn], @@ -418,7 +430,12 @@ protected function processSchema( // Insert a foreign key to this model and index it $table->addColumn("parent_id", "integer", ['notnull' => false]); - $table->addForeignKeyConstraint($foreignTableName, ["parent_id"], ['id'], [], $tablePrefix.$tName . "_parent_id_fkey"); + + $fkName = $tablePrefix.$tName . "_parent_id_fkey"; + $fkName = $this->normalizeConstraintName($fkName); + + $table->addForeignKeyConstraint($foreignTableName, ["parent_id"], ['id'], [], $fkName); + $pendingIndexes[] = [ 'columns' => ["parent_id"], 'name' => $tablePrefix.$tName."_it1", @@ -453,8 +470,12 @@ protected function processSchema( $table->addColumn("revision", "integer", ['notnull' => false]); $table->addColumn("deleted", "boolean", ['notnull' => false]); $table->addColumn("actor_identifier", "string", ['length' => 256, 'notnull' => false]); - - $table->addForeignKeyConstraint($table, [$clColumn], ['id'], [], $tName . "_" . $clColumn . "_fkey"); + + $fkName = $tName . "_" . $clColumn . "_fkey"; + $fkName = $this->normalizeConstraintName($fkName); + + $table->addForeignKeyConstraint($table, [$clColumn], ['id'], [], $fkName); + $pendingIndexes[] = [ 'columns' => [$clColumn], 'name' => $tablePrefix.$tName . "_icl", @@ -527,4 +548,60 @@ protected function processSchema( // but so far we don't have an example indicating it's needed. } -} \ No newline at end of file + /** + * Shorten a constraint name to satisfy MySQL identifier limits (64 chars). + * Postgres is not affected, so callers should use this only for MySQL. + * + * @param string $constraintName + * @param int $maxLength + * @return string + */ + protected function shortenConstraintName(string $constraintName, int $maxLength = 64): string { + if(strlen($constraintName) <= $maxLength) { + return $constraintName; + } + + $listOfWords = explode('_', $constraintName); + + foreach($listOfWords as $idx => $word) { + if(strlen($word) > 3) { + $partialPostfix = substr($word, 3); + $partialPostfix = str_ireplace(['a','e','i','o','u',' '], '', $partialPostfix); + $listOfWords[$idx] = substr($word, 0, 3) . $partialPostfix; + } + } + + $newName = implode('_', $listOfWords); + + if(strlen($newName) <= $maxLength) { + return $newName; + } + + // Final fallback: keep prefix and add a deterministic hash suffix to avoid collisions. + $hash = substr(sha1($constraintName), 0, 12); + $suffix = '_' . $hash; + + $keep = $maxLength - strlen($suffix); + if($keep < 1) { + // Defensive: should never happen with maxLength=64, but ensure non-empty. + return substr($hash, 0, $maxLength); + } + + return substr($newName, 0, $keep) . $suffix; + } + + /** + * Normalize a constraint name for the active database. + * + * @param string $constraintName + * @return string + */ + protected function normalizeConstraintName(string $constraintName): string { + if(method_exists($this->conn, 'isMySQL') && $this->conn->isMySQL()) { + return $this->shortenConstraintName($constraintName, 64); + } + + return $constraintName; + } + +} From 78ac736f744dbc0fbe2d5bf1b5434469c8f771aa Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 13 Jun 2026 18:09:13 +0300 Subject: [PATCH 3/4] Fix ext_identity_sources_records_i3 MySql key length --- app/config/schema/schema.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/config/schema/schema.json b/app/config/schema/schema.json index 173eb1114..5688e1a78 100644 --- a/app/config/schema/schema.json +++ b/app/config/schema/schema.json @@ -933,7 +933,13 @@ "indexes": { "ext_identity_sources_records_i1": { "columns": [ "external_identity_source_id" ] }, "ext_identity_sources_records_i2": { "columns": [ "external_identity_id" ] }, - "ext_identity_sources_records_i3": { "columns": [ "external_identity_source_id", "source_key" ] }, + "ext_identity_sources_records_i3": { + "comment": "MySQL/MariaDB: InnoDB max key length (often 3072 bytes) + utf8mb4 (4 bytes/char) requires prefix indexing. We use source_key(767) here. If you change charset/row_format, re-check SHOW VARIABLES LIKE 'innodb_%'; and verify CREATE INDEX succeeds.", + "columns": [ + "external_identity_source_id", + { "name": "source_key", "length": 767 } + ] + }, "ext_identity_sources_records_i4": { "needed": false, "columns": [ "adopted_person_id" ] } } }, From d915102bbe38dbc71a63dd8ca8c3eaa760553848 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Sat, 13 Jun 2026 18:37:32 +0300 Subject: [PATCH 4/4] MySql issue: Fix ApiConnector api_source_records_i2 use a prefix index length to ensure CREATE INDEX succeeds --- app/availableplugins/ApiConnector/config/plugin.json | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/availableplugins/ApiConnector/config/plugin.json b/app/availableplugins/ApiConnector/config/plugin.json index 50a98e0c8..9d9721475 100644 --- a/app/availableplugins/ApiConnector/config/plugin.json +++ b/app/availableplugins/ApiConnector/config/plugin.json @@ -38,7 +38,13 @@ }, "indexes": { "api_source_records_i1": { "columns": [ "api_source_id" ] }, - "api_source_records_i2": { "columns": [ "api_source_id", "source_key" ] } + "api_source_records_i2": { + "comment": "MySQL/MariaDB compatibility: source_key can be large (utf8mb4 up to 4 bytes/char), so indexing the full 1024 characters can exceed InnoDB’s max index key length (commonly 3072 bytes) in composite indexes. We keep the column at 1024 for storage, but use a prefix index length to ensure CREATE INDEX succeeds across MySQL variants.", + "columns": [ + "api_source_id", + { "name": "source_key", "length": 191 } + ] + } }, "changelog": false, "clone_relation": true