CFM-528_MySQL_database_setup_script_failures

app/availableplugins/ApiConnector/config/plugin.json

@@ -38,7 +38,13 @@
         },
         "indexes": {
           "api_source_records_i1": { "columns": [ "api_source_id" ] },
-          "api_source_records_i2": { "columns": [ "api_source_id", "source_key" ] }
+          "api_source_records_i2": {
+            "comment": "MySQL/MariaDB compatibility: source_key can be large (utf8mb4 up to 4 bytes/char), so indexing the full 1024 characters can exceed InnoDB’s max index key length (commonly 3072 bytes) in composite indexes. We keep the column at 1024 for storage, but use a prefix index length to ensure CREATE INDEX succeeds across MySQL variants.",
+            "columns": [
+              "api_source_id",
+              { "name": "source_key", "length": 191 }
+            ]
+          }
         },
         "changelog": false,
         "clone_relation": true

app/config/schema/schema.json

@@ -618,7 +618,7 @@
         "notifications_i2": { "columns": [ "subject_group_id" ] },
         "notifications_i3": { "columns": [ "recipient_person_id" ] },
         "notifications_i4": { "columns": [ "recipient_group_id" ] },
-        "notifications_i5": { "columns": [ "source" ] },
+        "notifications_i5": { "columns": [ { "name": "source", "length": 255 } ] },
         "notifications_i6": { "needed": false, "columns": [ "actor_person_id" ] },
         "notifications_i7": { "needed": false, "columns": [ "resolver_person_id" ] }
       }
@@ -933,7 +933,13 @@
       "indexes": {
         "ext_identity_sources_records_i1": { "columns": [ "external_identity_source_id" ] },
         "ext_identity_sources_records_i2": { "columns": [ "external_identity_id" ] },
-        "ext_identity_sources_records_i3": { "columns": [ "external_identity_source_id", "source_key" ] },
+        "ext_identity_sources_records_i3": {
+          "comment": "MySQL/MariaDB: InnoDB max key length (often 3072 bytes) + utf8mb4 (4 bytes/char) requires prefix indexing. We use source_key(767) here. If you change charset/row_format, re-check SHOW VARIABLES LIKE 'innodb_%'; and verify CREATE INDEX succeeds.",
+          "columns": [
+              "external_identity_source_id",
+              { "name": "source_key", "length": 767 }
+          ]
+        },
         "ext_identity_sources_records_i4": { "needed": false, "columns": [ "adopted_person_id" ] }
       }
     },

app/src/Lib/Util/SchemaManager.php

@@ -165,6 +165,13 @@ protected function processSchema(
       $qualifiedTableName = $this->conn->qualifyTableName($tablePrefix.$tName);
       $table = $schema->createTable($qualifiedTableName);
 
+      // Defer index/unique creation until the very end of table definition
+      $pendingIndexes = [];
+      $pendingUniques = [];
+
+      // Counter for auto-generated "_imN" index names (shared across MVEA + sourced)
+      $imIndexCounter = 1;
+
       foreach($tCfg->columns as $cName => $cCfg) {
         // We allow "inherited" definitions from the fieldLibrary, so merge together
         // the configurations (if appropriate)
@@ -239,13 +246,17 @@ protected function processSchema(
 
         if(isset($colCfg->foreignkey)) {
           $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$colCfg->foreignkey->table);
+
+          $fkName = $tablePrefix.$tName . "_" . $cName . "_fkey";
+          $fkName = $this->normalizeConstraintName($fkName);
+
           $table->addForeignKeyConstraint($foreignTableName,
                                           [$cName],
                                           [$colCfg->foreignkey->column],
                                           [],
                                           // We name our foreign keys the same way they
                                           // were previously named by adodb
-                                          $tablePrefix.$tName . "_" . $cName . "_fkey");
+                                          $fkName);
         }
       }
 
@@ -258,11 +269,21 @@ protected function processSchema(
         // have UUIDs, and there may be cases where skeletal records are created
         // before an object is fully active.
         $table->addColumn("uuid", "guid", ['notnull' => false]);
-        $table->addIndex(["uuid"], $tablePrefix.$tName."_id1");
+        $pendingIndexes[] = [
+          'columns' => ["uuid"],
+          'name' => $tablePrefix.$tName."_id1",
+          'flags' => [],
+          'options' => []
+        ];
 
         // While cri consists of deployer specific values, we want it searchable
         $table->addColumn("cri", "string", ['notnull' => false]);
-        $table->addIndex(["cri"], $tablePrefix.$tName."_id2");
+        $pendingIndexes[] = [
+          'columns' => ["cri"],
+          'name' => $tablePrefix.$tName."_id2",
+          'flags' => [],
+          'options' => []
+        ];
 
         // Flag indicating 
         $table->addColumn("do_not_clone", "boolean", ['notnull' => false]);
@@ -276,25 +297,38 @@ protected function processSchema(
         // foreign key, since minimally it crosses a CO (which would violate AR-GMR-2)
         // and maybe crosses CMPs (in which case it's a foreign key to another database).
         $table->addColumn("originalid", "integer", ['notnull' => false]);
-        $table->addIndex(["originalid"], $tablePrefix.$tName."_icb", [], []);
+        $pendingIndexes[] = [
+          'columns' => ["originalid"],
+          'name' => $tablePrefix.$tName."_icb",
+          'flags' => [],
+          'options' => []
+        ];
       }
 
       // (For Registry) If MVEA models are specified, emit the appropriate
       // columns and indexes. MVEA attributes must be added before indexes, in
       // case the table has composite indexes referencing MVEA columns.
 
       if(!empty($tCfg->mvea)) {
-        $i = 1;
-
         foreach($tCfg->mvea as $m) {
           $mColumn = $m . "_id";
           $fkTable = \Cake\Utility\Inflector::tableize($m);
           $foreignTableName = $this->conn->qualifyTableName($tablePrefix.$fkTable);
 
           // Insert a foreign key to this model and index it
           $table->addColumn($mColumn, "integer", ['notnull' => false]);
-          $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $tablePrefix.$tName . "_" . $mColumn . "_fkey");
-          $table->addIndex([$mColumn], $tablePrefix.$tName . "_im" . $i++);
+
+          $fkName = $tablePrefix.$tName . "_" . $mColumn . "_fkey";
+          $fkName = $this->normalizeConstraintName($fkName);
+
+          $table->addForeignKeyConstraint($foreignTableName, [$mColumn], ['id'], [], $fkName);
+
+          $pendingIndexes[] = [
+            'columns' => [$mColumn],
+            'name' => $tablePrefix.$tName . "_im" . $imIndexCounter++,
+            'flags' => [],
+            'options' => []
+          ];
         }
 
         // MVEA tables also support frozen flags
@@ -313,10 +347,50 @@ protected function processSchema(
           $flags = [];
           $options = [];
 
+          // Support schema.json formats:
+          // - columns: ["colA", "colB"]
+          // - columns: [{"name":"colA","length":255}, "colB"]
+          $indexColumns = [];
+          $lengths = [];
+          $hasLengths = false;
+
+          foreach($iCfg->columns as $col) {
+            if(is_string($col)) {
+              $indexColumns[] = $col;
+              $lengths[] = null;
+            } elseif(is_object($col) && !empty($col->name) && is_string($col->name)) {
+              $indexColumns[] = $col->name;
+
+              if(isset($col->length)) {
+                $lengths[] = (int)$col->length;
+                $hasLengths = true;
+              } else {
+                $lengths[] = null;
+              }
+            } else {
+              throw new \RuntimeException(__d('error', 'schema.parse', ['Invalid index column definition for ' . $tName . '.' . $iName]));
+            }
+          }
+
+          // Doctrine DBAL uses the 'lengths' option for MySQL prefix indexes
+          if($hasLengths && method_exists($this->conn, 'isMySQL') && $this->conn->isMySQL()) {
+            $options['lengths'] = $lengths;
+          }
+
           if(isset($iCfg->unique) && $iCfg->unique) {
-            $table->addUniqueConstraint($iCfg->columns, $tablePrefix.$iName, $flags, $options);
+            $pendingUniques[] = [
+              'columns' => $indexColumns,
+              'name' => $tablePrefix.$iName,
+              'flags' => $flags,
+              'options' => $options
+            ];
           } else {
-            $table->addIndex($iCfg->columns, $tablePrefix.$iName, $flags, $options);
+            $pendingIndexes[] = [
+              'columns' => $indexColumns,
+              'name' => $tablePrefix.$iName,
+              'flags' => $flags,
+              'options' => $options
+            ];
           }
         }
       }
@@ -331,8 +405,18 @@ protected function processSchema(
 
         // Insert a foreign key to this model and index it
         $table->addColumn($sColumn, "integer", ['notnull' => false]);
-        $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $tablePrefix.$tName . "_" . $sColumn . "_fkey");
-        $table->addIndex([$sColumn], $tablePrefix.$tName . "_im" . $i++);
+
+        $fkName = $tablePrefix.$tName . "_" . $sColumn . "_fkey";
+        $fkName = $this->normalizeConstraintName($fkName);
+
+        $table->addForeignKeyConstraint($foreignTableName, [$sColumn], ['id'], [], $fkName);
+
+        $pendingIndexes[] = [
+          'columns' => [$sColumn],
+          'name' => $tablePrefix.$tName . "_im" . $imIndexCounter++,
+          'flags' => [],
+          'options' => []
+        ];
       }
 
       // If this table uses TreeBehavior, emit the appropriate columnsand indexes.
@@ -346,12 +430,27 @@ protected function processSchema(
 
         // Insert a foreign key to this model and index it
         $table->addColumn("parent_id", "integer", ['notnull' => false]);
-        $table->addForeignKeyConstraint($foreignTableName, ["parent_id"], ['id'], [], $tablePrefix.$tName . "_parent_id_fkey");
-        $table->addIndex(["parent_id"], $tablePrefix.$tName."_it1");
+
+        $fkName = $tablePrefix.$tName . "_parent_id_fkey";
+        $fkName = $this->normalizeConstraintName($fkName);
+
+        $table->addForeignKeyConstraint($foreignTableName, ["parent_id"], ['id'], [], $fkName);
+
+        $pendingIndexes[] = [
+          'columns' => ["parent_id"],
+          'name' => $tablePrefix.$tName."_it1",
+          'flags' => [],
+          'options' => []
+        ];
 
         // Add the other columns
         $table->addColumn("lft", "integer", ['notnull' => false]);
-        $table->addIndex(["lft"], $tablePrefix.$tName."_it2");
+        $pendingIndexes[] = [
+          'columns' => ["lft"],
+          'name' => $tablePrefix.$tName."_it2",
+          'flags' => [],
+          'options' => []
+        ];
 
         $table->addColumn("rght", "integer", ['notnull' => false]);
       }
@@ -371,9 +470,27 @@ protected function processSchema(
         $table->addColumn("revision", "integer", ['notnull' => false]);
         $table->addColumn("deleted", "boolean", ['notnull' => false]);
         $table->addColumn("actor_identifier", "string", ['length' => 256, 'notnull' => false]);
-
-        $table->addForeignKeyConstraint($table, [$clColumn], ['id'], [], $tName . "_" . $clColumn . "_fkey");
-        $table->addIndex([$clColumn], $tablePrefix.$tName . "_icl", [], []);
+
+        $fkName = $tName . "_" . $clColumn . "_fkey";
+        $fkName = $this->normalizeConstraintName($fkName);
+
+        $table->addForeignKeyConstraint($table, [$clColumn], ['id'], [], $fkName);
+
+        $pendingIndexes[] = [
+          'columns' => [$clColumn],
+          'name' => $tablePrefix.$tName . "_icl",
+          'flags' => [],
+          'options' => []
+        ];
+      }
+
+      // Create deferred unique constraints and indexes last (so they can refer to any columns)
+      foreach($pendingUniques as $u) {
+        $table->addUniqueConstraint($u['columns'], $u['name'], $u['flags'], $u['options']);
+      }
+
+      foreach($pendingIndexes as $idx) {
+        $table->addIndex($idx['columns'], $idx['name'], $idx['flags'], $idx['options']);
       }
     }
 
@@ -410,7 +527,7 @@ protected function processSchema(
           // Remove the DROP SEQUENCE statements in $fromSql because they're Postgres automagic
           // being misinterpreted. (Note toSaveSql might mask this now.)
           // XXX Maybe debug and file a PR to not emit DROP SEQUENCE on PG for autoincrementesque fields?
-          if($this->io) $io->out("Skipping sequence drop");
+          if($this->io) $this->io->out("Skipping sequence drop");
         } else {
           if(!$diffOnly) {
             $stmt = $this->conn->executeQuery($sql);
@@ -431,4 +548,60 @@ protected function processSchema(
     // but so far we don't have an example indicating it's needed.
   }
 
-}
+  /**
+   * Shorten a constraint name to satisfy MySQL identifier limits (64 chars).
+   * Postgres is not affected, so callers should use this only for MySQL.
+   *
+   * @param string $constraintName
+   * @param int $maxLength
+   * @return string
+   */
+  protected function shortenConstraintName(string $constraintName, int $maxLength = 64): string {
+    if(strlen($constraintName) <= $maxLength) {
+      return $constraintName;
+    }
+
+    $listOfWords = explode('_', $constraintName);
+
+    foreach($listOfWords as $idx => $word) {
+      if(strlen($word) > 3) {
+        $partialPostfix = substr($word, 3);
+        $partialPostfix = str_ireplace(['a','e','i','o','u',' '], '', $partialPostfix);
+        $listOfWords[$idx] = substr($word, 0, 3) . $partialPostfix;
+      }
+    }
+
+    $newName = implode('_', $listOfWords);
+
+    if(strlen($newName) <= $maxLength) {
+      return $newName;
+    }
+
+    // Final fallback: keep prefix and add a deterministic hash suffix to avoid collisions.
+    $hash = substr(sha1($constraintName), 0, 12);
+    $suffix = '_' . $hash;
+
+    $keep = $maxLength - strlen($suffix);
+    if($keep < 1) {
+      // Defensive: should never happen with maxLength=64, but ensure non-empty.
+      return substr($hash, 0, $maxLength);
+    }
+
+    return substr($newName, 0, $keep) . $suffix;
+  }
+
+  /**
+   * Normalize a constraint name for the active database.
+   *
+   * @param string $constraintName
+   * @return string
+   */
+  protected function normalizeConstraintName(string $constraintName): string {
+    if(method_exists($this->conn, 'isMySQL') && $this->conn->isMySQL()) {
+      return $this->shortenConstraintName($constraintName, 64);
+    }
+
+    return $constraintName;
+  }
+
+}