CFM-505_Github_actions_bootstraping_and_comanage_setup_testing

.github/workflows/registry-ci.yml

@@ -0,0 +1,256 @@
+name: COmanage Registry setup + PHPUnit (multi-PHP, multi-DB)
+
+on:
+  workflow_dispatch:
+  push:
+  pull_request:
+
+jobs:
+  setup-and-test:
+    name: setup-and-test (php=${{ matrix.php }}, db=${{ matrix.db.engine }})
+    runs-on:
+      - codebuild-comanage-pipeline-${{ github.run_id }}-${{ github.run_attempt }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        php: ["8.4"]
+        db:
+          - engine: postgres
+            image: postgres:16-alpine
+            port: 5432
+            health_cmd: 'pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB"'
+          - engine: mysql
+            image: mysql:8.0
+            port: 3306
+            health_cmd: 'mysqladmin ping -h 127.0.0.1 -uroot -p"$MYSQL_ROOT_PASSWORD" --silent'
+          - engine: mariadb
+            image: mariadb:11
+            port: 3306
+            health_cmd: 'mariadb-admin ping -h 127.0.0.1 -uroot -p"$MARIADB_ROOT_PASSWORD" --silent'
+
+    # Exactly ONE service container per matrix run (the image changes)
+    services:
+      db:
+        image: ${{ matrix.db.image }}
+        # Publish the DB port so the job can connect via Docker-host networking.
+        # NOTE: If your runner executes steps in a container, 127.0.0.1 won't work;
+        # we compute the Docker host gateway IP in a later step.
+        ports:
+          - ${{ matrix.db.port }}:${{ matrix.db.port }}
+        env:
+          # Postgres vars (used only by postgres image)
+          POSTGRES_DB: registry_test
+          POSTGRES_USER: test_user
+          POSTGRES_PASSWORD: test_password
+
+          # MySQL vars (used only by mysql image; mariadb image ignores these)
+          MYSQL_DATABASE: registry_test
+          MYSQL_USER: test_user
+          MYSQL_PASSWORD: test_password
+          MYSQL_ROOT_PASSWORD: root_password
+
+          # MariaDB vars (used only by mariadb image)
+          MARIADB_DATABASE: registry_test
+          MARIADB_USER: test_user
+          MARIADB_PASSWORD: test_password
+          MARIADB_ROOT_PASSWORD: root_password
+        options: >-
+          --health-cmd "${{ matrix.db.health_cmd }}"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 20
+
+    env:
+      COMANAGE_REGISTRY_DIR: /srv/comanage-registry
+
+      # Matrix DB selection for this run
+      DB_ENGINE: ${{ matrix.db.engine }}
+
+      # Values used by your PHPUnit setup test
+      COMANAGE_REGISTRY_ADMIN_GIVEN_NAME_TEST: Admin
+      COMANAGE_REGISTRY_ADMIN_FAMILY_NAME_TEST: User
+      COMANAGE_REGISTRY_ADMIN_USERNAME_TEST: admin
+      COMANAGE_REGISTRY_SECURITY_SALT_TEST: phpunit-security-salt
+
+      # DB credentials/name (host/port will be set dynamically in a step)
+      COMANAGE_REGISTRY_DATABASE_TEST: registry_test
+      COMANAGE_REGISTRY_DATABASE_USER_TEST: test_user
+      COMANAGE_REGISTRY_DATABASE_USER_PASSWORD_TEST: test_password
+      COMANAGE_REGISTRY_DATABASE_PERSISTENT_TEST: "false"
+
+    steps:
+      - name: Upgrade OS packages
+        shell: bash
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo apt-get upgrade -y
+
+      - name: Checkout repository at the exact commit
+        shell: bash
+        run: |
+          set -euxo pipefail
+          git clone "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" "${COMANAGE_REGISTRY_DIR}"
+          cd "${COMANAGE_REGISTRY_DIR}"
+          git fetch --no-tags --prune --depth=1 origin "${GITHUB_SHA}"
+          git checkout --force "${GITHUB_SHA}"
+          git rev-parse HEAD
+
+      - name: Install PHP ${{ matrix.php }} and extensions
+        shell: bash
+        run: |
+          set -euxo pipefail
+          sudo apt-get install -y --no-install-recommends \
+            software-properties-common ca-certificates gnupg
+          sudo add-apt-repository -y ppa:ondrej/php
+          sudo apt-get update
+
+          PHP_VER="${{ matrix.php }}"
+          sudo apt-get install -y --no-install-recommends \
+            php${PHP_VER}-cli \
+            php${PHP_VER}-mbstring \
+            php${PHP_VER}-intl \
+            php${PHP_VER}-ldap \
+            php${PHP_VER}-xml \
+            php${PHP_VER}-zip \
+            php${PHP_VER}-pdo \
+            php${PHP_VER}-mysql \
+            php${PHP_VER}-pgsql \
+            php${PHP_VER}-gd \
+            php${PHP_VER}-xsl \
+            php${PHP_VER}-memcached \
+            php${PHP_VER}-curl
+
+          sudo update-alternatives --set php /usr/bin/php${PHP_VER}
+          sudo ln -sf /usr/bin/php${PHP_VER} /usr/local/bin/php
+
+          echo "PHP_VER=${PHP_VER}" >> "$GITHUB_ENV"
+          echo "/usr/local/bin" >> "$GITHUB_PATH"
+
+      - name: Install OS packages needed for setup
+        shell: bash
+        run: |
+          set -euxo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            wget curl tar ca-certificates \
+            git unzip \
+            libicu-dev \
+            libldap2-dev \
+            libxml2 \
+            zlib1g \
+            libsodium23 \
+            libpng-dev \
+            libjpeg-dev \
+            libfreetype6-dev \
+            libxslt1.1 \
+            libmemcached11 \
+            tree 
+
+      - name: Show versions
+        shell: bash
+        run: |
+          set -euxo pipefail
+          cat /etc/os-release || true
+          uname -a
+          php -v
+          composer --version
+          docker --version
+          docker version
+          echo "DOCKER_API_VERSION=${DOCKER_API_VERSION-}"
+          echo "${DOCKER_HOST-}"
+          docker context show
+
+      - name: Wait for DB to be ready (inside the service container)
+        shell: bash
+        run: |
+          set -euxo pipefail
+          case "${DB_ENGINE}" in
+            postgres)
+              docker exec "${{ job.services.db.id }}" sh -lc 'for i in $(seq 1 60); do pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" && exit 0; sleep 1; done; exit 1'
+              ;;
+            mysql)
+              docker exec "${{ job.services.db.id }}" sh -lc 'for i in $(seq 1 60); do mysqladmin ping -h 127.0.0.1 -uroot -p"$MYSQL_ROOT_PASSWORD" --silent && exit 0; sleep 1; done; exit 1'
+              ;;
+            mariadb)
+              docker exec "${{ job.services.db.id }}" sh -lc 'for i in $(seq 1 60); do mariadb-admin ping -h 127.0.0.1 -uroot -p"$MARIADB_ROOT_PASSWORD" --silent && exit 0; sleep 1; done; exit 1'
+              ;;
+            *)
+              echo "Unknown DB_ENGINE=${DB_ENGINE}"
+              exit 1
+              ;;
+          esac
+
+      - name: Determine DB host/port for published ports (Option 1)
+        shell: bash
+        run: |
+          set -euxo pipefail
+
+          # If steps run inside a container, localhost is the *job container*.
+          # Use the default gateway (Docker host) to reach published ports.
+          if [ -f /.dockerenv ]; then
+            DB_HOST="$(ip route | awk '/default/ {print $3; exit}')"
+          else
+            DB_HOST="127.0.0.1"
+          fi
+
+          DB_PORT="${{ matrix.db.port }}"
+
+          echo "Using DB host=${DB_HOST} port=${DB_PORT} engine=${DB_ENGINE}"
+
+          {
+            echo "COMANAGE_REGISTRY_DATABASE_HOST_TEST=${DB_HOST}"
+            echo "COMANAGE_REGISTRY_DATABASE_PORT_TEST=${DB_PORT}"
+          } >> "$GITHUB_ENV"
+
+      - name: Smoke test DB TCP connectivity (from the job environment)
+        shell: bash
+        run: |
+          set -euxo pipefail
+          php -r '
+            $h=getenv("COMANAGE_REGISTRY_DATABASE_HOST_TEST"); $p=(int)getenv("COMANAGE_REGISTRY_DATABASE_PORT_TEST");
+            $fp=@fsockopen($h,$p,$errno,$errstr,5);
+            if(!$fp){fwrite(STDERR,"TCP connect failed to $h:$p: $errno $errstr\n"); exit(1);}
+            fclose($fp);
+            echo "TCP connect OK to $h:$p\n";
+          '
+
+      - name: Create local/config/database.php placeholder (optional)
+        shell: bash
+        run: |
+          set -euxo pipefail
+          cd "${COMANAGE_REGISTRY_DIR}/local/config"
+          sudo mkdir -p .
+          sudo tee database.php > /dev/null <<'PHP'
+          <?php
+          // Intentionally empty for CI: tests/bootstrap.php configures the 'test' datasource via env vars.
+          return [];
+          PHP
+          sudo chown www-data:www-data database.php || true
+
+      - name: Show (tree) working directory
+        working-directory: ${{ env.COMANAGE_REGISTRY_DIR }}
+        shell: bash
+        run: |
+          set -euxo pipefail
+          tree -L 3 "${COMANAGE_REGISTRY_DIR}"
+
+      - name: Composer update phpunit only
+        shell: bash
+        working-directory: ${{ env.COMANAGE_REGISTRY_DIR }}/app
+        run: |
+          set -euxo pipefail
+          composer update phpunit/phpunit --with-all-dependencies
+
+      - name: Run PHPUnit (DB_ENGINE=${{ matrix.db.engine }})
+        shell: bash
+        working-directory: ${{ env.COMANAGE_REGISTRY_DIR }}/app
+        run: |
+          set -euxo pipefail
+          echo "DB_ENGINE=${DB_ENGINE}"
+          echo "COMANAGE_REGISTRY_DATABASE_HOST_TEST=${COMANAGE_REGISTRY_DATABASE_HOST_TEST}"
+          echo "COMANAGE_REGISTRY_DATABASE_PORT_TEST=${COMANAGE_REGISTRY_DATABASE_PORT_TEST}"
+          vendor/bin/phpunit --group registry-setup --testdox
+          vendor/bin/phpunit --group registry-available-plugins --testdox
+          vendor/bin/phpunit --exclude-group registry-setup --exclude-group registry-available-plugins --testdox

app/availableplugins/ApiConnector/config/plugin.json

@@ -38,7 +38,13 @@
         },
         "indexes": {
           "api_source_records_i1": { "columns": [ "api_source_id" ] },
-          "api_source_records_i2": { "columns": [ "api_source_id", "source_key" ] }
+          "api_source_records_i2": {
+            "comment": "MySQL/MariaDB compatibility: source_key can be large (utf8mb4 up to 4 bytes/char), so indexing the full 1024 characters can exceed InnoDB’s max index key length (commonly 3072 bytes) in composite indexes. We keep the column at 1024 for storage, but use a prefix index length to ensure CREATE INDEX succeeds across MySQL variants.",
+            "columns": [
+              "api_source_id",
+              { "name": "source_key", "length": 191 }
+            ]
+          }
         },
         "changelog": false,
         "clone_relation": true

app/availableplugins/FileConnector/src/Model/Table/FileSourcesTable.php

@@ -166,9 +166,9 @@ public function inventory(
     }
 
     // The first line of a CSV v3 file is our configuration
-    fgetcsv($handle);
+    fgetcsv($handle, escape: '\\');
 
-    while(($data = fgetcsv($handle)) !== false) {
+    while(($data = fgetcsv($handle, escape: '\\')) !== false) {
       // The source key is always the first field in each line, make sure it is not empty
 
       if(!empty($data[0]) && !ctype_space($data[0])) {
@@ -306,9 +306,9 @@ protected function processChangeList(
       }
 
       // Ignore the header line
-      fgetcsv($handle);
+      fgetcsv($handle, escape: '\\');
 
-      while(($data = fgetcsv($handle)) !== false) {
+      while(($data = fgetcsv($handle, escape: '\\')) !== false) {
         // Implode the record back together for string comparison purposes.
         // This may not be the same as the original line due to quotes, etc.
         // $data[0] is the SORID
@@ -327,9 +327,9 @@ protected function processChangeList(
       }
 
       // Ignore the header line
-      fgetcsv($handle);
+      fgetcsv($handle, escape: '\\');
 
-      while(($data = fgetcsv($handle)) !== false) {
+      while(($data = fgetcsv($handle, escape: '\\')) !== false) {
         // $data[0] is the SORID
         if(array_key_exists($data[0], $knownRecords)) {
           $newData = implode(',', $data);
@@ -445,7 +445,7 @@ protected function readFieldConfig(
     }
 
     // The first line is our configuration
-    $cfg = fgetcsv($handle);
+    $cfg = fgetcsv($handle, escape: '\\');
 
     fclose($handle);
 
@@ -700,13 +700,13 @@ public function retrieve(
     // If there is more than one record, we'll return the first one we find.
 
     // The first line of a CSV v3 file is our configuration
-    fgetcsv($handle);
+    fgetcsv($handle, escape: '\\');
 
     // Set null defaults in case we don't find a matching record
     $ret['source_record'] = null;
     $ret['entity_data'] = null;
 
-    while(($data = fgetcsv($handle)) !== false) {
+    while(($data = fgetcsv($handle, escape: '\\')) !== false) {
       if($data[0] == $source_key) {
         // This is our record
 
@@ -809,9 +809,9 @@ public function search(
     }
 
     // The first line of a CSV v3 file is our configuration
-    fgetcsv($handle);
+    fgetcsv($handle, escape: '\\');
 
-    while(($data = fgetcsv($handle)) !== false) {
+    while(($data = fgetcsv($handle, escape: '\\')) !== false) {
       // strtolower, previous behavior was full string only so dupe that
 
       $match = collection($data)

app/composer.json

@@ -55,7 +55,12 @@
         "cs-check": "phpcs --colors -p  src/ tests/",
         "cs-fix": "phpcbf --colors -p src/ tests/",
         "stan": "phpstan analyse",
-        "test": "phpunit --colors=always"
+        "test": "phpunit --colors=always",
+        "pre-commit": [
+            "vendor/bin/phpunit --group registry-setup --testdox",
+            "vendor/bin/phpunit --group registry-available-plugins --testdox",
+            "vendor/bin/phpunit --exclude-group registry-setup --exclude-group registry-available-plugins --testdox"
+        ]
     },
     "prefer-stable": true,
     "config": {

app/config/routes.php

@@ -22,6 +22,7 @@
  */
 
 use Cake\Http\Middleware\BodyParserMiddleware;
+use App\Middleware\ForceJsonIfAcceptedMiddleware;
 use Cake\Http\Middleware\CsrfProtectionMiddleware;
 use Cake\Routing\Route\DashedRoute;
 use Cake\Routing\RouteBuilder;
@@ -54,12 +55,15 @@
   // BodyParserMiddleware will automatically parse JSON bodies, but we only
   // want that for API transactions, so we only apply it to the /api scope.
   $builder->registerMiddleware('bodyparser', new BodyParserMiddleware());
+  $builder->registerMiddleware('forceJsonIfAccepted', new ForceJsonIfAcceptedMiddleware());
+
   /*
    * Apply a middleware to the current route scope.
    * Requires middleware to be registered through `Application::routes()` with `registerMiddleware()`
    */
   $builder->setExtensions(['json']);
   $builder->applyMiddleware('bodyparser');
+  $builder->applyMiddleware('forceJsonIfAccepted');
   // Use setPass to make parameter show up as function parameter
   // Model specific actions, which will usually have more specific URLs:
   // Note that while the UI uses dashes in URL paths, because we use {model} for the generic