[NO TICKET] Refactoring_search_improvements_fixes

app/src/Controller/StandardController.php

@@ -646,6 +646,8 @@ public function index() {
     $table = $this->$modelsName;
     // $tableName = models
     $tableName = $table->getTable();
+    // AutoViewVarsTrait
+    $this->populateAutoViewVars();
     // Construct the Query
     $query = $this->getIndexQuery();
 

app/src/Lib/Enum/MetadataModesEnum.php

@@ -0,0 +1,36 @@
+<?php
+/**
+ * COmanage Registry Metadata Modes Enum
+ *
+ * Portions licensed to the University Corporation for Advanced Internet
+ * Development, Inc. ("UCAID") under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * UCAID licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @link          https://www.internet2.edu/comanage COmanage Project
+ * @package       registry
+ * @since         COmanage Registry v5.0.0
+ * @license       Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+ */
+
+declare(strict_types = 1);
+
+namespace App\Lib\Enum;
+
+class MetadataModesEnum extends StandardEnum {
+  const Search      = 'S';
+  const Pipeline    = 'P';
+  const CoreApi     = 'C';
+}

app/src/Lib/Traits/AutoViewVarsTrait.php

@@ -33,7 +33,6 @@
 use App\Lib\Util\FunctionUtilities;
 use App\Lib\Util\StringUtilities;
 use Cake\ORM\TableRegistry;
-use Cake\Utility\Hash;
 
 trait AutoViewVarsTrait {
   // Array (and configuration) of view variables to automatically populate

app/src/Lib/Traits/IndexQueryTrait.php

@@ -180,9 +180,7 @@ public function getIndexQuery(bool $pickerMode = false, array $requestParams = [
 
     // Attributes to search for
     if(method_exists($table, 'getSearchableAttributes')) {
-      $searchableAttributes = $table->getSearchableAttributes($this->name,
-                                                              $this->viewBuilder()
-                                                                   ->getVar('vv_tz'));
+      $searchableAttributes = $table->getSearchableAttributes($this->name, $this->viewBuilder());
 
       // Extra View Variables
       foreach ($table->getViewVars() as $key => $variable) {

app/src/Lib/Traits/PipelineTrait.php

@@ -0,0 +1,274 @@
+<?php
+
+/**
+ * COmanage Registry Pipeline Trait
+ *
+ * Portions licensed to the University Corporation for Advanced Internet
+ * Development, Inc. ("UCAID") under one or more contributor license agreements.
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * UCAID licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * @link          https://www.internet2.edu/comanage COmanage Project
+ * @package       registry
+ * @since         COmanage Registry v5.2.0
+ * @license       Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
+ */
+
+declare(strict_types = 1);
+
+namespace App\Lib\Traits;
+
+use App\Lib\Enum\ExternalIdentityStatusEnum;
+use App\Lib\Enum\MetadataModesEnum;
+use App\Model\Entity\Pipeline;
+use Cake\ORM\TableRegistry;
+use Cake\Utility\Inflector;
+
+trait PipelineTrait {
+  use TableMetaTrait;
+  /**
+   * Copy the data from an entity and filter metadata, returning an array
+   * suitable for creating a new entity. Related models are also removed.
+   *
+   * @since  COmanage Registry v5.0.0
+   * @param  Entity $entity       Entity to copy
+   * @return array                Array of filtered entity data
+   */
+
+  protected function duplicateFilterEntityData($entity): array {
+    $newdata = $entity->toArray();
+
+    // This list is a combination of eliminating fields that create
+    // noise in change detection for History creation, as well as
+    // functional attributes that cause problems if set (eg: frozen).
+    foreach ($this->getMetadataFields(MetadataModesEnum::Pipeline) as $field) {
+      if(isset($newdata[$field])) {
+        unset($newdata[$field]);
+      }
+    }
+
+    // Get the list of "visible" fields -- this should correlate with the
+    // set of fields defined on the entity, whether or not they are populated,
+    // including metadata fields.
+
+    $visible = $entity->getVisible();
+
+    // Timestamps are FrozenTime objects in the entity data, and is_scalar
+    // will filter them out, so convert them to strings
+
+    foreach(['valid_from', 'valid_through'] as $attr) {
+      if(in_array($attr, $visible)) {
+        if(!empty($entity->$attr)) {
+          $newdata[$attr] = $entity->$attr->i18nFormat('yyyy-MM-dd HH:mm:ss');
+        } else {
+          // Populate a blank value so removal works correctly (but don't inject
+          // the fields to models that don't have them)
+          $newdata[$attr] = "";
+        }
+      }
+    }
+
+    // This will remove anything that isn't stringy
+    return array_filter($newdata, 'is_scalar');
+  }
+
+  /**
+   * Map entity data returned from an EIS Backend to the CO.
+   *
+   * @since  COmanage Registry v5.0.0
+   * @param  Pipeline  $pipeline        Pipeline
+   * @param  array     $eisAttributes   Attributes provided by EIS Backend
+   * @return array                      Attributes adjusted for the CO
+   */
+
+  protected function mapAttributesToCO(
+    Pipeline          $pipeline,
+    ?array            $eisAttributes
+  ): array {
+    if(empty($eisAttributes)) { return []; }
+
+    $ret = [ 'source_key' => (string)$eisAttributes['source_key'] ];
+
+    // date_of_birth
+    $ret = array_merge($ret, $this->mapDateOfBirth($eisAttributes));
+
+    // typed lists
+    $ret = array_merge($ret, $this->mapTypedAttributesList(
+      $pipeline->co_id,
+      $eisAttributes,
+      ['addresses','email_addresses','identifiers','names','pronouns','telephone_numbers','urls'],
+    ));
+
+    // ad_hoc_attributes pass-through
+    if(!empty($eisAttributes['ad_hoc_attributes'])) {
+      $ret['ad_hoc_attributes'] = $eisAttributes['ad_hoc_attributes'];
+    }
+
+    // external_identity_roles
+    $ret = array_merge($ret, $this->mapExternalIdentityRoles(
+      $pipeline->co_id,
+      $pipeline->sync_affiliation_type_id,
+      $eisAttributes,
+    ));
+
+    return $ret;
+  }
+
+  /**
+   * Map date_of_birth attribute from EIS format to CO format
+   *
+   * @param  ?array $eisAttributes Array of attributes from EIS backend
+   * @return array  Array containing mapped date_of_birth or empty array
+   * @since  COmanage Registry v5.2.0
+   */
+  protected function mapDateOfBirth(?array $eisAttributes): array {
+    $ret = [];
+    if(!empty($eisAttributes['date_of_birth'])) {
+      $dob = \DateTimeImmutable::createFromFormat('Y-m-d', $eisAttributes['date_of_birth']);
+      if($dob) {
+        $ret['date_of_birth'] = $dob->format('Y-m-d');
+      }
+    }
+    return $ret;
+  }
+
+  /**
+   * Map an Identifier of the configured type to a Person ID.
+   *
+   * @since  COmanage Registry v5.0.0
+   * @param  int    $typeId     Identifier Type ID
+   * @param  string $identifier Identifier
+   * @return int|null           Person ID
+   */
+
+  protected function mapIdentifier(int $typeId, string $identifier): ?int {
+    try {
+      $Identifiers = TableRegistry::getTableLocator()->get('Identifiers');
+
+      return $Identifiers->lookupPerson($typeId, $identifier);
+    }
+    catch(\Exception $e) {
+      return null;
+    }
+  }
+
+  /**
+   * Map typed attribute lists from EIS format to CO format
+   *
+   * @param int $coId CO ID
+   * @param  ?array $attributes Array of attributes
+   * @return array  Array of mapped typed attributes
+   * @since  COmanage Registry v5.2.0
+   */
+  protected function mapTypedAttributesList(int $coId, ?array $attributes, array $mvModels): array {
+    if(empty($attributes) || empty($mvModels)) {
+      return [];
+    }
+
+    $typeOfRecord = match(true) {
+      isset($attributes['source_key']) => 'EIS',
+      isset($attributes['role_key']) => 'EIS Role',
+      default => 'Unknown'
+    };
+
+    $ret = [];
+    foreach($mvModels as $m) {
+      if(!empty($attributes[$m])) {
+        foreach($attributes[$m] as $attr) {
+          $copy = $attr;
+          try {
+            $copy['type_id'] = $this->Cos->Types->getTypeId(
+              $coId,
+              Inflector::camelize($m).".type",
+              $attr['type']
+            );
+            unset($copy['type']);
+            $ret[$m][] = $copy;
+          }
+          catch(\Exception $e) {
+            $this->llog(
+              'error',
+              "Failed to map $attr type \"" . $attr['type'] . "\" to a valid Type ID for $typeOfRecord record "
+              . ($attributes['source_key'] ?? $attributes['role_key'] ?? 'unknown')
+              . ", skipping"
+            );
+          }
+        }
+      }
+    }
+    return $ret;
+  }
+
+
+  /**
+   * Map External Identity Roles from an EIS Record to CO roles.
+   *
+   * @param int $coId CO ID
+   * @param int|null $syncAffiliationTypeId Default affiliation type ID to use
+   * @param  ?array $eisAttributes Array of role attributes from EIS backend
+   * @return array   Array of mapped CO role data
+   * @since  COmanage Registry v5.2.0
+   */
+  protected function mapExternalIdentityRoles(int $coId, ?int $syncAffiliationTypeId, ?array $eisAttributes): array {
+    $ret = [];
+    if(empty($eisAttributes['external_identity_roles'])) {
+      return $ret;
+    }
+    foreach($eisAttributes['external_identity_roles'] as $role) {
+      $rolecopy = [];
+      // Basic fields for the role
+      foreach($role as $attr => $val) {
+        if(is_array($val)) { continue; }
+        if($attr == 'role_key') {
+          $rolecopy['role_key'] = (string)$val;
+        } elseif($attr == 'affiliation') {
+          $rolecopy['affiliation_type_id'] = $this->Cos->Types->getTypeId(
+            $coId,
+            'PersonRoles.affiliation_type',
+            $val
+          );
+        } elseif($attr == 'status') {
+          $rolecopy['status'] = $val == ExternalIdentityStatusEnum::Deleted ? ExternalIdentityStatusEnum::Archived : $val;
+        } else {
+          // XXX need to add sponsor/manager mapping CFM-33; remove from duplicateFilterEntityData
+          $rolecopy[$attr] = $val;
+        }
+      }
+
+      // The pipeline affiliation type ID configuration always takes precedence
+      // XXX Consider adding a default affiliation in the configuration. Currently,
+      // if neither the pipeline affiliation type ID nor the source affiliation is provided,
+      // the save operation will fail ORM validation.
+      $rolecopy['affiliation_type_id'] = $syncAffiliationTypeId ?? $rolecopy['affiliation_type_id'] ?? null;
+
+      // Map typed attributes, linked multi-value models
+      $typed = $this->mapTypedAttributesList($coId, $role, ['addresses','telephone_numbers']);
+      // Pass the typed attributes through to the role copy
+      foreach ($typed as $k => $v) {
+        if (!empty($v)) {
+          $rolecopy[$k] = $v;
+        }
+      }
+
+      // Add hoc attributes
+      if(!empty($role['ad_hoc_attributes'])) {
+        $rolecopy['ad_hoc_attributes'] = $role['ad_hoc_attributes'];
+      }
+      $ret['external_identity_roles'][] = $rolecopy;
+    }
+
+    return $ret;
+  }
+}

app/src/Lib/Traits/SearchFilterTrait.php

@@ -33,9 +33,10 @@
 use Bake\Utility\Model\AssociationFilter;
 use Cake\Database\Expression\QueryExpression;
 use Cake\Http\ServerRequest;
+use Cake\I18n\FrozenTime;
 use Cake\ORM\Query;
 use Cake\Utility\Inflector;
-use Cake\I18n\FrozenTime;
+use Cake\View\ViewBuilder;
 
 trait SearchFilterTrait {
   /**
@@ -233,11 +234,12 @@ public function getFilterConfig(): array {
    *
    * @since  COmanage Registry v5.0.0
    * @param  string       $controller Controller name
-   * @param  DateTimeZone $vv_tz      Current time zone, if known
+   * @param  ViewBuilder  $viewBuilder View Builder Object
    * @return array Array of permitted search attributes and configuration elements needed for display
    */
 
-  public function getSearchableAttributes(string $controller, \DateTimeZone $vv_tz=null): array {
+  public function getSearchableAttributes(string $controller, ViewBuilder $viewBuilder): array {
+    $vv_tz = $viewBuilder->getVar('vv_tz') ?? null;
     $modelname = Inflector::classify(Inflector::underscore($controller));
     $filterConfig = $this->getFilterConfig();