diff --git a/app/src/Controller/StandardController.php b/app/src/Controller/StandardController.php index 7d3583784..4a55b5365 100644 --- a/app/src/Controller/StandardController.php +++ b/app/src/Controller/StandardController.php @@ -646,6 +646,8 @@ public function index() { $table = $this->$modelsName; // $tableName = models $tableName = $table->getTable(); + // AutoViewVarsTrait + $this->populateAutoViewVars(); // Construct the Query $query = $this->getIndexQuery(); diff --git a/app/src/Lib/Enum/MetadataModesEnum.php b/app/src/Lib/Enum/MetadataModesEnum.php new file mode 100644 index 000000000..2ed33a61c --- /dev/null +++ b/app/src/Lib/Enum/MetadataModesEnum.php @@ -0,0 +1,36 @@ +getSearchableAttributes($this->name, - $this->viewBuilder() - ->getVar('vv_tz')); + $searchableAttributes = $table->getSearchableAttributes($this->name, $this->viewBuilder()); // Extra View Variables foreach ($table->getViewVars() as $key => $variable) { diff --git a/app/src/Lib/Traits/PipelineTrait.php b/app/src/Lib/Traits/PipelineTrait.php new file mode 100644 index 000000000..04cd8458e --- /dev/null +++ b/app/src/Lib/Traits/PipelineTrait.php @@ -0,0 +1,274 @@ +toArray(); + + // This list is a combination of eliminating fields that create + // noise in change detection for History creation, as well as + // functional attributes that cause problems if set (eg: frozen). + foreach ($this->getMetadataFields(MetadataModesEnum::Pipeline) as $field) { + if(isset($newdata[$field])) { + unset($newdata[$field]); + } + } + + // Get the list of "visible" fields -- this should correlate with the + // set of fields defined on the entity, whether or not they are populated, + // including metadata fields. + + $visible = $entity->getVisible(); + + // Timestamps are FrozenTime objects in the entity data, and is_scalar + // will filter them out, so convert them to strings + + foreach(['valid_from', 'valid_through'] as $attr) { + if(in_array($attr, $visible)) { + if(!empty($entity->$attr)) { + $newdata[$attr] = $entity->$attr->i18nFormat('yyyy-MM-dd HH:mm:ss'); + } else { + // Populate a blank value so removal works correctly (but don't inject + // the fields to models that don't have them) + $newdata[$attr] = ""; + } + } + } + + // This will remove anything that isn't stringy + return array_filter($newdata, 'is_scalar'); + } + + /** + * Map entity data returned from an EIS Backend to the CO. + * + * @since COmanage Registry v5.0.0 + * @param Pipeline $pipeline Pipeline + * @param array $eisAttributes Attributes provided by EIS Backend + * @return array Attributes adjusted for the CO + */ + + protected function mapAttributesToCO( + Pipeline $pipeline, + ?array $eisAttributes + ): array { + if(empty($eisAttributes)) { return []; } + + $ret = [ 'source_key' => (string)$eisAttributes['source_key'] ]; + + // date_of_birth + $ret = array_merge($ret, $this->mapDateOfBirth($eisAttributes)); + + // typed lists + $ret = array_merge($ret, $this->mapTypedAttributesList( + $pipeline->co_id, + $eisAttributes, + ['addresses','email_addresses','identifiers','names','pronouns','telephone_numbers','urls'], + )); + + // ad_hoc_attributes pass-through + if(!empty($eisAttributes['ad_hoc_attributes'])) { + $ret['ad_hoc_attributes'] = $eisAttributes['ad_hoc_attributes']; + } + + // external_identity_roles + $ret = array_merge($ret, $this->mapExternalIdentityRoles( + $pipeline->co_id, + $pipeline->sync_affiliation_type_id, + $eisAttributes, + )); + + return $ret; + } + + /** + * Map date_of_birth attribute from EIS format to CO format + * + * @param ?array $eisAttributes Array of attributes from EIS backend + * @return array Array containing mapped date_of_birth or empty array + * @since COmanage Registry v5.2.0 + */ + protected function mapDateOfBirth(?array $eisAttributes): array { + $ret = []; + if(!empty($eisAttributes['date_of_birth'])) { + $dob = \DateTimeImmutable::createFromFormat('Y-m-d', $eisAttributes['date_of_birth']); + if($dob) { + $ret['date_of_birth'] = $dob->format('Y-m-d'); + } + } + return $ret; + } + + /** + * Map an Identifier of the configured type to a Person ID. + * + * @since COmanage Registry v5.0.0 + * @param int $typeId Identifier Type ID + * @param string $identifier Identifier + * @return int|null Person ID + */ + + protected function mapIdentifier(int $typeId, string $identifier): ?int { + try { + $Identifiers = TableRegistry::getTableLocator()->get('Identifiers'); + + return $Identifiers->lookupPerson($typeId, $identifier); + } + catch(\Exception $e) { + return null; + } + } + + /** + * Map typed attribute lists from EIS format to CO format + * + * @param int $coId CO ID + * @param ?array $attributes Array of attributes + * @return array Array of mapped typed attributes + * @since COmanage Registry v5.2.0 + */ + protected function mapTypedAttributesList(int $coId, ?array $attributes, array $mvModels): array { + if(empty($attributes) || empty($mvModels)) { + return []; + } + + $typeOfRecord = match(true) { + isset($attributes['source_key']) => 'EIS', + isset($attributes['role_key']) => 'EIS Role', + default => 'Unknown' + }; + + $ret = []; + foreach($mvModels as $m) { + if(!empty($attributes[$m])) { + foreach($attributes[$m] as $attr) { + $copy = $attr; + try { + $copy['type_id'] = $this->Cos->Types->getTypeId( + $coId, + Inflector::camelize($m).".type", + $attr['type'] + ); + unset($copy['type']); + $ret[$m][] = $copy; + } + catch(\Exception $e) { + $this->llog( + 'error', + "Failed to map $attr type \"" . $attr['type'] . "\" to a valid Type ID for $typeOfRecord record " + . ($attributes['source_key'] ?? $attributes['role_key'] ?? 'unknown') + . ", skipping" + ); + } + } + } + } + return $ret; + } + + + /** + * Map External Identity Roles from an EIS Record to CO roles. + * + * @param int $coId CO ID + * @param int|null $syncAffiliationTypeId Default affiliation type ID to use + * @param ?array $eisAttributes Array of role attributes from EIS backend + * @return array Array of mapped CO role data + * @since COmanage Registry v5.2.0 + */ + protected function mapExternalIdentityRoles(int $coId, ?int $syncAffiliationTypeId, ?array $eisAttributes): array { + $ret = []; + if(empty($eisAttributes['external_identity_roles'])) { + return $ret; + } + foreach($eisAttributes['external_identity_roles'] as $role) { + $rolecopy = []; + // Basic fields for the role + foreach($role as $attr => $val) { + if(is_array($val)) { continue; } + if($attr == 'role_key') { + $rolecopy['role_key'] = (string)$val; + } elseif($attr == 'affiliation') { + $rolecopy['affiliation_type_id'] = $this->Cos->Types->getTypeId( + $coId, + 'PersonRoles.affiliation_type', + $val + ); + } elseif($attr == 'status') { + $rolecopy['status'] = $val == ExternalIdentityStatusEnum::Deleted ? ExternalIdentityStatusEnum::Archived : $val; + } else { + // XXX need to add sponsor/manager mapping CFM-33; remove from duplicateFilterEntityData + $rolecopy[$attr] = $val; + } + } + + // The pipeline affiliation type ID configuration always takes precedence + // XXX Consider adding a default affiliation in the configuration. Currently, + // if neither the pipeline affiliation type ID nor the source affiliation is provided, + // the save operation will fail ORM validation. + $rolecopy['affiliation_type_id'] = $syncAffiliationTypeId ?? $rolecopy['affiliation_type_id'] ?? null; + + // Map typed attributes, linked multi-value models + $typed = $this->mapTypedAttributesList($coId, $role, ['addresses','telephone_numbers']); + // Pass the typed attributes through to the role copy + foreach ($typed as $k => $v) { + if (!empty($v)) { + $rolecopy[$k] = $v; + } + } + + // Add hoc attributes + if(!empty($role['ad_hoc_attributes'])) { + $rolecopy['ad_hoc_attributes'] = $role['ad_hoc_attributes']; + } + $ret['external_identity_roles'][] = $rolecopy; + } + + return $ret; + } +} diff --git a/app/src/Lib/Traits/SearchFilterTrait.php b/app/src/Lib/Traits/SearchFilterTrait.php index 34f1d5bc4..c1641cb45 100644 --- a/app/src/Lib/Traits/SearchFilterTrait.php +++ b/app/src/Lib/Traits/SearchFilterTrait.php @@ -33,9 +33,10 @@ use Bake\Utility\Model\AssociationFilter; use Cake\Database\Expression\QueryExpression; use Cake\Http\ServerRequest; +use Cake\I18n\FrozenTime; use Cake\ORM\Query; use Cake\Utility\Inflector; -use Cake\I18n\FrozenTime; +use Cake\View\ViewBuilder; trait SearchFilterTrait { /** @@ -233,11 +234,12 @@ public function getFilterConfig(): array { * * @since COmanage Registry v5.0.0 * @param string $controller Controller name - * @param DateTimeZone $vv_tz Current time zone, if known + * @param ViewBuilder $viewBuilder View Builder Object * @return array Array of permitted search attributes and configuration elements needed for display */ - public function getSearchableAttributes(string $controller, \DateTimeZone $vv_tz=null): array { + public function getSearchableAttributes(string $controller, ViewBuilder $viewBuilder): array { + $vv_tz = $viewBuilder->getVar('vv_tz') ?? null; $modelname = Inflector::classify(Inflector::underscore($controller)); $filterConfig = $this->getFilterConfig(); diff --git a/app/src/Lib/Traits/TableMetaTrait.php b/app/src/Lib/Traits/TableMetaTrait.php index a91aa4eac..e36b1d66c 100644 --- a/app/src/Lib/Traits/TableMetaTrait.php +++ b/app/src/Lib/Traits/TableMetaTrait.php @@ -29,12 +29,27 @@ namespace App\Lib\Traits; +use App\Lib\Enum\MetadataModesEnum; +use App\Lib\Util\ArrayUtilities; use Cake\ORM\TableRegistry; use Cake\Utility\Inflector; use App\Lib\Enum\TableTypeEnum; use App\Lib\Util\StringUtilities; trait TableMetaTrait { + /** + * Base common fields excluding the model foreign key which is dynamic per table. + */ + protected const COMMON_META_FIELDS = [ + 'actor_identifier', + 'created', + 'deleted', + 'id', + 'modified', + 'revision', + ]; + + // What type of Table is this? private $tableType = null; @@ -163,7 +178,8 @@ protected function filterMetadataForCopy( } /** - * Filter metadata fields. + * Introspect a Table’s schema and associations to classify columns into “metadata” vs “non-metadata”, + * returning column names mapped to types, not values. Used primarily for UI and search/filter building. * * @since COmanage Registry v5.0.0 * @return array An array of columns distinguished in metadata and non-metadata @@ -172,57 +188,15 @@ protected function filterMetadataForCopy( protected function filterMetadataFields() { // Get the list of columns $coltype = $this->getSchema()->typeMap(); - $entity = $this->getEntityClass(); - $entity_namespace = explode('\\', $entity); - $modelName = end($entity_namespace); - - // Get the list of belongs_to associations and construct an exclude array - $assc_keys = []; - foreach ($this->associations() as $assc) { - if($assc->type() === "manyToOne") { - $assc_keys[] = Inflector::underscore(Inflector::classify($assc->getClassName())) . "_id"; - } - } - // Map the model (eg: Person) to the changelog key (person_id) - $mfk = Inflector::underscore($modelName) . "_id"; - $meta_fields = [ - ...$assc_keys, - $mfk, - 'actor_identifier', - // 'provisioning_target_id', - 'created', // todo: I might need to revisit this. We might want to filter according to date in some occassions. Like petitions - 'deleted', - 'id', - 'modified', - 'revision', - 'lft', // XXX For now i skip lft.rght column for tree structures - 'rght', - 'api_key', - // XXX maybe replace this with a regex, source_*_id? - 'source_ad_hoc_attribute_id', - 'source_address_id', - 'source_email_address_id', - 'source_external_identity_id', - 'source_identifier_id', - 'source_name_id', - 'source_pronoun_id', - 'source_telephone_number_id', - 'source_url_id', - 'owners_group_id', - 'enrollee_person_id', - 'petitioner_person_id', - 'authz_group_id', - 'authz_cou_id', - 'redirect_on_finalize', - 'collect_enrollee_email' - ]; + // Central source for metadata fields + $meta_fields = $this->getMetadataFields(); $newa = array(); foreach($coltype as $clmn => $type) { - // XXX We need to check if the type is an enum or plain string. The enum is a string - // but during filtering we do not use like but eq - // If required we can treat enum types as string types + // XXX We need to check if the type is an enum or plain string. The enum is a string, + // but during filtering we do not use the ORM `like` method but `eq` method. + // If required, we can treat enum types as string types $fType = $type; // XXX Cakephp Inflector's camel-case function returns a Pascal case string while the variable function @@ -243,7 +217,141 @@ protected function filterMetadataFields() { return $newa ?? []; } - + + /** + * Return the list of metadata field names for this Table by merging association-derived + * keys, changelog/model keys, and known curated metadata fields, plus additional fields + * filtered during duplication flows. + * + * @since COmanage Registry v5.2.0 + * @return array + * @todo consider caching the output + */ + protected function getMetadataFields(string $mode = MetadataModesEnum::Search): array { + $modelName = StringUtilities::tableToEntityName($this); + + // Association foreign keys (belongsTo/manyToOne) + $asscKeys = []; + foreach ($this->associations() as $assc) { + if($assc->type() === "manyToOne") { + $asscKeys[] = Inflector::underscore(Inflector::classify($assc->getClassName())) . "_id"; + } + } + // Map the model (eg: Person) to the changelog key (person_id) + $mfk = Inflector::underscore($modelName) . "_id"; + + // Common metadata fields used across UI/search and schema classification + $common = array_merge([$mfk], self::COMMON_META_FIELDS); + + // Dynamically detect source_*_id keys + $sourceKeys = $this->detectSourceForeignKeys(); + + // At the moment we exclude them for all modes + $foreignKeysNonSource = [ + // Group filter block + 'owners_group_id', + // Petitioner filter block + 'enrollee_person_id', + 'petitioner_person_id', + 'authz_group_id', + 'authz_cou_id', + // Pipeline filter Entity Data + 'external_identity_id', + 'external_identity_role_id' + ]; + + // No required for search, but we want to include in the list + $searchSpecific = [ + 'redirect_on_finalize', + 'manager_person_id', + 'sponsor_person_id', + 'collect_enrollee_email', + 'api_key', + 'lft', + 'rght', + ]; + + // Fields filtered during duplication/pipeline flows + // This list is a combination of eliminating fields that create + // noise in change detection for History creation, as well as + // functional attributes that cause problems if set (eg: frozen). + $dupSpecific = [ + 'frozen', + 'full_name', + // XXX we temporarily filter manager and sponsor identifiers because + // we haven't yet implemented support for mapping them + // sponsor/manager mapping CFM-33 - related to mapExternalIdentityRoles() + 'manager_identifier', + 'sponsor_identifier', + 'primary_name', + 'role_key', + // We don't want status for the External Identity, and we handle it + // especially for External Identity Roles + 'status', + ]; + + $modeSpecific = match ($mode) { + MetadataModesEnum::Search => array_merge($searchSpecific, $asscKeys), + MetadataModesEnum::Pipeline => $dupSpecific, + MetadataModesEnum::CoreApi => [], + default => throw new \InvalidArgumentException("Unknown Metadata Mode: $mode"), + }; + + return ArrayUtilities::uniquePreserveOrder(array_merge( + $common, + $sourceKeys, + $foreignKeysNonSource, + $modeSpecific + )); + + } + + /** + * Detect columns that look like "source_*_id". + * + * - Convention match: /^source_[a-z_]+_id$/ + * - Optional validation: ensure there is a BelongsTo association whose foreignKey matches + * + * @param bool $requireAssociation If true, only include keys backed by a matching BelongsTo association + * @return array + * @since COmanage Registry v5.2.0 + * */ + protected function detectSourceForeignKeys(bool $requireAssociation = false): array + { + $columns = $this->getSchema()->columns(); + + // 1) Convention-based candidates + $candidates = array_values(array_filter($columns, static function (string $col): bool { + return (bool)preg_match('/^source_[a-z_]+_id$/', $col); + })); + + if (!$requireAssociation) { + return $candidates; + } + + // 2) Validate candidates against BelongsTo associations and foreignKey + $validated = []; + foreach ($candidates as $col) { + // remove "_id" + $base = substr($col, 0, -3); + // e.g., source_ad_hoc_attribute -> SourceAdHocAttribute + $alias = Inflector::camelize(Inflector::singularize($base)); + + if ($this->associations()->has($alias)) { + $assoc = $this->associations()->get($alias); + + $isBelongsTo = $assoc instanceof BelongsTo; + $fkMatches = method_exists($assoc, 'getForeignKey') ? ($assoc->getForeignKey() === $col) : false; + + if ($isBelongsTo && $fkMatches) { + $validated[] = $col; + } + } + } + + return $validated; + } + /** * Determine if this Table represents Registry artifacts. * diff --git a/app/src/Lib/Util/ArrayUtilities.php b/app/src/Lib/Util/ArrayUtilities.php index 416465a10..2552cfab3 100644 --- a/app/src/Lib/Util/ArrayUtilities.php +++ b/app/src/Lib/Util/ArrayUtilities.php @@ -54,4 +54,27 @@ public static function sortAssociative(array $assocArray): array { return $assocArray; } + /** + * Preserve order while removing duplicates. + * + * @param array $items + * @return array + * @since COmanage Registry v5.0.0 + * + */ + + public static function uniquePreserveOrder(array $items): array + { + $seen = []; + $out = []; + foreach ($items as $item) { + if (!isset($seen[$item])) { + $seen[$item] = true; + $out[] = $item; + } + } + return $out; + } + + } \ No newline at end of file diff --git a/app/src/Lib/Util/StringUtilities.php b/app/src/Lib/Util/StringUtilities.php index 571e39b60..5f4ff5589 100644 --- a/app/src/Lib/Util/StringUtilities.php +++ b/app/src/Lib/Util/StringUtilities.php @@ -132,6 +132,33 @@ public static function columnKey( return ($cfield !== $c) ? $cfield : \Cake\Utility\Inflector::humanize($c); } + /** + * Convert a database column name to an auto view variable name + * + * @param string $column Database column name to convert + * @return string Converted view variable name + * @since COmanage Registry v5.2.0 + */ + public static function columnToAutoViewVar(string $column): string { + // strip trailing _type_id if present, else _id + $base = preg_replace('/_type_id$/', '', $column); + $base = preg_replace('/_id$/', '', $base); + + // if it originally was *_type_id, we want “...Types” + $isType = str_ends_with($column, '_type_id'); + + // convert snake_case to camelCase + $camel = Inflector::variable($base); // eg email_address -> emailAddress + + if ($isType) { + // ensure a plural sense by appending “Types” (matches repo usage) + return $camel . 'Types'; + } + + // default pluralization + return Inflector::variable(Inflector::pluralize($base)); + } + /** * Determine the class basename of a Cake Entity. * diff --git a/app/src/Model/Table/CousTable.php b/app/src/Model/Table/CousTable.php index 0afbe7534..c572adea2 100644 --- a/app/src/Model/Table/CousTable.php +++ b/app/src/Model/Table/CousTable.php @@ -116,12 +116,6 @@ public function initialize(array $config): void { ]); $this->setFilterConfig([ - 'identifier' => [ - 'type' => 'string', - 'model' => 'Identifiers', - 'active' => true, - 'order' => 4 - ], 'parent_id' => [ // We want to keep the default column configuration and add extra functionality. // Here the extra functionality is additional to select options since the parent_id diff --git a/app/src/View/Helper/FilterHelper.php b/app/src/View/Helper/FilterHelper.php index 5c7b169ae..b6258bb14 100644 --- a/app/src/View/Helper/FilterHelper.php +++ b/app/src/View/Helper/FilterHelper.php @@ -29,9 +29,10 @@ namespace App\View\Helper; +use App\Lib\Util\StringUtilities; use Cake\Collection\Collection; use Cake\ORM\TableRegistry; -use Cake\Utility\{Inflector, Hash}; +use Cake\Utility\Hash; use Cake\View\Helper; class FilterHelper extends Helper @@ -50,7 +51,7 @@ public function calculateFieldParams(string $columnName, string $label): array{ $populatedVarData = $this->getView()->get( // The populated variables are in plural while the column names are singular // Convention: It is a prerequisite that the vvar should be the plural of the column name - lcfirst(Inflector::pluralize(Inflector::camelize($columnName))) + StringUtilities::columnToAutoViewVar($columnName) ); // Field options diff --git a/app/templates/element/filter/topButtons.php b/app/templates/element/filter/topButtons.php index b56107bd8..91ecbdeb0 100644 --- a/app/templates/element/filter/topButtons.php +++ b/app/templates/element/filter/topButtons.php @@ -27,6 +27,7 @@ declare(strict_types = 1); +use App\Lib\Util\StringUtilities; use Cake\Utility\{Inflector, Hash}; // Construct aria-controls string @@ -41,7 +42,7 @@ // The populated variables are in plural while the column names are singular // Convention: It is a prerequisite that the vvar should be the plural of the column name -$populated_vvar = lcfirst(Inflector::pluralize(Inflector::camelize($key))); +$populated_vvar = StringUtilities::columnToAutoViewVar($key); $button_label = 'Range'; if(isset($$populated_vvar) && isset($$populated_vvar[$params])) { // Get label name from AutoViewPopulated vars