diff --git a/app/src/Lib/Traits/PipelineTrait.php b/app/src/Lib/Traits/PipelineTrait.php new file mode 100644 index 000000000..e4be01c34 --- /dev/null +++ b/app/src/Lib/Traits/PipelineTrait.php @@ -0,0 +1,291 @@ +toArray(); + + // This list is a combination of eliminating fields that create + // noise in change detection for History creation, as well as + // functional attributes that cause problems if set (eg: frozen). + unset( + $newdata['id'], + $newdata['external_identity_id'], + $newdata['external_identity_role_id'], + $newdata['actor_identifier'], + $newdata['created'], + $newdata['deleted'], + $newdata['frozen'], + $newdata['full_name'], + // XXX we temporarily filter manager and sponsor identifiers because + // we haven't yet implemented support for mapping them + // sponsor/manager mapping CFM-33 - relateed to mapExternalIdentityRoles() + $newdata['manager_identifier'], + $newdata['sponsor_identifier'], + $newdata['modified'], + $newdata['primary_name'], + $newdata['revision'], + $newdata['role_key'], + // We don't want status for the External Identity, and we handle it + // specially for External Identity Roles + $newdata['status'] + ); + + // Get the list of "visible" fields -- this should correlate with the + // set of fields defined on the entity, whether or not they are populated, + // including metadata fields. + + $visible = $entity->getVisible(); + + // Timestamps are FrozenTime objects in the entity data, and is_scalar + // will filter them out, so convert them to strings + + foreach(['valid_from', 'valid_through'] as $attr) { + if(in_array($attr, $visible)) { + if(!empty($entity->$attr)) { + $newdata[$attr] = $entity->$attr->i18nFormat('yyyy-MM-dd HH:mm:ss'); + } else { + // Populate a blank value so removal works correctly (but don't inject + // the fields to models that don't have them) + $newdata[$attr] = ""; + } + } + } + + // This will remove anything that isn't stringy + return array_filter($newdata, 'is_scalar'); + } + + /** + * Map entity data returned from an EIS Backend to the CO. + * + * @since COmanage Registry v5.0.0 + * @param Pipeline $pipeline Pipeline + * @param array $eisAttributes Attributes provided by EIS Backend + * @return array Attributes adjusted for the CO + */ + + protected function mapAttributesToCO( + Pipeline $pipeline, + ?array $eisAttributes + ): array { + if(empty($eisAttributes)) { return []; } + + $ret = [ 'source_key' => (string)$eisAttributes['source_key'] ]; + + // date_of_birth + $ret = array_merge($ret, $this->mapDateOfBirth($eisAttributes)); + + // typed lists + $ret = array_merge($ret, $this->mapTypedAttributesList( + $pipeline->co_id, + $eisAttributes, + ['addresses','email_addresses','identifiers','names','pronouns','telephone_numbers','urls'], + )); + + // ad_hoc_attributes pass-through + if(!empty($eisAttributes['ad_hoc_attributes'])) { + $ret['ad_hoc_attributes'] = $eisAttributes['ad_hoc_attributes']; + } + + // external_identity_roles + $ret = array_merge($ret, $this->mapExternalIdentityRoles( + $pipeline->co_id, + $pipeline->sync_affiliation_type_id, + $eisAttributes, + )); + + return $ret; + } + + /** + * Map date_of_birth attribute from EIS format to CO format + * + * @param ?array $eisAttributes Array of attributes from EIS backend + * @return array Array containing mapped date_of_birth or empty array + * @since COmanage Registry v5.2.0 + */ + protected function mapDateOfBirth(?array $eisAttributes): array { + $ret = []; + if(!empty($eisAttributes['date_of_birth'])) { + $dob = \DateTimeImmutable::createFromFormat('Y-m-d', $eisAttributes['date_of_birth']); + if($dob) { + $ret['date_of_birth'] = $dob->format('Y-m-d'); + } + } + return $ret; + } + + /** + * Map an Identifier of the configured type to a Person ID. + * + * @since COmanage Registry v5.0.0 + * @param int $typeId Identifier Type ID + * @param string $identifier Identifier + * @return int|null Person ID + */ + + protected function mapIdentifier(int $typeId, string $identifier): ?int { + try { + $Identifiers = TableRegistry::getTableLocator()->get('Identifiers'); + + return $Identifiers->lookupPerson($typeId, $identifier); + } + catch(\Exception $e) { + return null; + } + } + + /** + * Map typed attribute lists from EIS format to CO format + * + * @param int $coId CO ID + * @param ?array $attributes Array of attributes + * @return array Array of mapped typed attributes + * @since COmanage Registry v5.2.0 + */ + protected function mapTypedAttributesList(int $coId, ?array $attributes, array $mvModels): array { + if(empty($attributes) || empty($mvModels)) { + return []; + } + + $typeOfRecord = match(true) { + isset($attributes['source_key']) => 'EIS', + isset($attributes['role_key']) => 'EIS Role', + default => 'Unknown' + }; + + $ret = []; + foreach($mvModels as $m) { + if(!empty($attributes[$m])) { + foreach($attributes[$m] as $attr) { + $copy = $attr; + try { + $copy['type_id'] = $this->Cos->Types->getTypeId( + $coId, + Inflector::camelize($m).".type", + $attr['type'] + ); + unset($copy['type']); + $ret[$m][] = $copy; + } + catch(\Exception $e) { + $this->llog( + 'error', + "Failed to map $attr type \"" . $attr['type'] . "\" to a valid Type ID for $typeOfRecord record " + . ($attributes['source_key'] ?? $attributes['role_key'] ?? 'unknown') + . ", skipping" + ); + } + } + } + } + return $ret; + } + + + /** + * Map External Identity Roles from an EIS Record to CO roles. + * + * @param int $coId CO ID + * @param int|null $syncAffiliationTypeId Default affiliation type ID to use + * @param ?array $eisAttributes Array of role attributes from EIS backend + * @return array Array of mapped CO role data + * @since COmanage Registry v5.2.0 + */ + protected function mapExternalIdentityRoles(int $coId, ?int $syncAffiliationTypeId, ?array $eisAttributes): array { + $ret = []; + if(empty($eisAttributes['external_identity_roles'])) { + return $ret; + } + foreach($eisAttributes['external_identity_roles'] as $role) { + $rolecopy = []; + // Basic fields for the role + foreach($role as $attr => $val) { + if(is_array($val)) { continue; } + if($attr == 'role_key') { + $rolecopy['role_key'] = (string)$val; + } elseif($attr == 'affiliation') { + $rolecopy['affiliation_type_id'] = $this->Cos->Types->getTypeId( + $coId, + 'PersonRoles.affiliation_type', + $val + ); + } elseif($attr == 'status') { + $rolecopy['status'] = $val == ExternalIdentityStatusEnum::Deleted ? ExternalIdentityStatusEnum::Archived : $val; + } else { + // XXX need to add sponsor/manager mapping CFM-33; remove from duplicateFilterEntityData + $rolecopy[$attr] = $val; + } + } + + // The pipeline affiliation type ID configuration always takes precedence + // XXX Consider adding a default affiliation in the configuration. Currently, + // if neither the pipeline affiliation type ID nor the source affiliation is provided, + // the save operation will fail ORM validation. + $rolecopy['affiliation_type_id'] = $syncAffiliationTypeId ?? $rolecopy['affiliation_type_id'] ?? null; + + // Map typed attributes, linked multi-value models + $typed = $this->mapTypedAttributesList($coId, $role, ['addresses','telephone_numbers']); + // Pass the typed attributes through to the role copy + foreach ($typed as $k => $v) { + if (!empty($v)) { + $rolecopy[$k] = $v; + } + } + + // Add hoc attributes + if(!empty($role['ad_hoc_attributes'])) { + $rolecopy['ad_hoc_attributes'] = $role['ad_hoc_attributes']; + } + $ret['external_identity_roles'][] = $rolecopy; + } + + return $ret; + } +} diff --git a/app/src/Model/Table/PipelinesTable.php b/app/src/Model/Table/PipelinesTable.php index 433c78d44..8335d1bc5 100644 --- a/app/src/Model/Table/PipelinesTable.php +++ b/app/src/Model/Table/PipelinesTable.php @@ -29,18 +29,11 @@ namespace App\Model\Table; -use Cake\ORM\Query; -use Cake\ORM\RulesChecker; use Cake\ORM\Table; use Cake\ORM\TableRegistry; use Cake\Utility\Hash; use Cake\Utility\Inflector; use Cake\Validation\Validator; -use \App\Model\Entity\ExternalIdentity; -use \App\Model\Entity\ExternalIdentitySource; -use \App\Model\Entity\ExtIdentitySourceRecord; -use \App\Model\Entity\Person; -use \App\Model\Entity\Pipeline; use \App\Lib\Enum\ActionEnum; use \App\Lib\Enum\DeletedRoleStatusEnum; use \App\Lib\Enum\ExternalIdentityStatusEnum; @@ -50,6 +43,11 @@ use \App\Lib\Enum\StatusEnum; use \App\Lib\Enum\SuspendableStatusEnum; use \App\Lib\Util\StringUtilities; +use \App\Model\Entity\ExtIdentitySourceRecord; +use \App\Model\Entity\ExternalIdentity; +use \App\Model\Entity\ExternalIdentitySource; +use \App\Model\Entity\Person; +use \App\Model\Entity\Pipeline; class PipelinesTable extends Table { use \App\Lib\Traits\AutoViewVarsTrait; @@ -61,7 +59,8 @@ class PipelinesTable extends Table { use \App\Lib\Traits\PrimaryLinkTrait; use \App\Lib\Traits\TableMetaTrait; use \App\Lib\Traits\ValidationTrait; - + use \App\Lib\Traits\PipelineTrait; + /** * Perform Cake Model initialization. * @@ -463,70 +462,6 @@ protected function dispatchFlanges( return $ret; } - /** - * Copy the data from an entity and filter metadata, returning an array - * suitable for creating a new entity. Related models are also removed. - * - * @since COmanage Registry v5.0.0 - * @param Entity $entity Entity to copy - * @return array Array of filtered entity data - */ - - protected function duplicateFilterEntityData($entity): array { - // There's some overlap with TableMetaTrait::filterMetadataFields... - - $newdata = $entity->toArray(); - - // This list is a combination of eliminating fields that create - // noise in change detection for History creation, as well as - // functional attributes that cause problems if set (eg: frozen). - unset( - $newdata['id'], - $newdata['external_identity_id'], - $newdata['external_identity_role_id'], - $newdata['actor_identifier'], - $newdata['created'], - $newdata['deleted'], - $newdata['frozen'], - $newdata['full_name'], - // XXX we temporarily filter manager and sponsor identifiers because - // we haven't yet implemented support for mapping them - $newdata['manager_identifier'], - $newdata['sponsor_identifier'], - $newdata['modified'], - $newdata['primary_name'], - $newdata['revision'], - $newdata['role_key'], - // We don't want status for the External Identity, and we handle it - // specially for External Identity Roles - $newdata['status'] - ); - - // Get the list of "visible" fields -- this should correlate with the - // set of fields defined on the entity, whether or not they are populated, - // including metadata fields. - - $visible = $entity->getVisible(); - - // Timestamps are FrozenTime objects in the entity data, and is_scalar - // will filter them out, so convert them to strings - - foreach(['valid_from', 'valid_through'] as $attr) { - if(in_array($attr, $visible)) { - if(!empty($entity->$attr)) { - $newdata[$attr] = $entity->$attr->i18nFormat('yyyy-MM-dd HH:mm:ss'); - } else { - // Populate a blank value so removal works correctly (but don't inject - // the fields to models that don't have them) - $newdata[$attr] = ""; - } - } - } - - // This will remove anything that isn't stringy - return array_filter($newdata, 'is_scalar'); - } - /** * Execute the specified Pipeline on the provided EIS data. * @@ -853,194 +788,6 @@ protected function manageEISRecord( ]; } - /** - * Map entity data returned from an EIS Backend to the CO. - * - * @since COmanage Registry v5.0.0 - * @param Pipeline $pipeline Pipeline - * @param array $eisAttributes Attributes provided by EIS Backend - * @return array Attributes adjusted for the CO - */ - - protected function mapAttributesToCO( - Pipeline $pipeline, - ?array $eisAttributes - ): array { - // Check if we'er handling a deleted record - if(empty($eisAttributes)) { - return []; - } - - // We explicitly list the valid models, which will effectively filter - // out any unsupported noise from the backend. (Unsupported attributes - // will be ignored on save or throw errors.) - - $ret = [ - // Make sure source_key is a string - 'source_key' => (string)$eisAttributes['source_key'] - ]; - - if(!empty($eisAttributes['date_of_birth'])) { - // While we're here, make sure it's in YYYY-MM-DD format. This should fail - // if the inbound attribute is invalid. - $dob = \DateTimeImmutable::createFromFormat('Y-m-d', $eisAttributes['date_of_birth']); - - if($dob) { - $ret['date_of_birth'] = $dob->format('Y-m-d'); - } - } - - foreach([ - 'addresses', - 'email_addresses', - 'identifiers', - 'names', - 'pronouns', - 'telephone_numbers', - 'urls' - ] as $m) { - if(!empty($eisAttributes[$m])) { - foreach($eisAttributes[$m] as $attr) { - $copy = $attr; - - // Map the type string to a type ID. If we fail to map the string, - // log an error but keep going. - - try { - $copy['type_id'] = $this->Cos->Types - ->getTypeId( - $pipeline->co_id, - Inflector::camelize($m).".type", - $attr['type'] - ); - unset($copy['type']); - $ret[$m][] = $copy; - } - catch(\Exception $e) { - // If we can't find a type we can't insert this record - $this->llog('error', "Failed to map $attr type \"" . $attr['type'] . "\" to a valid Type ID for EIS record " . $eisAttributes['source_key'] . ", skipping"); - } - } - } - } - - // ad_hoc_attributes require no special handling - if(!empty($eisAttributes['ad_hoc_attributes'])) { - $ret['ad_hoc_attributes'] = $eisAttributes['ad_hoc_attributes']; - } - - if(!empty($eisAttributes['external_identity_roles'])) { - foreach($eisAttributes['external_identity_roles'] as $role) { - $rolecopy = []; - - // Start with the single value attributes - foreach($role as $attr => $val) { - if(is_array($val)) { - // This is a related model, skip it for now - continue; - } - - if($attr == 'role_key') { - // Make sure the role key is a string - $rolecopy['role_key'] = (string)$val; - } elseif($attr == 'affiliation') { - // Affiliation needs to be mapped - - $rolecopy['affiliation_type_id'] = $this->Cos->Types - ->getTypeId( - $pipeline->co_id, - 'PersonRoles.affiliation_type', - $val - ); - } elseif($attr == 'status') { - // Generally we'll let validation and recalcuation handle status, - // but if for some reason the backend asserts Deleted (which is used - // internally as a sync status, and so is not permitted to be asserted - // by the backend) we'll just convert it to Archived rather than futz - // around with context specific validation rules. - - // Strictly speaking this is not an Application Rule since backends - // shouldn't assert Deleted status so we don't need to document a - // behavior for what happens when they do. - - $rolecopy['status'] = - $val == ExternalIdentityStatusEnum::Deleted - ? ExternalIdentityStatusEnum::Archived - : $val; - } else { -// XXX need to add sponsor/manager mapping CFM-33; remove from duplicateFilterEntityData - // Just copy the attribute - $rolecopy[$attr] = $val; - } - } - - // If no affiliation type was provided by the backend, - // use the Pipeline's configuration - if(empty($rolecopy['affiliation_type_id'])) { - $rolecopy['affiliation_type_id'] = $pipeline->sync_affiliation_type_id; - } - - // Now handle related models - foreach([ - 'addresses', - 'telephone_numbers' - ] as $m) { - if(!empty($role[$m])) { - foreach($role[$m] as $attr) { - $copy = $attr; - - // Map the type string to a type ID. If we fail to map the string, - // log an error but keep going. - - try { - $copy['type_id'] = $this->Cos->Types - ->getTypeId( - $pipeline->co_id, - Inflector::camelize($m).".type", - $attr['type'] - ); - unset($copy['type']); - $rolecopy[$m][] = $copy; - } - catch(\Exception $e) { - $this->llog('error', "Failed to map $attr type \"" . $attr['type'] . "\" to a valid Type ID for EIS role record " . $role['role_key'] . ", skipping"); - } - } - } - } - - // And just copy ad hoc attributes - if(!empty($role['ad_hoc_attributes'])) { - $rolecopy['ad_hoc_attributes'] = $role['ad_hoc_attributes']; - } - - $ret['external_identity_roles'][] = $rolecopy; - } - } - - return $ret; - } - - /** - * Map an Identifier of the configured type to a Person ID. - * - * @since COmanage Registry v5.0.0 - * @param int $typeId Identifier Type ID - * @param string $identifier Identifier - * @return int Person ID - */ - - protected function mapIdentifier(int $typeId, string $identifier): ?int { - try { - $Identifiers = TableRegistry::getTableLocator()->get('Identifiers'); - - return $Identifiers->lookupPerson($typeId, $identifier); - } - catch(\Exception $e) { - return null; - } - } - /** * Pipeline step to obtain a Person associated with the $eisRecord, possibly * by executing the Match Strategy.