Skip to content

Commit

Permalink
Record Adoption (CFM-396) and Relinking (CFM-125)
Browse files Browse the repository at this point in the history
  • Loading branch information
Benn Oshrin committed Apr 5, 2025
1 parent a97265f commit f2ca6f5
Show file tree
Hide file tree
Showing 42 changed files with 1,161 additions and 89 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -182,34 +182,56 @@ protected function mapApiToRegistry(string $model, array $attributes): array {
* Remove a record from the External Identity Source.
*
* @since COmanage Registry v5.0.0
* @param ExternalIdentitySource $source EIS Entity with instantiated plugin configuration
* @param string $sorId API System of Record ID
* @return bool True on success
* @param int $id Api Source ID
* @param string $sorLabel System of Record Label
* @param string $sorId API System of Record ID
* @return bool True on success
* @throws RecordNotFoundException
*/

public function remove(
\App\Model\Entity\ExternalIdentitySource $source,
string $sorId
): array {
public function remove(int $id, string $sorLabel, string $sorId): bool {
// We call this remove() so as not to interfere with the default table::delete().
$apiSource = $this->get($id, ['contain' => ['ExternalIdentitySources']]);

// Pull our configuration
$apiSource = $this->get($id, ['contain' => ['ExternalIdentitySources']]);

// Like upsert(), we don't really need $sorLabel, but we check it for
// consistency with upsert() (which also doesn't really need it).

if(empty($apiSource->external_identity_source->sor_label)
|| $apiSource->external_identity_source->sor_label != $sorLabel) {
throw new \InvalidArgumentException("Requested SOR Label $sorLabel does not match configuration");
}

// Remove the ApiSourceRecord for this $source_key from the cache

$apiSourceRecord = $this->ApiSourceRecords->find()
->where([
'api_source_id' => $source->api_source->id,
'source_key' => $sorId
])
->firstOrFail();
try {
// Start a Transaction
$cxn = $this->getConnection();
$cxn->begin();

$apiSourceRecord = $this->ApiSourceRecords->find()
->where([
'api_source_id' => $id,
'source_key' => $sorId
])
->firstOrFail();

$this->ApiSourceRecords->delete($apiSourceRecord);
$this->ApiSourceRecords->delete($apiSourceRecord);

// Run sync
$this->ExternalIdentitySources->sync($apiSource->external_identity_source_id, $sorId);

// Run sync
// XXX do we need some sort of return value to pass back in the API response?
$this->ExternalIdentitySources->sync($source->id, $sorId);
$cxn->commit();

return true;
return true;
}
catch(\Exception $e) {
$cxn->rollback();

throw $e;
}
}


Expand Down Expand Up @@ -397,8 +419,8 @@ public function upsert(
// Pull our configuration
$apiSource = $this->get($id, ['contain' => ['ExternalIdentitySources']]);

// Strictly speaking we don't need $sorid since we know which configuration
// to use from the ApiSource ID, and $sorlabel might not be unique across COs
// Strictly speaking we don't need $sorLabel since we know which configuration
// to use from the ApiSource ID, and $sorLabel might not be unique across COs
// in a multi-tenant environment. Eventually we could support multiple
// Systems of Record within the same ApiSource configuration, but for now
// we just make sure $sorLabel matches the configuration and throw an error
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ msgstr "The file \"{0}\" is not writable"
msgid "error.header"
msgstr "Did not find CSV file header"

msgid "error.header.sorid"
msgstr "Did not find SORID as first defined column, check file header definition"

msgid "field.FileProvisioners.filename"
msgstr "File Name"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,42 @@ public function buildRules(RulesChecker $rules): RulesChecker {
return $rules;
}

/**
* Obtain the full set of records from the source database.
*
* @since COmanage Registry v5.1.0
* @param ExternalIdentitySource $source External Identity Source
* @return array An array of source keys
*/

public function inventory(
\App\Model\Entity\ExternalIdentitySource $source
): array {
$ret = [];

$handle = fopen($source->file_source->filename, "r");

if(!$handle) {
throw new \RuntimeException(__d('file_connector', 'error.filename.readable', [$source->file_source->filename]));
}

// The first line of a CSV v3 file is our configuration
fgetcsv($handle);

while(($data = fgetcsv($handle)) !== false) {
// The source key is always the first field in each line

$ret[] = $data[0];
}

fclose($handle);

// It's not clear we really need to sort the array, but why not...
sort($ret);

return $ret;
}

/**
* Obtain the file field configuration.
*
Expand Down Expand Up @@ -175,6 +211,10 @@ protected function readFieldConfig(
switch(count($bits)) {
case 1:
// SORID (special case)
// While we're here check to make sure the field is as expected
if($bits[0] != 'SORID') {
throw new \RuntimeException(__d('file_connector', 'error.header.sorid'));
}
$this->fieldCfg[ $bits[0] ] = $i;
break;
case 2:
Expand Down
6 changes: 4 additions & 2 deletions app/config/schema/schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -870,12 +870,14 @@
"source_record": { "type": "text" },
"last_update": { "type": "datetime" },
"external_identity_id": {},
"reference_identifier": {}
"reference_identifier": {},
"adopted_person_id": { "type": "integer", "foreignkey": { "table": "people", "column": "id" } }
},
"indexes": {
"ext_identity_sources_records_i1": { "columns": [ "external_identity_source_id" ] },
"ext_identity_sources_records_i2": { "columns": [ "external_identity_id" ] },
"ext_identity_sources_records_i3": { "columns": [ "external_identity_source_id", "source_key" ] }
"ext_identity_sources_records_i3": { "columns": [ "external_identity_source_id", "source_key" ] },
"ext_identity_sources_records_i4": { "needed": false, "columns": [ "adopted_person_id" ] }
}
},

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ msgid "information.EmailVerifiers.sending"
msgstr "Sending"

msgid "information.EmailVerifiers.success"
msgstr "New Code Submitted!"
msgstr "New code sent"

msgid "information.EmailVerifiers.abort"
msgstr "Abort"
Expand Down
27 changes: 27 additions & 0 deletions app/plugins/CoreJob/resources/locales/en_US/core_job.po
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@
msgid "error.co_id"
msgstr "Requested {0} entity {1} is not in CO {2}"

msgid "opt.adopter.external_identity_source_id"
msgstr "External Identity Source ID"

msgid "opt.adopter.source_keys"
msgstr "Source Keys to process, comma separated (requires external_identity_source_id)"

msgid "opt.assigner.context"
msgstr "Identifier Assignment context"

Expand All @@ -49,6 +55,27 @@ msgstr "If true, force records to process even if no changes have been detected"
msgid "opt.sync.source_keys"
msgstr "Source Keys to process, comma separated (requires external_identity_source_id)"

msgid "Adopter.error.adopted"
msgstr "Record has already been adopted as Person {0}"

msgid "Adopter.error.status"
msgstr "EIS must be disabled before adoption process is run"

msgid "Adopter.error.synced"
msgstr "Record has not been synced (no corresponding External Identity) and so cannot be adopted"

msgid "Adopter.finish_summary.count"
msgstr "Adopter Finished ({0} records adopted, {1} errors)"

msgid "Adopter.result.adopted"
msgstr "Adopted External Identity {0} as Person {1}"

msgid "Adopter.start_summary.eis"
msgstr "Adopting all records from EIS {0}"

msgid "Adopter.start_summary.keys"
msgstr "Adopting {0} record(s) from EIS {1}"

msgid "Assigner.cancel_summary"
msgstr "Job canceled after reviewing {0} entities and assigning {1} Identifier(s)"

Expand Down
169 changes: 169 additions & 0 deletions app/plugins/CoreJob/src/Lib/Jobs/AdopterJob.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
<?php
/**
* COmanage Registry Adopter Job
*
* Portions licensed to the University Corporation for Advanced Internet
* Development, Inc. ("UCAID") under one or more contributor license agreements.
* See the NOTICE file distributed with this work for additional information
* regarding copyright ownership.
*
* UCAID licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @link https://www.internet2.edu/comanage COmanage Project
* @package registry
* @since COmanage Registry v5.1.0
* @license Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
*/

declare(strict_types = 1);

namespace CoreJob\Lib\Jobs;

use Cake\Datasource\ConnectionManager;
use Cake\ORM\TableRegistry;
use \App\Lib\Enum\JobStatusEnum;
use \App\Lib\Enum\SyncModeEnum;

class AdopterJob {
use \App\Lib\Traits\LabeledLogTrait;

/**
* Obtain the list of parameters supported by this Job.
*
* @since COmanage Registry v5.1.0
* @return Array Array of supported parameters.
* @throws \InvalidArgumentException
* @throws \RuntimeException
*/

public function parameterFormat(): array {
return [
'external_identity_source_id' => [
'help' => __d('core_job', 'opt.adopter.external_identity_source_id'),
'type' => 'fk',
'required' => true
],
'source_keys' => [
'help' => __d('core_job', 'opt.adopter.source_keys'),
'type' => 'string',
'required' => false
]
];
}

/**
* Run the requested Job.
*
* @since COmanage Registry v5.0.0
* @param JobsTable $JobsTable Jobs table, for updating the Job status
* @param JobHistoryRecordsTable $JobHistoryRecordsTable Job History Records table, for recording additional history
* @param Job $job Job entity
* @param array $parameters Parameters for this Job
*/

public function run(
\App\Model\Table\JobsTable $JobsTable,
\App\Model\Table\JobHistoryRecordsTable $JobHistoryRecordsTable,
\App\Model\Entity\Job $job,
array $parameters
) {
$count = 0; // Count of records successfully processed
$errors = 0; // Count of records that had errors
$todo = []; // The set of source keys to process

// Pull the EIS configuration
$EISTable = TableRegistry::getTableLocator()->get('ExternalIdentitySources');
$EITable = TableRegistry::getTableLocator()->get('ExternalIdentities');

$eis = $EISTable->get($parameters['external_identity_source_id']);

// The EIS must be disabled to prevent conflicts (and to encourage the admin to
// consider whether it should be enabled after the adoption process runs.)

if($eis->status != SyncModeEnum::Disabled) {
throw new \InvalidArgumentException(__d('core_job', 'Adopter.error.status'));
}

if(!empty($parameters['source_keys'])) {
$todo = explode(',', $parameters['source_keys']);

$JobsTable->start(
job: $job,
summary: __d('core_job', 'Adopter.start_summary.keys', [count($todo), $eis->id])
);
} else {
// Note inventory() loads all records into memory, so we might have issues with
// extremely large datasets.
$todo = $EISTable->inventory($eis->id);

$JobsTable->start(
job: $job,
summary: __d('core_job', 'Adopter.start_summary.eis', [$eis->id])
);
}

$this->llog('trace', "Adopting " . count($todo) . " record(s) from EIS "
. $eis->description . " (job " . $job->id . ")");

foreach($todo as $sourceKey) {
try {
// We need to map the $sourceKey to an External Identity, which we do via
// the EIS Record.

$eisrecord = $EISTable->ExtIdentitySourceRecords->find()
->where([
'external_identity_source_id' => $eis->id,
'source_key' => $sourceKey
])
->firstOrFail();

if(!empty($eisrecord->adopted_person_id)) {
throw new \InvalidArgumentException(__d('core_job', 'Adopter.error.adopted', [$eisrecord->adopted_person_id]));
}

if(empty($eisrecord->external_identity_id)) {
throw new \InvalidArgumentException(__d('core_job', 'Adopter.error.synced'));
}

$this->llog('trace', "Mapped source key $sourceKey to External Identity " . $eisrecord->external_identity_id);

$personId = $EITable->adopt($eisrecord->external_identity_id);

$this->llog('trace', "Adopted External Identity " . $eisrecord->external_identity_id . " as Person " . $personId);

$JobHistoryRecordsTable->record(
jobId: $job->id,
recordKey: $sourceKey,
comment: __d('core_job', 'Adopter.result.adopted', [$eisrecord->external_identity_id, $personId]),
status: JobStatusEnum::Complete
);

$count++;
}
catch(\Exception $e) {
$this->llog('trace', "$sourceKey could not be adopted: " . $e->getMessage());

$JobHistoryRecordsTable->record(
jobId: $job->id,
recordKey: $sourceKey,
comment: $e->getMessage(),
status: JobStatusEnum::Failed
);

$errors++;
}
}

$JobsTable->finish(job: $job, summary: __d('core_job', 'Adopter.finish_summary.count', [$count, $errors]));
}
}
Loading

0 comments on commit f2ca6f5

Please sign in to comment.