diff --git a/app/availableplugins/ApiConnector/src/config/plugin.json b/app/availableplugins/ApiConnector/config/plugin.json similarity index 97% rename from app/availableplugins/ApiConnector/src/config/plugin.json rename to app/availableplugins/ApiConnector/config/plugin.json index 69a8f6010..9f5b08eb5 100644 --- a/app/availableplugins/ApiConnector/src/config/plugin.json +++ b/app/availableplugins/ApiConnector/config/plugin.json @@ -3,7 +3,7 @@ "api": [ "ApiSourceEndpoints" ], - "source": [ + "external_identity_source": [ "ApiSources" ] }, diff --git a/app/availableplugins/FileConnector/src/config/plugin.json b/app/availableplugins/FileConnector/config/plugin.json similarity index 88% rename from app/availableplugins/FileConnector/src/config/plugin.json rename to app/availableplugins/FileConnector/config/plugin.json index 977e900b2..13973daa5 100644 --- a/app/availableplugins/FileConnector/src/config/plugin.json +++ b/app/availableplugins/FileConnector/config/plugin.json @@ -1,9 +1,9 @@ { "types": { - "provisioner": [ + "provisioning_target": [ "FileProvisioners" ], - "source": [ + "external_identity_source": [ "FileSources" ] }, @@ -26,7 +26,7 @@ "filename": { "type": "string", "size": 256 }, "format": { "type": "string", "size": 2 }, "archivedir": { "type": "string", "size": 256 }, - "threshold_warn": { "type": "integer" }, + "threshold_check": { "type": "integer" }, "threshold_override": { "type": "boolean" } }, "indexes": { diff --git a/app/availableplugins/FileConnector/resources/locales/en_US/file_connector.po b/app/availableplugins/FileConnector/resources/locales/en_US/file_connector.po index 5569bfd6f..6ca5cf325 100644 --- a/app/availableplugins/FileConnector/resources/locales/en_US/file_connector.po +++ b/app/availableplugins/FileConnector/resources/locales/en_US/file_connector.po @@ -28,15 +28,30 @@ msgstr "{0,plural,=1{File Provisioner} other{File Provisioners}}" msgid "enumeration.FileSourceFormatEnum.C3" msgstr "CSV v3" +msgid "error.filename.absolute" +msgstr "File path \"{0}\" does not begin with \"/\"" + msgid "error.filename.readable" msgstr "The file \"{0}\" is not readable" -msgid "error.filename.writeable" +msgid "error.filename.writable" msgstr "The file \"{0}\" is not writable" +msgid "error.FileSource.copy" +msgstr "Failed to copy {0} to {1}" + +msgid "error.FileSource.threshold" +msgstr "{0} of {1} records changed ({2}%, including new records), exceeding threshold of {3}% - processing canceled" + +msgid "error.FileSource.threshold.config" +msgstr "Threshold Check requires Archive Directory" + msgid "error.header" msgstr "Did not find CSV file header" +msgid "error.header.invalid" +msgstr "\"{0}\" is not a valid field" + msgid "error.header.sorid" msgstr "Did not find SORID as first defined column, check file header definition" @@ -61,14 +76,17 @@ msgstr "Full path to file to read from, which must exist and be readable" msgid "field.FileSources.format" msgstr "File Format" -msgid "field.FileSources.threshold_warn" +msgid "field.FileSources.threshold_check" msgstr "Warning Threshold" -msgid "field.FileSources.threshold_warn.desc" +msgid "field.FileSources.threshold_check.desc" msgstr "If the number of changed records exceeds the specified percentage, a warning will be generated and processing will stop (requires Archive Directory)" msgid "field.FileSources.threshold_override" msgstr "Warning Threshold Override" msgid "field.FileSources.threshold_override.desc" -msgstr "If set, the next Full sync will ignore the Warning Threshold" \ No newline at end of file +msgstr "If set, the next Full sync will ignore the Warning Threshold" + +msgid "result.FileProvisioner.done" +msgstr "Wrote 1 record to file" \ No newline at end of file diff --git a/app/availableplugins/FileConnector/src/Model/Table/FileProvisionersTable.php b/app/availableplugins/FileConnector/src/Model/Table/FileProvisionersTable.php index 3a0aff846..baf3e93c9 100644 --- a/app/availableplugins/FileConnector/src/Model/Table/FileProvisionersTable.php +++ b/app/availableplugins/FileConnector/src/Model/Table/FileProvisionersTable.php @@ -34,7 +34,7 @@ use Cake\Validation\Validator; use App\Lib\Enum\ProvisioningEligibilityEnum; use App\Lib\Enum\ProvisioningStatusEnum; -use \FileConnector\Model\Entity\FileProvisioner; +use App\Model\Entity\ProvisioningTarget; class FileProvisionersTable extends Table { use \App\Lib\Traits\ChangelogBehaviorTrait; @@ -98,10 +98,10 @@ public function initialize(array $config): void { */ public function buildRules(RulesChecker $rules): RulesChecker { - // The requested file must exist and be writeable. + // The requested file must exist and be writable. - $rules->add([$this, 'ruleIsFileWriteable'], - 'isFileWriteable', + $rules->add([$this, 'ruleIsFileWritable'], + 'isFileWritable', ['errorField' => 'filename']); return $rules; @@ -110,17 +110,17 @@ public function buildRules(RulesChecker $rules): RulesChecker { /** * Provision object data to the provisioning target. * - * @param FileProvisioner $provisioningTarget FileProvisioner configuration - * @param string $entityName - * @param object $data Provisioning data in Entity format (eg: \App\Model\Entity\Person) - * @param string $eligibility Provisioning Eligibility Enum + * @param ProvisioningTarget $provisioningTarget FileProvisioner configuration + * @param string $entityName + * @param object $data Provisioning data in Entity format (eg: \App\Model\Entity\Person) + * @param string $eligibility Provisioning Eligibility Enum * * @return array Array of status, comment, and optional identifier * @since COmanage Registry v5.0.0 */ public function provision( - FileProvisioner $provisioningTarget, + ProvisioningTarget $provisioningTarget, string $entityName, object $data, string $eligibility @@ -133,22 +133,22 @@ public function provision( } if(file_put_contents( - filename: $provisioningTarget->filename, + filename: $provisioningTarget->file_provisioner->filename, data: json_encode($output, JSON_INVALID_UTF8_SUBSTITUTE) . "\n", flags: FILE_APPEND ) === false) { - throw new \RuntimeException("Write to " . $provisioningTarget->filename . " failed"); + throw new \RuntimeException("Write to " . $provisioningTarget->file_provisioner->filename . " failed"); } return [ 'status' => ProvisioningStatusEnum::Provisioned, - 'comment' => "Wrote 1 record to file", + 'comment' => __d('file_connector', 'result.FileProvisioner.done'), 'identifier' => null ]; } /** - * Application Rule to determine if the current entity is a writeable file. + * Application Rule to determine if the current entity is a writable file. * * @param Entity $entity Entity to be validated * @param array $options Application rule options @@ -157,9 +157,9 @@ public function provision( * @since COmanage Registry v5.0.0 */ - public function ruleIsFileWriteable($entity, array $options): string|bool { + public function ruleIsFileWritable($entity, array $options): string|bool { if(!is_writable($entity->filename)) { - return __d('file_connector', 'error.filename.writeable', [$entity->filename]); + return __d('file_connector', 'error.filename.writable', [$entity->filename]); } return true; diff --git a/app/availableplugins/FileConnector/src/Model/Table/FileSourcesTable.php b/app/availableplugins/FileConnector/src/Model/Table/FileSourcesTable.php index 8029d6961..9918554fb 100644 --- a/app/availableplugins/FileConnector/src/Model/Table/FileSourcesTable.php +++ b/app/availableplugins/FileConnector/src/Model/Table/FileSourcesTable.php @@ -32,6 +32,7 @@ use Cake\ORM\RulesChecker; use Cake\ORM\Table; use Cake\ORM\TableRegistry; +use Cake\Utility\Inflector; use Cake\Validation\Validator; use \App\Model\Entity\ExternalIdentity; use \FileConnector\Lib\Enum\FileSourceFormatEnum; @@ -40,6 +41,7 @@ class FileSourcesTable extends Table { use \App\Lib\Traits\AutoViewVarsTrait; use \App\Lib\Traits\ChangelogBehaviorTrait; use \App\Lib\Traits\CoLinkTrait; + use \App\Lib\Traits\LabeledLogTrait; use \App\Lib\Traits\PermissionsTrait; use \App\Lib\Traits\PrimaryLinkTrait; use \App\Lib\Traits\TableMetaTrait; @@ -49,6 +51,13 @@ class FileSourcesTable extends Table { // Cache of the field configuration protected $fieldCfg = null; + // Cache of archive file paths + protected $archive1 = null; + protected $archive2 = null; + + // Whether postRunTasks should rotate the archive + protected $rotate = false; + /** * Perform Cake Model initialization. * @@ -126,11 +135,34 @@ public function buildRules(RulesChecker $rules): RulesChecker { 'isFileReadable', ['errorField' => 'filename']); -// XXX CFM-117 should we also check that the archive dir, if specified, is writeable? + $rules->add([$this, 'ruleIsArchiveWriteable'], + 'isArchiveWriteable', + ['errorField' => 'archivedir']); return $rules; } + /** + * Obtain the set of changed records from the source file. + * + * @since COmanage Registry v5.2.0 + * @param ExternalIdentitySource $source External Identity Source + * @param int $lastStart Timestamp of last run + * @param int $curStart Timestamp of current run + * @return array|bool An array of changed source keys, or false + * @throws RuntimeException + */ + + public function getChangeList( + \App\Model\Entity\ExternalIdentitySource $source, + int $lastStart, // timestamp of last run + int $curStart // timestamp of current run + ): array|bool { + $changeList = $this->processChangeList($source, $lastStart, $curStart); + + return ($changeList == false) ? false : $changeList['changeList']; + } + /** * Obtain the full set of records from the source database. * @@ -154,9 +186,11 @@ public function inventory( fgetcsv($handle); while(($data = fgetcsv($handle)) !== false) { - // The source key is always the first field in each line + // The source key is always the first field in each line, make sure it is not empty - $ret[] = $data[0]; + if(!empty($data[0]) && !ctype_space($data[0])) { + $ret[] = $data[0]; + } } fclose($handle); @@ -167,6 +201,236 @@ public function inventory( return $ret; } + /** + * Perform checks before a Sync Job proceeds. + * + * @since COmanage Registry v5.2.0 + * @param ExternalIdentitySource $source External Identity Source + * @param int $lastStart Timestamp of last run + * @param int $curStart Timestamp of current run + * @throws RuntimeException + */ + + public function preRunChecks( + \App\Model\Entity\ExternalIdentitySource $source, + int $lastStart, + int $curStart + ) { + // If a threshold is set, check to make sure less than that many records changed + // (by percent). + + // By default, we'll rotate the archive files in postRunTasks. (If we throw an + // exception, that hook won't be called.) + + $this->rotate = true; + + if(!empty($source->file_source->threshold_check) + && $source->file_source->threshold_check > 0) { + // threshold_check requires archive directories since we can't otherwise + // efficiently calculate diffs. + + if(empty($source->file_source->archivedir)) { + $this->llog('debug', 'Threshold Check for ' . $source->description . ' is configured but no Archive Directory is available, ignoring'); + throw new \RuntimeException(__d('file_connector', 'error.FileSource.threshold.config')); + } + + // Check the number of changed records vs warning threshold. Note this + // check (correctly) does not run the first time a file is processed + // since there will be no archive file to compare against. + + if($source->file_source->threshold_override) { + // Ignore thresholds, but unset this configuration for our next run + + $source->file_source->threshold_override = false; + $this->saveOrFail($source->file_source, ['associated' => false]); + + $this->llog('trace', 'Threshold Check for ' . $source->description . ' is overridden, ignoring this time only'); + } else { + $info = $this->processChangeList($source, $lastStart, $curStart); + + if($info['knownCount'] > 0) { + $changed = count($info['changeList']) + $info['newCount']; + $pct = floor(($changed * 100) / $info['knownCount']); + + if($pct > $source->file_source->threshold_check) { + $this->llog('trace', 'Threshold Check for ' . $source->description . ' exceeded, stopping processing (changed=' . $changed . ', known=' . $info['knownCount'] . ', percent=' . $pct . ')'); + + throw new \RuntimeException(__d('file_connector', 'error.FileSource.threshold', [ + $changed, $info['knownCount'], $pct, $source->file_source->threshold_check + ])); + } + } + // else no previous records, so treat as all new + + if(empty($info['changeList'] + && $info['newCount'] == 0) + && $info['knownCount'] > 0) { + // We don't want to rotate the Archive files if there were no changed records + // (since nothing happened). + + $this->rotate = false; + } + } + } + } + + /** + * Obtain the set of changed records from the source file. + * + * @since COmanage Registry v5.2.0 + * @param ExternalIdentitySource $source External Identity Source + * @param int $lastStart Timestamp of last run + * @param int $curStart Timestamp of current run + * @return array|bool An array of + * changelist: Changed record Source Keys + * newCount: Count of new records + * knownCount: Count of known records + * or false + * @throws RuntimeException + */ + + protected function processChangeList( + \App\Model\Entity\ExternalIdentitySource $source, + int $lastStart, // timestamp of last run + int $curStart // timestamp of current run + ): array|bool { + if(empty($source->file_source->archivedir)) { + // If there is no archivedir we don't support changelist calculation + return false; + } + + $ret = []; + $knownCount = 0; + $newCount = 0; + + $infile = $source->file_source->filename; + $basename = basename($infile); + $this->archive1 = $source->file_source->archivedir . DS . $basename . ".1"; + $this->archive2 = $source->file_source->archivedir . DS . $basename . ".2"; + + // We could either read the files simultaneously in order (lower memory requirement), + // or read one and hash it (can read records out of sequence). For now we'll take + // the second approach. + + if(is_readable($this->archive1)) { + // Start by creating a set of previously known records. + $knownRecords = []; + + $handle = fopen($this->archive1, "r"); + + if(!$handle) { + throw new \RuntimeException(__d('file_connector', 'error.filename.readable', [$this->archive1])); + } + + // Ignore the header line + fgetcsv($handle); + + while(($data = fgetcsv($handle)) !== false) { + // Implode the record back together for string comparison purposes. + // This may not be the same as the original line due to quotes, etc. + // $data[0] is the SORID + $knownRecords[ $data[0] ] = implode(',', $data); + } + + $knownCount = count($knownRecords); + + fclose($handle); + + // Now read the new file and look for changes. + $handle = fopen($infile, "r"); + + if(!$handle) { + throw new \RuntimeException(__d('file_connector', 'error.filename.readable', [$infile])); + } + + // Ignore the header line + fgetcsv($handle); + + while(($data = fgetcsv($handle)) !== false) { + // $data[0] is the SORID + if(array_key_exists($data[0], $knownRecords)) { + $newData = implode(',', $data); + + if($newData != $knownRecords[ $data[0] ]) { + // This record changed, push the SORID onto the change list + $ret[] = $data[0]; + } + + // Unset the key so we can see which records were deleted. + unset($knownRecords[ $data[0] ]); + } else { + // This is a new record (ie: in $infile, not in $archive1), + // so we ignore it, except to count it. + $newCount++; + } + } + + fclose($handle); + + // Finally, any remaining keys in $knownRecords are delete operations. + if(!empty($knownRecords)) { + $ret = array_merge($ret, array_keys($knownRecords)); + } + } else { + // If there is no archive file, we've either never run at all, or the admin + // updated the configuration and we have no idea what changed. In either case + // we'll report all records as new, which will cause them all to be + // (re)processed. In the latter case, admins can avoid this by manually creating + // the .1 file before updating the configuration. + + $ret = $this->inventory($source); + $newCount = count($ret); + } + + return [ + 'changeList' => $ret, + 'newCount' => $newCount, + 'knownCount' => $knownCount + ]; + } + + /** + * Perform tasks following a Sync Job. + * + * @since COmanage Registry v5.2.0 + * @param ExternalIdentitySource $source External Identity Source + */ + + public function postRunTasks( + \App\Model\Entity\ExternalIdentitySource $source + ) { + // Update the archive file. updateCache() is called after processing is complete, + // and only if at least one record changed. It's possible an irregular exit will + // prevent the cache from being updated, in that case we'll just end up reprocessing + // some records, which should effectively be a no-op. Historically, we kept two backup + // copies in case something went wrong, we still do so here, though it's less critical now. + + if(!$this->rotate) { + $this->llog('trace', 'Not rotating archive files due to no changes'); + return; + } + + if(is_readable($this->archive1)) { + $this->llog('trace', 'Copying ' . $this->archive1 . ' to ' . $this->archive2); + + if(!copy($this->archive1, $this->archive2)) { + throw new \RuntimeException(__d('file_connector', 'error.FileSource.copy', [ + $this->archive1, $this->archive2 + ])); + } + } + + if(is_readable($source->file_source->filename)) { + $this->llog('trace', 'Copying ' . $source->file_source->filename . ' to ' . $this->archive1); + + if(!copy($source->file_source->filename, $this->archive1)) { + throw new \RuntimeException(__d('file_connector', 'error.FileSource.copy', [ + $source->file_source->filename, $this->archive1 + ])); + } + } + } + /** * Obtain the file field configuration. * @@ -202,9 +466,12 @@ protected function readFieldConfig( throw new \RuntimeException(__d('error.header')); } + // Calculate the CO for $filesource, which we'll need for field type validation + $coId = $this->calculateCoForRecord($filesource); + foreach($cfg as $i => $label) { // Labels are of the forms described in the switch statement. - // Parse them out into the fieldcfg array. + // Parse them out into the fieldcfg array. We also validate them as we parse them. $bits = explode('.', $label, 5); @@ -221,20 +488,37 @@ protected function readFieldConfig( // external_identity.field // ad_hoc_attributes.tag (attached to EI) // related_model.field (not currently used) + if(!in_array($bits[0], ['ad_hoc_attributes', 'external_identity'])) { + throw new \RuntimeException(__d('file_connector', 'error.header.invalid', [$label])); + } + + if($bits[0] != 'ad_hoc_attributes' && !$this->validField($bits[0], $bits[1], $coId)) { + throw new \RuntimeException(__d('file_connector', 'error.header.invalid', [$label])); + } + $this->fieldCfg[ $bits[0] ][ $bits[1] ] = $i; break; case 3: - // related_models.type.field + // related_models.field.type // external_identity_roles.#.field (special case) - // Note we _no longer_ flip the order model/type/field - // (this is inverted from CSV v2) + // Note the old v2 order model/type/field is inverted here // and identifier+login is no longer supported if($bits[0] == 'external_identity_roles') { // Store based on role + + if(!$this->validField($bits[0], $bits[2], $coId)) { + throw new \RuntimeException(__d('file_connector', 'error.header.invalid', [$label])); + } + $this->fieldCfg[ $bits[0] ]['roles'][ $bits[1] ]['fields'][ $bits[2] ] = $i; } else { // Store based on type - $this->fieldCfg[ $bits[0] ]['types'][ $bits[1] ][ $bits[2] ] = $i; + + if(!$this->validField($bits[0], $bits[1], $coId, $bits[2])) { + throw new \RuntimeException(__d('file_connector', 'error.header.invalid', [$label])); + } + + $this->fieldCfg[ $bits[0] ]['types'][ $bits[2] ][ $bits[1] ] = $i; } break; case 4: @@ -242,9 +526,9 @@ protected function readFieldConfig( $this->fieldCfg[ $bits[0] ]['roles'][ $bits[1] ]['related'][ $bits[2] ][ $bits[3] ] = $i; break; case 5: - // external_identity_roles.#.related_models.type.field + // external_identity_roles.#.related_models.field.type // Note these are keyed on an SOR Role ID - $this->fieldCfg[ $bits[0] ]['roles'][ $bits[1] ]['related'][ $bits[2] ]['types'][ $bits[3] ][ $bits[4] ] = $i; + $this->fieldCfg[ $bits[0] ]['roles'][ $bits[1] ]['related'][ $bits[2] ]['types'][ $bits[4] ][ $bits[3] ] = $i; break; } } @@ -308,7 +592,7 @@ protected function resultToEntityData(array $result): array { } } } - +/* External Identities no longer have Primary Names // Make sure we have a Primary Name $primaryNameSet = false; @@ -321,7 +605,7 @@ protected function resultToEntityData(array $result): array { if(!$primaryNameSet) { $eidata['names'][0]['primary_name'] = true; - } + }*/ // Process Ad Hoc Attributes (case 2) if(!empty($this->fieldCfg['ad_hoc_attributes'])) { @@ -457,16 +741,52 @@ public function retrieve( } /** - * Application Rule to determine if the current entity is a readable file. + * Application Rule to determine if the current entity has a writeable archive directory. * - * @param Entity $entity Entity to be validated - * @param array $options Application rule options + * @since COmanage Registry v5.2.0 + * @param Entity $entity Entity to be validated + * @param array $options Application rule options + * @return string|bool true if the Rule check passes, false otherwise + */ + + public function ruleIsArchiveWriteable($entity, array $options): string|bool { + // Archive Directory is optional, so we only complain if it's set but not writeable + + if(!empty($entity->archivedir)) { + // We also check if the archive directory is absolute or relative. This is partly to give a + // more helpful message, and partly because if a deployer configures the directory into + // $webroot it'll be readable by the web server but not the command line. + + if(mb_substr($entity->archivedir, 0, 1) != '/') { + return __d('file_connector', 'error.filename.absolute', [$entity->archivedir]); + } + + if(!is_writable($entity->archivedir)) { + return __d('file_connector', 'error.filename.writable', [$entity->archivedir]); + } + } + + return true; + } + + /** + * Application Rule to determine if the current entity has a readable file. * - * @return string|bool true if the Rule check passes, false otherwise * @since COmanage Registry v5.0.0 + * @param Entity $entity Entity to be validated + * @param array $options Application rule options + * @return string|bool true if the Rule check passes, false otherwise */ public function ruleIsFileReadable($entity, array $options): string|bool { + // We also check if the file path is absolute or relative. This is partly to give a + // more helpful message, and partly because if a deployer drops the file into $webroot + // it'll be readable by the web server but not the command line. + + if(mb_substr($entity->filename, 0, 1) != '/') { + return __d('file_connector', 'error.filename.absolute', [$entity->filename]); + } + if(!is_readable($entity->filename)) { return __d('file_connector', 'error.filename.readable', [$entity->filename]); } @@ -542,6 +862,61 @@ public function searchableAttributes(): array { 'q' => __d('field', 'search.placeholder') ]; } + + /** + * Determine if $field is a valid field for $model. + * + * @since COmanage Registry v5.2.0 + * @param string $model Model, in under_score format + * @param string $field Field name + * @param int $coId Current CO ID (only used for Type validation) + * @param string $type Field type, for MVEAs + * @return bool true if $field is a field of $model, false otherwise + */ + + protected function validField( + string $model, + string $field, + int $coId, + ?string $type=null + ): bool { + // As a first pass, we just check the schema for the field, which means metadata + // such as revision or created will be accepted as valid. A better approach would + // be to do somethnig like TabelMetaTrait::filterMetadataFields, since we probably + // don't want to accept metadata fields as valid. + + $tableName = Inflector::pluralize(Inflector::classify($model)); + + $Table = TableRegistry::getTableLocator()->get($tableName); + + $schema = $Table->getSchema(); + + if(!$schema->hasColumn($field)) { + return false; + } + + if($type) { + // We need to see if $type is a valid Type value. This will mostly be + // $tableName.type (eg: Names.type), except for ExternalIdentityRoles + // which will be PersonRoles.affiliation. + + $attr = ($tableName == 'ExternalIdentityRoles') + ? "PersonRoles.affiliation_type" + : ($tableName . ".type"); + + try { + $Types = TableRegistry::getTableLocator()->get("Types"); + + // This will throw an exception if not valid + $Types->getTypeId($coId, $attr, $type); + } + catch(\Exception $e) { + return false; + } + } + + return true; + } /** * Set validation rules. @@ -570,13 +945,13 @@ public function validationDefault(Validator $validator): Validator { $this->registerStringValidation($validator, $schema, 'archivedir', false); - $validator->add('threshold_warn', [ + $validator->add('threshold_check', [ 'content' => ['rule' => 'isInteger'] ]); - $validator->add('threshold_warn', [ + $validator->add('threshold_check', [ 'range' => ['rule' => 'range', 0, 100] ]); - $validator->allowEmptyString('threshold_warn'); + $validator->allowEmptyString('threshold_check'); $validator->add('threshold_override', [ 'content' => ['rule' => ['boolean']] diff --git a/app/availableplugins/FileConnector/templates/FileSources/fields.inc b/app/availableplugins/FileConnector/templates/FileSources/fields.inc index e9e7ae621..2307d35f9 100644 --- a/app/availableplugins/FileConnector/templates/FileSources/fields.inc +++ b/app/availableplugins/FileConnector/templates/FileSources/fields.inc @@ -31,7 +31,7 @@ if($vv_action == 'add' || $vv_action == 'edit') { 'filename', 'format', 'archivedir', - 'threshold_warn', + 'threshold_check', 'threshold_override', ] as $field) { print $this->element('form/listItem', [ diff --git a/app/availableplugins/PipelineToolkit/src/config/plugin.json b/app/availableplugins/PipelineToolkit/config/plugin.json similarity index 100% rename from app/availableplugins/PipelineToolkit/src/config/plugin.json rename to app/availableplugins/PipelineToolkit/config/plugin.json diff --git a/app/availableplugins/SqlConnector/src/config/plugin.json b/app/availableplugins/SqlConnector/config/plugin.json similarity index 99% rename from app/availableplugins/SqlConnector/src/config/plugin.json rename to app/availableplugins/SqlConnector/config/plugin.json index 5d32cb333..2a0953ca3 100644 --- a/app/availableplugins/SqlConnector/src/config/plugin.json +++ b/app/availableplugins/SqlConnector/config/plugin.json @@ -1,9 +1,9 @@ { "types": { - "provisioner": [ + "provisioning_target": [ "SqlProvisioners" ], - "source": [ + "external_identity_source": [ "SqlSources" ] }, diff --git a/app/availableplugins/SqlConnector/src/Model/Table/SqlProvisionersTable.php b/app/availableplugins/SqlConnector/src/Model/Table/SqlProvisionersTable.php index ba0835387..9a4bf96f3 100644 --- a/app/availableplugins/SqlConnector/src/Model/Table/SqlProvisionersTable.php +++ b/app/availableplugins/SqlConnector/src/Model/Table/SqlProvisionersTable.php @@ -256,9 +256,8 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'servers' => [ - 'type' => 'select', - 'model' => 'Servers', - 'where' => ['plugin' => 'CoreServer.SqlServers'] + 'type' => 'plugin', + 'model' => 'CoreServer.SqlServers' ] ]); @@ -355,7 +354,7 @@ public function localAfterSave(\Cake\Event\EventInterface $event, \Cake\Datasour * Provision object data to the provisioning target. * * @since COmanage Registry v5.0.0 - * @param SqlProvisioner $provisioningTarget SqlProvisioner configuration + * @param ProvisioningTarget $provisioningTarget SqlProvisioner configuration * @param string $className Class name of primary object being provisioned * @param object $data Provisioning data in Entity format (eg: \App\Model\Entity\Person) * @param ProvisioningEligibilityEnum $eligibility Provisioning Eligibility Enum @@ -363,16 +362,16 @@ public function localAfterSave(\Cake\Event\EventInterface $event, \Cake\Datasour */ public function provision( - \SqlConnector\Model\Entity\SqlProvisioner $provisioningTarget, + \App\Model\Entity\ProvisioningTarget $provisioningTarget, string $className, object $data, // $data is currently only \App\Model\Entity\Person, but that might change string $eligibility ): array { // Connect to the target database - $this->Servers->SqlServers->connect($provisioningTarget->server_id, 'targetdb'); + $this->Servers->SqlServers->connect($provisioningTarget->sql_provisioner->server_id, 'targetdb'); return $this->syncEntity( - $provisioningTarget, + $provisioningTarget->sql_provisioner, $className, $data, $eligibility diff --git a/app/availableplugins/SqlConnector/src/Model/Table/SqlSourcesTable.php b/app/availableplugins/SqlConnector/src/Model/Table/SqlSourcesTable.php index a9170e643..a812ea513 100644 --- a/app/availableplugins/SqlConnector/src/Model/Table/SqlSourcesTable.php +++ b/app/availableplugins/SqlConnector/src/Model/Table/SqlSourcesTable.php @@ -128,9 +128,8 @@ public function initialize(array $config): void { 'attribute' => 'Pronouns.type' ], 'servers' => [ - 'type' => 'select', - 'model' => 'Servers', - 'where' => ['plugin' => 'CoreServer.SqlServers'] + 'type' => 'plugin', + 'model' => 'CoreServer.SqlServers' ], 'telephoneNumberTypes' => [ 'type' => 'type', diff --git a/app/plugins/CoreApi/src/config/plugin.json b/app/plugins/CoreApi/config/plugin.json similarity index 100% rename from app/plugins/CoreApi/src/config/plugin.json rename to app/plugins/CoreApi/config/plugin.json diff --git a/app/plugins/CoreApi/src/Model/Table/MatchCallbacksTable.php b/app/plugins/CoreApi/src/Model/Table/MatchCallbacksTable.php index f69c0f495..1ed67d392 100644 --- a/app/plugins/CoreApi/src/Model/Table/MatchCallbacksTable.php +++ b/app/plugins/CoreApi/src/Model/Table/MatchCallbacksTable.php @@ -73,9 +73,8 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'servers' => [ - 'type' => 'select', - 'model' => 'Servers', - 'where' => ['plugin' => 'CoreServer.MatchServers'] + 'type' => 'plugin', + 'model' => 'CoreServer.MatchServers' ] ]); diff --git a/app/plugins/CoreAssigner/src/config/plugin.json b/app/plugins/CoreAssigner/config/plugin.json similarity index 98% rename from app/plugins/CoreAssigner/src/config/plugin.json rename to app/plugins/CoreAssigner/config/plugin.json index b94193d5c..4182ec6f8 100644 --- a/app/plugins/CoreAssigner/src/config/plugin.json +++ b/app/plugins/CoreAssigner/config/plugin.json @@ -1,6 +1,6 @@ { "types": { - "assigner": [ + "identifier_assignment": [ "FormatAssigners", "SqlAssigners" ] diff --git a/app/plugins/CoreAssigner/src/Model/Table/SqlAssignersTable.php b/app/plugins/CoreAssigner/src/Model/Table/SqlAssignersTable.php index d762372bc..bc67efc66 100644 --- a/app/plugins/CoreAssigner/src/Model/Table/SqlAssignersTable.php +++ b/app/plugins/CoreAssigner/src/Model/Table/SqlAssignersTable.php @@ -76,9 +76,8 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'servers' => [ - 'type' => 'select', - 'model' => 'Servers', - 'where' => ['plugin' => 'CoreServer.SqlServers'] + 'type' => 'plugin', + 'model' => 'CoreServer.SqlServers' ], 'types' => [ 'type' => 'type', diff --git a/app/plugins/CoreEnroller/src/config/plugin.json b/app/plugins/CoreEnroller/config/plugin.json similarity index 99% rename from app/plugins/CoreEnroller/src/config/plugin.json rename to app/plugins/CoreEnroller/config/plugin.json index 8870e9358..0b06ae498 100644 --- a/app/plugins/CoreEnroller/src/config/plugin.json +++ b/app/plugins/CoreEnroller/config/plugin.json @@ -1,6 +1,6 @@ { "types": { - "enroller": [ + "enrollment_flow_step": [ "ApprovalCollectors", "AttributeCollectors", "BasicAttributeCollectors", diff --git a/app/plugins/CoreJob/src/config/plugin.json b/app/plugins/CoreJob/config/plugin.json similarity index 100% rename from app/plugins/CoreJob/src/config/plugin.json rename to app/plugins/CoreJob/config/plugin.json diff --git a/app/plugins/CoreJob/src/Lib/Jobs/SyncJob.php b/app/plugins/CoreJob/src/Lib/Jobs/SyncJob.php index 24770f5d1..34e2df7fb 100644 --- a/app/plugins/CoreJob/src/Lib/Jobs/SyncJob.php +++ b/app/plugins/CoreJob/src/Lib/Jobs/SyncJob.php @@ -284,7 +284,7 @@ protected function getRunContext( $this->runContext->eis = $this->runContext->EISTable->get( $eisId, - ['contain' => 'SqlSources'] + ['contain' => $this->runContext->EISTable->getPluginRelations()] ); } diff --git a/app/plugins/CoreServer/src/config/plugin.json b/app/plugins/CoreServer/config/plugin.json similarity index 100% rename from app/plugins/CoreServer/src/config/plugin.json rename to app/plugins/CoreServer/config/plugin.json diff --git a/app/plugins/EnvSource/src/config/plugin.json b/app/plugins/EnvSource/config/plugin.json similarity index 98% rename from app/plugins/EnvSource/src/config/plugin.json rename to app/plugins/EnvSource/config/plugin.json index 0b8fd35bf..d48fe3b2e 100644 --- a/app/plugins/EnvSource/src/config/plugin.json +++ b/app/plugins/EnvSource/config/plugin.json @@ -1,12 +1,12 @@ { "types": { - "enroller": [ + "enrollment_flow_step": [ "EnvSourceCollectors" ], - "source": [ + "external_identity_source": [ "EnvSources" ], - "traffic": [ + "traffic_detour": [ "EnvSourceDetours" ] }, diff --git a/app/plugins/EnvSource/src/Model/Table/EnvSourceCollectorsTable.php b/app/plugins/EnvSource/src/Model/Table/EnvSourceCollectorsTable.php index 5bce8a208..c4c23051c 100644 --- a/app/plugins/EnvSource/src/Model/Table/EnvSourceCollectorsTable.php +++ b/app/plugins/EnvSource/src/Model/Table/EnvSourceCollectorsTable.php @@ -99,9 +99,8 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'externalIdentitySources' => [ - 'type' => 'select', - 'model' => 'ExternalIdentitySources', - 'where' => ['plugin' => 'EnvSource.EnvSources'] + 'type' => 'plugin', + 'model' => 'EnvSource.EnvSources' ] ]); diff --git a/app/plugins/OrcidSource/src/config/plugin.json b/app/plugins/OrcidSource/config/plugin.json similarity index 97% rename from app/plugins/OrcidSource/src/config/plugin.json rename to app/plugins/OrcidSource/config/plugin.json index d83f9ea59..ac81588e1 100644 --- a/app/plugins/OrcidSource/src/config/plugin.json +++ b/app/plugins/OrcidSource/config/plugin.json @@ -1,9 +1,9 @@ { "types": { - "enroller": [ + "enrollment_flow_step": [ "OrcidSourceCollectors" ], - "source": [ + "external_identity_source": [ "OrcidSources" ] }, diff --git a/app/plugins/OrcidSource/src/Model/Table/OrcidSourcesTable.php b/app/plugins/OrcidSource/src/Model/Table/OrcidSourcesTable.php index a72fa7a08..7275be0b3 100644 --- a/app/plugins/OrcidSource/src/Model/Table/OrcidSourcesTable.php +++ b/app/plugins/OrcidSource/src/Model/Table/OrcidSourcesTable.php @@ -140,9 +140,8 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'servers' => [ - 'type' => 'select', - 'model' => 'Servers', - 'where' => ['plugin' => 'CoreServer.Oauth2Servers'] + 'type' => 'plugin', + 'model' => 'CoreServer.Oauth2Servers' ], 'api_tiers' => [ 'type' => 'enum', diff --git a/app/resources/locales/en_US/error.po b/app/resources/locales/en_US/error.po index 6753ebf83..8ab5fe927 100644 --- a/app/resources/locales/en_US/error.po +++ b/app/resources/locales/en_US/error.po @@ -229,6 +229,9 @@ msgstr "Could not copy." msgid "javascript.requires.https" msgstr "This feature requires HTTPS." +msgid "Jobs.command.plugin" +msgstr "Job Plugin not provided" + msgid "Jobs.failed.abnormal" msgstr "The Job terminated unexpectedly" diff --git a/app/src/Command/JobCommand.php b/app/src/Command/JobCommand.php index 8eed31355..f4ef29831 100644 --- a/app/src/Command/JobCommand.php +++ b/app/src/Command/JobCommand.php @@ -248,6 +248,15 @@ public function execute(Arguments $args, ConsoleIo $io) $pidcount--; } } else { + // We have a specific job to process. Note that JobCommand can't require -j + // since the -r usage doesn't need it, so we have to check for it manually. + + $jobPlugin = $args->getOption('job'); + + if(empty($jobPlugin)) { + throw new \InvalidArgumentException(__d('error', 'Jobs.command.plugin')); + } + // Run the requested job synchronously? $synchronous = $args->getOption('synchronous'); @@ -268,7 +277,7 @@ public function execute(Arguments $args, ConsoleIo $io) $job = $JobTable->register( coId: (int)$args->getOption('co_id'), - plugin: $args->getOption('job'), + plugin: $jobPlugin, parameters: $params, registerSummary: __d('result', 'Jobs.registered', [$pwent['name'], $pwent['uid']]), synchronous: $synchronous diff --git a/app/src/Controller/Component/BreadcrumbComponent.php b/app/src/Controller/Component/BreadcrumbComponent.php index 9bbe6bb91..349d16c7e 100644 --- a/app/src/Controller/Component/BreadcrumbComponent.php +++ b/app/src/Controller/Component/BreadcrumbComponent.php @@ -180,13 +180,14 @@ public function injectPrimaryLink(object $link, bool $index=true, string $linkLa $modelsName = StringUtilities::foreignKeyToClassName($link->attr); if(!empty($this->getController()->viewBuilder()->getVar('vv_primary_link_model'))) { // $link doesn't seem to handle table aliases (eg "Groups" instead of "RecipientGroups" for - // Notifications)). + // Notifications)). This may also return a plugin qualified path (eg SshKeyAuthenticator.SshKeyAuthenticators). $modelsName = $this->getController()->viewBuilder()->getVar('vv_primary_link_model'); } $modelPath = $modelsName; if(!empty($link->plugin) && !str_starts_with($modelsName, $link->plugin . '.')) { - // eg: "CoreEnroller.AttributeCollectors" + // eg: "CoreEnroller.AttributeCollectors", however check first since we may have the + // path from vv_primary_link_model. $modelPath = $link->plugin . '.' . $modelsName; } diff --git a/app/src/Controller/StandardController.php b/app/src/Controller/StandardController.php index 2e33fef2c..7d3583784 100644 --- a/app/src/Controller/StandardController.php +++ b/app/src/Controller/StandardController.php @@ -252,7 +252,6 @@ public function delete($id) { try { $obj = $table->findById($id)->firstOrFail(); -// XXX throw 404 on RESTful not found? // By default, a delete is a soft delete. The exceptions are when // deleting a CO (AR-CO-1) or when an expunge flag is passed and // expunge is enabled within the CO (XXX not yet implemented). diff --git a/app/src/Model/Table/ExternalIdentitySourcesTable.php b/app/src/Model/Table/ExternalIdentitySourcesTable.php index 06acd84bf..ee793b20b 100644 --- a/app/src/Model/Table/ExternalIdentitySourcesTable.php +++ b/app/src/Model/Table/ExternalIdentitySourcesTable.php @@ -89,7 +89,7 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'plugins' => [ 'type' => 'plugin', - 'pluginType' => 'source' + 'pluginType' => 'external_identity_source' ], 'pipelines' => [ 'type' => 'select', diff --git a/app/src/Model/Table/IdentifierAssignmentsTable.php b/app/src/Model/Table/IdentifierAssignmentsTable.php index a502c759a..d34d19f8d 100644 --- a/app/src/Model/Table/IdentifierAssignmentsTable.php +++ b/app/src/Model/Table/IdentifierAssignmentsTable.php @@ -112,7 +112,7 @@ public function initialize(array $config): void { ], 'plugins' => [ 'type' => 'plugin', - 'pluginType' => 'assigner' + 'pluginType' => 'identifier_assignment' ], 'statuses' => [ 'type' => 'enum', diff --git a/app/src/Model/Table/PluginsTable.php b/app/src/Model/Table/PluginsTable.php index e9c69c419..f0f271b0c 100644 --- a/app/src/Model/Table/PluginsTable.php +++ b/app/src/Model/Table/PluginsTable.php @@ -284,7 +284,9 @@ public function getPluginModelsByType(\App\Model\Entity\Plugin $plugin, string $ if(isset($pConfig->$type) && is_array($pConfig->$type)) { return $pConfig->$type; } else { - $this->llog('debug', "Plugin $plugin->plugin does not have a valid types configuration"); + // This plugin does not implement the requested type. Don't log here since it'll + // cause noise and confusion in the logs. + // $this->llog('debug', "Plugin $plugin->plugin does not have a valid types configuration"); } } else { $this->llog('debug', "Plugin $plugin->plugin does not have a plugin.json file"); @@ -340,7 +342,7 @@ public function pluginPath(\App\Model\Entity\Plugin $plugin, string $file): stri */ protected function readPluginConfig(\App\Model\Entity\Plugin $plugin, string $key): ?object { - $cfg = $this->pluginPath($plugin, 'src' . DS . 'config' . DS . 'plugin.json');; + $cfg = $this->pluginPath($plugin, 'config' . DS . 'plugin.json');; $json = file_get_contents($cfg); diff --git a/app/src/Model/Table/ProvisioningTargetsTable.php b/app/src/Model/Table/ProvisioningTargetsTable.php index a8395ce97..75f723ba4 100644 --- a/app/src/Model/Table/ProvisioningTargetsTable.php +++ b/app/src/Model/Table/ProvisioningTargetsTable.php @@ -92,7 +92,7 @@ public function initialize(array $config): void { $this->setAutoViewVars([ 'plugins' => [ 'type' => 'plugin', - 'pluginType' => 'provisioner' + 'pluginType' => 'provisioning_target' ], 'provisioningGroups' => [ 'type' => 'select', @@ -200,7 +200,7 @@ public function provision( try { $this->llog('trace', "Provisioning $provisionedModel for $pluginModel (context: $context)", $t->id); - $result = $this->$pluginModel->provision($t->$uPluginModel, $provisionedModel, $data, $eligibility); + $result = $this->$pluginModel->provision($t, $provisionedModel, $data, $eligibility); $this->alog('trace', $result); diff --git a/app/templates/Jobs/fields.inc b/app/templates/Jobs/fields.inc index c34dece6e..a6e16d5f2 100644 --- a/app/templates/Jobs/fields.inc +++ b/app/templates/Jobs/fields.inc @@ -40,7 +40,10 @@ if($vv_action == 'add') { print $this->element('form/listItem', [ 'arguments' => [ 'fieldName' => 'plugin', - 'fieldLabel' => __d('controller', 'Jobs', [1]) + 'fieldLabel' => __d('controller', 'Jobs', [1]), + 'fieldOptions' => [ + 'type' => 'text' + ] ] ]);