diff --git a/app/config/app.php b/app/config/app.php index 8ab241079..cb4993ef9 100644 --- a/app/config/app.php +++ b/app/config/app.php @@ -207,7 +207,7 @@ */ 'Error' => [ 'errorLevel' => E_ALL, - 'exceptionRenderer' => ExceptionRenderer::class, + //'exceptionRenderer' => ExceptionRenderer::class, 'skipLog' => [], 'log' => true, 'trace' => true, diff --git a/app/config/bootstrap.php b/app/config/bootstrap.php index f93d37b82..131f566c8 100644 --- a/app/config/bootstrap.php +++ b/app/config/bootstrap.php @@ -37,8 +37,8 @@ use Cake\Database\TypeFactory; use Cake\Database\Type\StringType; use Cake\Datasource\ConnectionManager; -use Cake\Error\ConsoleErrorHandler; -use Cake\Error\ErrorHandler; +use Cake\Error\ErrorTrap; +use Cake\Error\ExceptionTrap; use Cake\Http\ServerRequest; use Cake\Log\Log; use Cake\Mailer\Mailer; @@ -128,11 +128,8 @@ * Register application error and exception handlers. */ $isCli = PHP_SAPI === 'cli'; -if ($isCli) { - (new ConsoleErrorHandler(Configure::read('Error')))->register(); -} else { - (new ErrorHandler(Configure::read('Error')))->register(); -} +(new ErrorTrap(Configure::read('Error')))->register(); +(new ExceptionTrap(Configure::read('Error')))->register(); /* * Include the CLI bootstrap overrides. diff --git a/app/config/schema/schema.json b/app/config/schema/schema.json index c452fa493..ce17ffe5a 100644 --- a/app/config/schema/schema.json +++ b/app/config/schema/schema.json @@ -230,6 +230,7 @@ "id": {}, "matchgrid_id": {}, "label": { "type": "string", "size": 64 }, + "trust_mode": { "type": "string", "size": 2 }, "resolution_mode": { "type": "string", "size": 2 }, "notification_email": { "type": "string", "size": 80 } }, diff --git a/app/resources/locales/en_US/default.po b/app/resources/locales/en_US/default.po index 60fd2dc93..2ebe2f156 100644 --- a/app/resources/locales/en_US/default.po +++ b/app/resources/locales/en_US/default.po @@ -53,6 +53,9 @@ msgid "match.banner.api_users.platform" msgstr "This page is for configuring Platform API Users, which have full read/write access to the entire platform. To create API Users restricted to a given Matchgrid, go to the management page for the desired Matchgrid and select API Users from there.
The Match API is available at {0}" ### Command Line text +msgid "match.cmd.arg.version" +msgstr "Version to upgrade to (default: current RELEASE)" + msgid "match.cmd.bl.index.off" msgstr "Dropping matchgrid indexes..." @@ -83,12 +86,21 @@ msgstr "Username of initial platform administrator" msgid "match.cmd.opt.force" msgstr "Force a rerun of setup (only if you know what you are doing)" +msgid "match.cmd.opt.forcecurrent" +msgstr "Force the specified current version -- ADVANCED USERS ONLY" + msgid "match.cmd.opt.not" msgstr "Calculate changes but do not apply" msgid "match.cmd.opt.skip-match" msgstr "Do not run Match Rules while processing records" +msgid "match.cmd.opt.skipdatabase" +msgstr "Skip database schema update -- ADVANCED USERS ONLY" + +msgid "match.cmd.opt.skipvalidation" +msgstr "Skip version validation -- ADVANCED USERS ONLY" + msgid "match.cmd.se.admin" msgstr "- Creating initial administrator permission" @@ -101,6 +113,24 @@ msgstr "Setup appears to have already run" msgid "match.cmd.se.salt" msgstr "- Generating salt file" +msgid "match.cmd.ug.current" +msgstr "Current version: {0}" + +msgid "match.cmd.ug.ok" +msgstr "Upgrade completed successfully" + +msgid "match.cmd.ug.post" +msgstr "Executing post-database step ({0})" + +msgid "match.cmd.ug.pre" +msgstr "Executing pre-database step ({0})" + +msgid "match.cmd.ug.target" +msgstr "Target version: {0}" + +msgid "match.cmd.ug.120.trust_mode" +msgstr "- Populating default values for trust_mode" + ### Controllers (Models) msgid "match.ct.ApiUsers" msgstr "{0,plural,=1{API User} other{API Users}}" @@ -245,6 +275,12 @@ msgstr "Suspended" msgid "match.en.StatusEnum.S.badge" msgstr "Danger" +msgid "match.en.TrustModeEnum.S" +msgstr "Standard" + +msgid "match.en.TrustModeEnum.T" +msgstr "Trust" + ### Error Messages msgid "match.er.args" msgstr "Incorrect arguments provided to {0}" @@ -258,6 +294,21 @@ msgstr "Error at line {0}: {1}" msgid "match.er.build" msgstr "Error applying matchgrid schema: {0}" +msgid "match.er.cmd.ug.blocked" +msgstr "Cannot automatically upgrade past version {0}. Please upgrade to that version first." + +msgid "match.er.cmd.ug.fail" +msgstr "ERROR: Upgrade failed" + +msgid "match.er.cmd.ug.order" +msgstr "Target version is before current version (cannot downgrade)" + +msgid "match.er.cmd.ug.same" +msgstr "Current and target versions are the same" + +msgid "match.er.cmd.ug.version" +msgstr "Unknown version \"{0}\"" + msgid "match.er.db.connect" msgstr "Failed to connect to database: {0}" @@ -561,6 +612,9 @@ msgstr "Table Name" msgid "match.fd.table_name.desc" msgstr "Unique name for matchgrid, must be a valid SQL identifier (will be prefixed mg_ for actual table name)" +msgid "match.fd.trust_mode" +msgstr "Trust Mode" + msgid "match.fd.url" msgstr "URL" diff --git a/app/src/Command/UpgradeVersionCommand.php b/app/src/Command/UpgradeVersionCommand.php new file mode 100644 index 000000000..11fafc18d --- /dev/null +++ b/app/src/Command/UpgradeVersionCommand.php @@ -0,0 +1,307 @@ + ['block' => false], + "1.1.0" => ['block' => false], + "1.2.0" => ['block' => false, 'post' => 'post120'] + ]; + + // ConsoleIo + protected $io = null; + + /** + * Register command specific options. + * + * @since COmanage Match v1.2.0 + * @param ConsoleOptionParser $parser Console Option Parser + * @return ConsoleOptionParser Console Option Parser + */ + + public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser { + $parser->addArgument( + 'version', + [ + 'help' => __('match.cmd.arg.version'), + 'required' => false + ] + )->addOption( + 'forcecurrent', + [ + 'short' => 'f', + 'help' => __('match.cmd.opt.forcecurrent'), + 'boolean' => false, + 'default' => false + ] + )->addOption( + 'skipdatabase', + [ + 'short' => 'D', + 'help' => __('match.cmd.opt.skipdatabase'), + 'boolean' => true, + 'default' => false + ] + )->addOption( + 'skipvalidation', + [ + 'short' => 'X', + 'help' => __('match.cmd.opt.skipvalidation'), + 'boolean' => false, + 'default' => false + ] + ); + + return $parser; + } + + /** + * Execute the Upgrade Version Command. + * + * @since COmanage Match v1.0.0 + * @param Arguments $args Command Arguments + * @param ConsoleIo $io Console IO + */ + + public function execute(Arguments $args, ConsoleIo $io) { + global $argv; + + $this->io = $io; + + // Determine the (PHP) code version (or use the one passed in) + $targetVersion = $args->getArgument('version'); + + if(!$targetVersion) { + // Read the current release from the VERSION file + $versionFile = CONFIG . DS . "VERSION"; + + $targetVersion = rtrim(file_get_contents($versionFile)); + } + + // Pull the current database version (or use the one passed in) + $Meta = $this->getTableLocator()->get('Meta'); + $currentVersion = $args->getOption('forcecurrent'); + + if(!$currentVersion) { + $currentVersion = $Meta->getUpgradeVersion(); + } + + $io->out(__('match.cmd.ug.current', [$currentVersion])); + $io->out(__('match.cmd.ug.target', [$targetVersion])); + + if(!$args->getOption('skipvalidation')) { + // Validate the version path + try { + $this->validateVersions($currentVersion, $targetVersion); + } + catch(\Exception $e) { + $io->out($e->getMessage()); + $io->out(__('match.er.cmd.ug.fail')); + return; + } + } + + // Run appropriate pre-database steps + + $fromFound = false; + + foreach($this->versions as $version => $params) { + if($version == $currentVersion) { + // Note we don't actually want to run the steps for $currentVersion + $fromFound = true; + continue; + } + + if(!$fromFound) { + // We haven't reached the from version yet + continue; + } + + if(isset($params['pre'])) { + $fn = $params['pre']; + + $io->out(__('match.cmd.ug.post', [$fn])); + $this->$fn(); + } + + if($version == $targetVersion) { + // We're done + break; + } + } + + if(!$args->getOption('skipdatabase')) { + // Call database shell + $this->executeCommand(DatabaseCommand::class); + } + + // Run appropriate post-database steps + + $fromFound = false; + + foreach($this->versions as $version => $params) { + if($version == $currentVersion) { + // Note we don't actually want to run the steps for $currentVersion + $fromFound = true; + continue; + } + + if(!$fromFound) { + // We haven't reached the from version yet + continue; + } + + if(isset($params['post'])) { + $fn = $params['post']; + + $io->out(__('match.cmd.ug.post', [$fn])); + $this->$fn(); + } + + if($version == $targetVersion) { + // We're done + break; + } + } + + // Now that we're done, update the current version + $Meta->setUpgradeVersion($targetVersion); + + $io->out(__('match.cmd.ug.ok')); + + return; + } + + /** + * Process post-upgrade steps for v1.2.0. + * + * @since COmanage Match v1.2.0 + */ + + protected function post120() { + // Set all Systems of Record to operate in Standard Trust Mode, which is + // the effective behavior prior to v1.2.0. + + $this->io->out(__('match.cmd.ug.120.trust_mode')); + $this->setNewDefault('SystemsOfRecord', 'trust_mode', 'S'); + } + + /** + * Set a new value for all rows in a column. $value will replace any null + * entries, but not any entries already set. + * + * @since COmanage Match v1.2.0 + * @param string $table Table name, in StudlyCap format + * @param string $column Column name + * @param string $value New default value + */ + + protected function setNewDefault(string $table, string $column, string $value) { + $Table = $this->getTableLocator()->get($table); + + $Table->query() + ->update() + ->set([$column => $value]) + ->where(["$column IS NULL"]) + ->execute(); + } + + /** + * Validate the requested from and to versions. + * + * @since COmanage Match v1.2.0 + * @param string $from "From" version (current database) + * @param string $to "To" version (current codebase) + * @return bool true if the requested range is valid + * @throws InvalidArgumentException + */ + + protected function validateVersions(string $from, string $to): bool { + // First make sure these are valid versions + + if(!array_key_exists($from, $this->versions)) { + throw new \InvalidArgumentException(__('match.er.cmd.ug.version', [$from])); + } + + if(!array_key_exists($to, $this->versions)) { + throw new \InvalidArgumentException(__('match.er.cmd.ug.version', [$to])); + } + + // If $from and $to are the same, nothing to do. + + if($from == $to) { + throw new \InvalidArgumentException(__('match.er.cmd.ug.same')); + } + + // Walk through the version array and check our version path + + $fromFound = false; + + foreach($this->versions as $version => $params) { + $blocks = $params['block']; + + if($version == $from) { + $fromFound = true; + } elseif($version == $to) { + if(!$fromFound) { + // Can't downgrade ($from must preceed $to) + throw new \InvalidArgumentException(__('match.er.cmd.ug.order')); + } else { + // We're good to go + break; + } + } else { + if($fromFound && $blocks) { + // We can't pass a blocker version + throw new \InvalidArgumentException(__('match.er.cmd.ug.blocked', [$version])); + } + } + } + + return true; + } +} \ No newline at end of file diff --git a/app/src/Controller/AppController.php b/app/src/Controller/AppController.php index ed4ca306e..340e090b3 100644 --- a/app/src/Controller/AppController.php +++ b/app/src/Controller/AppController.php @@ -72,8 +72,6 @@ public function initialize(): void { ], ]); - $this->loadComponent('Paginator'); - /* * Enable the following components for recommended CakePHP security settings. * see https://book.cakephp.org/3.0/en/controllers/components/security.html diff --git a/app/src/Controller/StandardController.php b/app/src/Controller/StandardController.php index d7d350b70..92f2b9806 100644 --- a/app/src/Controller/StandardController.php +++ b/app/src/Controller/StandardController.php @@ -405,7 +405,7 @@ public function index() { // but it doesn't seem to work in Cake 3/4. So we just use $this->pagination // ourselves here. - $this->set($tableName, $this->Paginator->paginate($query, $this->pagination)); + $this->set($tableName, $this->paginate($query, $this->pagination)); $this->set('vv_tablename', $tableName); $this->set('vv_modelname', $modelsName); diff --git a/app/src/Lib/Enum/StandardEnum.php b/app/src/Lib/Enum/StandardEnum.php index 7cd256aae..43a48218a 100644 --- a/app/src/Lib/Enum/StandardEnum.php +++ b/app/src/Lib/Enum/StandardEnum.php @@ -55,4 +55,20 @@ public static function getLocalizedConsts() { return $ret; } + + /** + * Get the values for the constants in the Enumeration. + * + * @since COmanage Match v1.2.0 + * @return array Array of enumeration values + */ + + public static function getConstValues() : array { + // Get the keys for this enum + $reflect = new ReflectionClass(get_called_class()); + + $consts = $reflect->getConstants(); + + return array_values($consts); + } } \ No newline at end of file diff --git a/app/src/Lib/Enum/TrustModeEnum.php b/app/src/Lib/Enum/TrustModeEnum.php new file mode 100644 index 000000000..1b150edce --- /dev/null +++ b/app/src/Lib/Enum/TrustModeEnum.php @@ -0,0 +1,35 @@ +setConfidenceMode($mode); @@ -757,6 +760,11 @@ protected function search(string $mode, $vals = []; + if($skipSor) { + $sql .= " AND sor <> ?"; + $vals[] = $sor; + } + foreach(array_keys($attrSql['sql']) as $attrId) { if(count($attrSql['sql'][$attrId]) > 1) { // We OR together all of the clauses @@ -821,10 +829,25 @@ protected function search(string $mode, */ public function searchReferenceId(string $sor, string $sorid, AttributeManager $attributes) { + // Before we start, pull the SystemOfRecord configuration. + + $SystemsOfRecord = TableRegistry::getTableLocator()->get('SystemsOfRecord'); + + $trustMode = $SystemsOfRecord->getTrustMode($this->mgConfig->id, $sor); + + if($trustMode == TrustModeEnum::Trust) { + Log::write('debug', $sor . "/" . $sorid . " Trust Mode enabled, ignoring existing records in the same SOR"); + } + // First try canonical matches - $canonicalMatches = $this->search(ConfidenceModeEnum::Canonical, $sor, $sorid, $attributes); - -// XXX add some logging on match candidates? or maybe in search() + $canonicalMatches = $this->search( + mode: ConfidenceModeEnum::Canonical, + sor: $sor, + sorid: $sorid, + attributes: $attributes, + skipSor: $trustMode == TrustModeEnum::Trust + ); + switch($canonicalMatches->count()) { case 1: // Exact match, return @@ -843,7 +866,13 @@ public function searchReferenceId(string $sor, string $sorid, AttributeManager $ } // Next try potential matches - $potentialMatches = $this->search(ConfidenceModeEnum::Potential, $sor, $sorid, $attributes); + $potentialMatches = $this->search( + mode: ConfidenceModeEnum::Potential, + sor: $sor, + sorid: $sorid, + attributes: $attributes, + skipSor: $trustMode == TrustModeEnum::Trust + ); // The calling code generally checks to see if any rules successfully ran, // since if there were no valid attributes or rules we treat that as an error. diff --git a/app/src/Model/Table/MetaTable.php b/app/src/Model/Table/MetaTable.php index db01b276a..4f30a4538 100644 --- a/app/src/Model/Table/MetaTable.php +++ b/app/src/Model/Table/MetaTable.php @@ -34,18 +34,29 @@ use Cake\Validation\Validator; class MetaTable extends Table { + /** + * Update the current "upgrade" version. + * + * @since COmanage Match v1.2.0 + * @return string Current "upgrade" version, per the database + * @throws RuntimeException + */ + public function getUpgradeVersion(): string { + $meta = $this->get(1); + + return $meta->upgrade_version; + } + /** * Update the current "upgrade" version. * * @since COmanage Match v1.0.0 - * @param String $version New current version - * @return Boolean True on success + * @param string $version New current version + * @return bool True on success * @throws RuntimeException */ - public function setUpgradeVersion($version) { -// XXX log this? print "Setting version to " . $version . "\n"; - + public function setUpgradeVersion(string $version): bool { $meta = $this->newEmptyEntity(); $meta->id = 1; $meta->upgrade_version = $version; diff --git a/app/src/Model/Table/SystemsOfRecordTable.php b/app/src/Model/Table/SystemsOfRecordTable.php index 992e67df7..98bb2693c 100644 --- a/app/src/Model/Table/SystemsOfRecordTable.php +++ b/app/src/Model/Table/SystemsOfRecordTable.php @@ -33,6 +33,7 @@ use Cake\Validation\Validator; use \App\Lib\Enum\ResolutionModeEnum; +use \App\Lib\Enum\TrustModeEnum; class SystemsOfRecordTable extends Table { use \App\Lib\Traits\AutoViewVarsTrait; @@ -63,10 +64,33 @@ public function initialize(array $config): void { 'resolutionModes' => [ 'type' => 'enum', 'class' => 'ResolutionModeEnum' + ], + 'trustModes' => [ + 'type' => 'enum', + 'class' => 'TrustModeEnum' ] ]); } + /** + * Determine the trust mode setting for the requested SOR within the matchgrid. + * + * @since COmanage Match v1.2.0 + * @param int $matchgridId Matchgrid ID + * @param string $sorLabel System of Record Label + */ + + public function getTrustMode(int $matchgridId, string $sorLabel) { + $systemOfRecord = $this->find() + ->where([ + 'matchgrid_id' => $matchgridId, + 'label' => $sorLabel + ]) + ->firstOrFail(); + + return $systemOfRecord->trust_mode; + } + /** * Set validation rules. * @@ -91,13 +115,17 @@ public function validationDefault(Validator $validator): Validator { ); $validator->notBlank('matchgrid_id'); + $validator->add( + 'trust_mode', + 'content', + [ 'rule' => [ 'inList', TrustModeEnum::getConstValues() ] ] + ); + $validator->notEmptyString('trust_mode'); + $validator->add( 'resolution_mode', 'content', - [ 'rule' => [ 'inList', [ - ResolutionModeEnum::External, - ResolutionModeEnum::Interactive - ] ] ] + [ 'rule' => [ 'inList', ResolutionModeEnum::getConstValues() ] ] ); $validator->notEmptyString('resolution_mode'); diff --git a/app/templates/Matchgrids/reconcile.php b/app/templates/Matchgrids/reconcile.php index 98672707e..03f09913f 100644 --- a/app/templates/Matchgrids/reconcile.php +++ b/app/templates/Matchgrids/reconcile.php @@ -58,7 +58,6 @@ // String: attribute field name | Array of arrays: [attribute value, candidate id, different?1:0] $canAttr = array(); for($i = 0; $i < count($fieldNames); $i++) { - // Put the field name in the first column. $canAttr[$i][0] = $fieldNames[$i]; @@ -68,7 +67,7 @@ foreach($c as $key => $val) { if($key == $fieldNames[$i]) { $id = $c['id']; - $diff = in_array($key,$vv_candidate_diff[$id]) ? 1 : 0; + $diff = (isset($vv_candidate_diff[$id]) && in_array($key,$vv_candidate_diff[$id])) ? 1 : 0; $canAttr[$i][1][] = [$val, $id, $diff]; } } diff --git a/app/templates/SystemsOfRecord/fields.inc b/app/templates/SystemsOfRecord/fields.inc index 56e2684b2..202e41c38 100644 --- a/app/templates/SystemsOfRecord/fields.inc +++ b/app/templates/SystemsOfRecord/fields.inc @@ -26,6 +26,7 @@ */ use \App\Lib\Enum\ResolutionModeEnum; +use \App\Lib\Enum\TrustModeEnum; ?>