Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Implement SystemOfRecord trust_mode (CO-2534)
Benn Oshrin committed Apr 1, 2023
1 parent be1e36c commit de23813
Showing 14 changed files with 507 additions and 27 deletions.
2 changes: 1 addition & 1 deletion app/config/app.php
@@ -207,7 +207,7 @@
*/
'Error' => [
'errorLevel' => E_ALL,
'exceptionRenderer' => ExceptionRenderer::class,
//'exceptionRenderer' => ExceptionRenderer::class,
'skipLog' => [],
'log' => true,
'trace' => true,
11 changes: 4 additions & 7 deletions app/config/bootstrap.php
@@ -37,8 +37,8 @@
use Cake\Database\TypeFactory;
use Cake\Database\Type\StringType;
use Cake\Datasource\ConnectionManager;
use Cake\Error\ConsoleErrorHandler;
use Cake\Error\ErrorHandler;
use Cake\Error\ErrorTrap;
use Cake\Error\ExceptionTrap;
use Cake\Http\ServerRequest;
use Cake\Log\Log;
use Cake\Mailer\Mailer;
@@ -128,11 +128,8 @@
* Register application error and exception handlers.
*/
$isCli = PHP_SAPI === 'cli';
if ($isCli) {
(new ConsoleErrorHandler(Configure::read('Error')))->register();
} else {
(new ErrorHandler(Configure::read('Error')))->register();
}
(new ErrorTrap(Configure::read('Error')))->register();
(new ExceptionTrap(Configure::read('Error')))->register();

/*
* Include the CLI bootstrap overrides.
1 change: 1 addition & 0 deletions app/config/schema/schema.json
@@ -230,6 +230,7 @@
"id": {},
"matchgrid_id": {},
"label": { "type": "string", "size": 64 },
"trust_mode": { "type": "string", "size": 2 },
"resolution_mode": { "type": "string", "size": 2 },
"notification_email": { "type": "string", "size": 80 }
},
54 changes: 54 additions & 0 deletions app/resources/locales/en_US/default.po
@@ -53,6 +53,9 @@ msgid "match.banner.api_users.platform"
msgstr "This page is for configuring Platform API Users, which have full read/write access to the entire platform. To create API Users restricted to a given Matchgrid, go to the management page for the desired Matchgrid and select <i>API Users</i> from there.<hr />The Match API is available at {0}"

### Command Line text
msgid "match.cmd.arg.version"
msgstr "Version to upgrade to (default: current RELEASE)"

msgid "match.cmd.bl.index.off"
msgstr "Dropping matchgrid indexes..."

@@ -83,12 +86,21 @@ msgstr "Username of initial platform administrator"
msgid "match.cmd.opt.force"
msgstr "Force a rerun of setup (only if you know what you are doing)"

msgid "match.cmd.opt.forcecurrent"
msgstr "Force the specified current version -- ADVANCED USERS ONLY"

msgid "match.cmd.opt.not"
msgstr "Calculate changes but do not apply"

msgid "match.cmd.opt.skip-match"
msgstr "Do not run Match Rules while processing records"

msgid "match.cmd.opt.skipdatabase"
msgstr "Skip database schema update -- ADVANCED USERS ONLY"

msgid "match.cmd.opt.skipvalidation"
msgstr "Skip version validation -- ADVANCED USERS ONLY"

msgid "match.cmd.se.admin"
msgstr "- Creating initial administrator permission"

@@ -101,6 +113,24 @@ msgstr "Setup appears to have already run"
msgid "match.cmd.se.salt"
msgstr "- Generating salt file"

msgid "match.cmd.ug.current"
msgstr "Current version: {0}"

msgid "match.cmd.ug.ok"
msgstr "Upgrade completed successfully"

msgid "match.cmd.ug.post"
msgstr "Executing post-database step ({0})"

msgid "match.cmd.ug.pre"
msgstr "Executing pre-database step ({0})"

msgid "match.cmd.ug.target"
msgstr "Target version: {0}"

msgid "match.cmd.ug.120.trust_mode"
msgstr "- Populating default values for trust_mode"

### Controllers (Models)
msgid "match.ct.ApiUsers"
msgstr "{0,plural,=1{API User} other{API Users}}"
@@ -245,6 +275,12 @@ msgstr "Suspended"
msgid "match.en.StatusEnum.S.badge"
msgstr "Danger"

msgid "match.en.TrustModeEnum.S"
msgstr "Standard"

msgid "match.en.TrustModeEnum.T"
msgstr "Trust"

### Error Messages
msgid "match.er.args"
msgstr "Incorrect arguments provided to {0}"
@@ -258,6 +294,21 @@ msgstr "Error at line {0}: {1}"
msgid "match.er.build"
msgstr "Error applying matchgrid schema: {0}"

msgid "match.er.cmd.ug.blocked"
msgstr "Cannot automatically upgrade past version {0}. Please upgrade to that version first."

msgid "match.er.cmd.ug.fail"
msgstr "ERROR: Upgrade failed"

msgid "match.er.cmd.ug.order"
msgstr "Target version is before current version (cannot downgrade)"

msgid "match.er.cmd.ug.same"
msgstr "Current and target versions are the same"

msgid "match.er.cmd.ug.version"
msgstr "Unknown version \"{0}\""

msgid "match.er.db.connect"
msgstr "Failed to connect to database: {0}"

@@ -561,6 +612,9 @@ msgstr "Table Name"
msgid "match.fd.table_name.desc"
msgstr "Unique name for matchgrid, must be a valid SQL identifier (will be prefixed mg_ for actual table name)"

msgid "match.fd.trust_mode"
msgstr "Trust Mode"

msgid "match.fd.url"
msgstr "URL"

307 changes: 307 additions & 0 deletions app/src/Command/UpgradeVersionCommand.php
@@ -0,0 +1,307 @@
<?php
/**
* COmanage Match Upgrade Version Command
*
* Portions licensed to the University Corporation for Advanced Internet
* Development, Inc. ("UCAID") under one or more contributor license agreements.
* See the NOTICE file distributed with this work for additional information
* regarding copyright ownership.
*
* UCAID licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @link https://www.internet2.edu/comanage COmanage Project
* @package match
* @since COmanage Match v1.2.0
* @license Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
*/

declare(strict_types = 1);

namespace App\Command;

use App\Application;
use Cake\Console\Arguments;
use Cake\Console\Command;
use Cake\Console\CommandRunner;
use Cake\Console\ConsoleIo;
use Cake\Console\ConsoleOptionParser;
use Cake\Utility\Security;
use \App\Command\DatabaseCommand;
use \App\Lib\Enum\PermissionEnum;

class UpgradeVersionCommand extends Command {
// Make sure to keep this list in order so we can walk the array rather than compare
// version strings. You must specify the 'block' parameter. If you flag a version
// as blocking, be sure to document why.
// In general, it should be safe to run the upgrade command multiple times (eg: if
// a deployment is tracking develop). Any pre or post operations should either
// operate in a "safe" way or else check if they've already been run.
protected $versions = [
"1.0.0" => ['block' => false],
"1.1.0" => ['block' => false],
"1.2.0" => ['block' => false, 'post' => 'post120']
];

// ConsoleIo
protected $io = null;

/**
* Register command specific options.
*
* @since COmanage Match v1.2.0
* @param ConsoleOptionParser $parser Console Option Parser
* @return ConsoleOptionParser Console Option Parser
*/

public function buildOptionParser(ConsoleOptionParser $parser): ConsoleOptionParser {
$parser->addArgument(
'version',
[
'help' => __('match.cmd.arg.version'),
'required' => false
]
)->addOption(
'forcecurrent',
[
'short' => 'f',
'help' => __('match.cmd.opt.forcecurrent'),
'boolean' => false,
'default' => false
]
)->addOption(
'skipdatabase',
[
'short' => 'D',
'help' => __('match.cmd.opt.skipdatabase'),
'boolean' => true,
'default' => false
]
)->addOption(
'skipvalidation',
[
'short' => 'X',
'help' => __('match.cmd.opt.skipvalidation'),
'boolean' => false,
'default' => false
]
);

return $parser;
}

/**
* Execute the Upgrade Version Command.
*
* @since COmanage Match v1.0.0
* @param Arguments $args Command Arguments
* @param ConsoleIo $io Console IO
*/

public function execute(Arguments $args, ConsoleIo $io) {
global $argv;

$this->io = $io;

// Determine the (PHP) code version (or use the one passed in)
$targetVersion = $args->getArgument('version');

if(!$targetVersion) {
// Read the current release from the VERSION file
$versionFile = CONFIG . DS . "VERSION";

$targetVersion = rtrim(file_get_contents($versionFile));
}

// Pull the current database version (or use the one passed in)
$Meta = $this->getTableLocator()->get('Meta');
$currentVersion = $args->getOption('forcecurrent');

if(!$currentVersion) {
$currentVersion = $Meta->getUpgradeVersion();
}

$io->out(__('match.cmd.ug.current', [$currentVersion]));
$io->out(__('match.cmd.ug.target', [$targetVersion]));

if(!$args->getOption('skipvalidation')) {
// Validate the version path
try {
$this->validateVersions($currentVersion, $targetVersion);
}
catch(\Exception $e) {
$io->out($e->getMessage());
$io->out(__('match.er.cmd.ug.fail'));
return;
}
}

// Run appropriate pre-database steps

$fromFound = false;

foreach($this->versions as $version => $params) {
if($version == $currentVersion) {
// Note we don't actually want to run the steps for $currentVersion
$fromFound = true;
continue;
}

if(!$fromFound) {
// We haven't reached the from version yet
continue;
}

if(isset($params['pre'])) {
$fn = $params['pre'];

$io->out(__('match.cmd.ug.post', [$fn]));
$this->$fn();
}

if($version == $targetVersion) {
// We're done
break;
}
}

if(!$args->getOption('skipdatabase')) {
// Call database shell
$this->executeCommand(DatabaseCommand::class);
}

// Run appropriate post-database steps

$fromFound = false;

foreach($this->versions as $version => $params) {
if($version == $currentVersion) {
// Note we don't actually want to run the steps for $currentVersion
$fromFound = true;
continue;
}

if(!$fromFound) {
// We haven't reached the from version yet
continue;
}

if(isset($params['post'])) {
$fn = $params['post'];

$io->out(__('match.cmd.ug.post', [$fn]));
$this->$fn();
}

if($version == $targetVersion) {
// We're done
break;
}
}

// Now that we're done, update the current version
$Meta->setUpgradeVersion($targetVersion);

$io->out(__('match.cmd.ug.ok'));

return;
}

/**
* Process post-upgrade steps for v1.2.0.
*
* @since COmanage Match v1.2.0
*/

protected function post120() {
// Set all Systems of Record to operate in Standard Trust Mode, which is
// the effective behavior prior to v1.2.0.

$this->io->out(__('match.cmd.ug.120.trust_mode'));
$this->setNewDefault('SystemsOfRecord', 'trust_mode', 'S');
}

/**
* Set a new value for all rows in a column. $value will replace any null
* entries, but not any entries already set.
*
* @since COmanage Match v1.2.0
* @param string $table Table name, in StudlyCap format
* @param string $column Column name
* @param string $value New default value
*/

protected function setNewDefault(string $table, string $column, string $value) {
$Table = $this->getTableLocator()->get($table);

$Table->query()
->update()
->set([$column => $value])
->where(["$column IS NULL"])
->execute();
}

/**
* Validate the requested from and to versions.
*
* @since COmanage Match v1.2.0
* @param string $from "From" version (current database)
* @param string $to "To" version (current codebase)
* @return bool true if the requested range is valid
* @throws InvalidArgumentException
*/

protected function validateVersions(string $from, string $to): bool {
// First make sure these are valid versions

if(!array_key_exists($from, $this->versions)) {
throw new \InvalidArgumentException(__('match.er.cmd.ug.version', [$from]));
}

if(!array_key_exists($to, $this->versions)) {
throw new \InvalidArgumentException(__('match.er.cmd.ug.version', [$to]));
}

// If $from and $to are the same, nothing to do.

if($from == $to) {
throw new \InvalidArgumentException(__('match.er.cmd.ug.same'));
}

// Walk through the version array and check our version path

$fromFound = false;

foreach($this->versions as $version => $params) {
$blocks = $params['block'];

if($version == $from) {
$fromFound = true;
} elseif($version == $to) {
if(!$fromFound) {
// Can't downgrade ($from must preceed $to)
throw new \InvalidArgumentException(__('match.er.cmd.ug.order'));
} else {
// We're good to go
break;
}
} else {
if($fromFound && $blocks) {
// We can't pass a blocker version
throw new \InvalidArgumentException(__('match.er.cmd.ug.blocked', [$version]));
}
}
}

return true;
}
}
2 changes: 0 additions & 2 deletions app/src/Controller/AppController.php
@@ -72,8 +72,6 @@ public function initialize(): void {
],
]);

$this->loadComponent('Paginator');

/*
* Enable the following components for recommended CakePHP security settings.
* see https://book.cakephp.org/3.0/en/controllers/components/security.html
2 changes: 1 addition & 1 deletion app/src/Controller/StandardController.php
@@ -405,7 +405,7 @@ public function index() {
// but it doesn't seem to work in Cake 3/4. So we just use $this->pagination
// ourselves here.

$this->set($tableName, $this->Paginator->paginate($query, $this->pagination));
$this->set($tableName, $this->paginate($query, $this->pagination));
$this->set('vv_tablename', $tableName);
$this->set('vv_modelname', $modelsName);

16 changes: 16 additions & 0 deletions app/src/Lib/Enum/StandardEnum.php
@@ -55,4 +55,20 @@ public static function getLocalizedConsts() {

return $ret;
}

/**
* Get the values for the constants in the Enumeration.
*
* @since COmanage Match v1.2.0
* @return array Array of enumeration values
*/

public static function getConstValues() : array {
// Get the keys for this enum
$reflect = new ReflectionClass(get_called_class());

$consts = $reflect->getConstants();

return array_values($consts);
}
}
35 changes: 35 additions & 0 deletions app/src/Lib/Enum/TrustModeEnum.php
@@ -0,0 +1,35 @@
<?php
/**
* COmanage Match Trust Mode Enum
*
* Portions licensed to the University Corporation for Advanced Internet
* Development, Inc. ("UCAID") under one or more contributor license agreements.
* See the NOTICE file distributed with this work for additional information
* regarding copyright ownership.
*
* UCAID licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with the
* License. You may obtain a copy of the License at:
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* @link https://www.internet2.edu/comanage COmanage Project
* @package match
* @since COmanage Match v1.2.0
* @license Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
*/

declare(strict_types = 1);

namespace App\Lib\Enum;

class TrustModeEnum extends StandardEnum {
const Standard = 'S';
const Trust = 'T';
}
39 changes: 34 additions & 5 deletions app/src/Lib/Match/MatchService.php
@@ -37,6 +37,7 @@
use \App\Lib\Enum\ReferenceIdEnum;
use \App\Lib\Enum\SearchTypeEnum;
use \App\Lib\Enum\StatusEnum;
use \App\Lib\Enum\TrustModeEnum;

class MatchService { //extends PostgresService {
use \App\Lib\Traits\DatabaseTrait;
@@ -590,6 +591,7 @@ public function removeReferenceId(string $sor, string $sorid) {
* @param string $sor SOR Label
* @param string $sorid SOR ID
* @param AttributeManager $attributes Search attibutes
* @param bool $skipSor If true, do not match against existing entries from $sor
* @return ResultManager Result Manager
* @throws LogicException
* @throws RuntimeException
@@ -598,7 +600,8 @@ public function removeReferenceId(string $sor, string $sorid) {
protected function search(string $mode,
string $sor,
string $sorid,
AttributeManager $attributes) {
AttributeManager $attributes,
bool $skipSor=false): ResultManager {
$results = new ResultManager;

$results->setConfidenceMode($mode);
@@ -757,6 +760,11 @@ protected function search(string $mode,

$vals = [];

if($skipSor) {
$sql .= " AND sor <> ?";
$vals[] = $sor;
}

foreach(array_keys($attrSql['sql']) as $attrId) {
if(count($attrSql['sql'][$attrId]) > 1) {
// We OR together all of the clauses
@@ -821,10 +829,25 @@ protected function search(string $mode,
*/

public function searchReferenceId(string $sor, string $sorid, AttributeManager $attributes) {
// Before we start, pull the SystemOfRecord configuration.

$SystemsOfRecord = TableRegistry::getTableLocator()->get('SystemsOfRecord');

$trustMode = $SystemsOfRecord->getTrustMode($this->mgConfig->id, $sor);

if($trustMode == TrustModeEnum::Trust) {
Log::write('debug', $sor . "/" . $sorid . " Trust Mode enabled, ignoring existing records in the same SOR");
}

// First try canonical matches
$canonicalMatches = $this->search(ConfidenceModeEnum::Canonical, $sor, $sorid, $attributes);

// XXX add some logging on match candidates? or maybe in search()
$canonicalMatches = $this->search(
mode: ConfidenceModeEnum::Canonical,
sor: $sor,
sorid: $sorid,
attributes: $attributes,
skipSor: $trustMode == TrustModeEnum::Trust
);

switch($canonicalMatches->count()) {
case 1:
// Exact match, return
@@ -843,7 +866,13 @@ public function searchReferenceId(string $sor, string $sorid, AttributeManager $
}

// Next try potential matches
$potentialMatches = $this->search(ConfidenceModeEnum::Potential, $sor, $sorid, $attributes);
$potentialMatches = $this->search(
mode: ConfidenceModeEnum::Potential,
sor: $sor,
sorid: $sorid,
attributes: $attributes,
skipSor: $trustMode == TrustModeEnum::Trust
);

// The calling code generally checks to see if any rules successfully ran,
// since if there were no valid attributes or rules we treat that as an error.
21 changes: 16 additions & 5 deletions app/src/Model/Table/MetaTable.php
@@ -34,18 +34,29 @@
use Cake\Validation\Validator;

class MetaTable extends Table {
/**
* Update the current "upgrade" version.
*
* @since COmanage Match v1.2.0
* @return string Current "upgrade" version, per the database
* @throws RuntimeException
*/
public function getUpgradeVersion(): string {
$meta = $this->get(1);

return $meta->upgrade_version;
}

/**
* Update the current "upgrade" version.
*
* @since COmanage Match v1.0.0
* @param String $version New current version
* @return Boolean True on success
* @param string $version New current version
* @return bool True on success
* @throws RuntimeException
*/

public function setUpgradeVersion($version) {
// XXX log this? print "Setting version to " . $version . "\n";

public function setUpgradeVersion(string $version): bool {
$meta = $this->newEmptyEntity();
$meta->id = 1;
$meta->upgrade_version = $version;
36 changes: 32 additions & 4 deletions app/src/Model/Table/SystemsOfRecordTable.php
@@ -33,6 +33,7 @@
use Cake\Validation\Validator;

use \App\Lib\Enum\ResolutionModeEnum;
use \App\Lib\Enum\TrustModeEnum;

class SystemsOfRecordTable extends Table {
use \App\Lib\Traits\AutoViewVarsTrait;
@@ -63,10 +64,33 @@ public function initialize(array $config): void {
'resolutionModes' => [
'type' => 'enum',
'class' => 'ResolutionModeEnum'
],
'trustModes' => [
'type' => 'enum',
'class' => 'TrustModeEnum'
]
]);
}

/**
* Determine the trust mode setting for the requested SOR within the matchgrid.
*
* @since COmanage Match v1.2.0
* @param int $matchgridId Matchgrid ID
* @param string $sorLabel System of Record Label
*/

public function getTrustMode(int $matchgridId, string $sorLabel) {
$systemOfRecord = $this->find()
->where([
'matchgrid_id' => $matchgridId,
'label' => $sorLabel
])
->firstOrFail();

return $systemOfRecord->trust_mode;
}

/**
* Set validation rules.
*
@@ -91,13 +115,17 @@ public function validationDefault(Validator $validator): Validator {
);
$validator->notBlank('matchgrid_id');

$validator->add(
'trust_mode',
'content',
[ 'rule' => [ 'inList', TrustModeEnum::getConstValues() ] ]
);
$validator->notEmptyString('trust_mode');

$validator->add(
'resolution_mode',
'content',
[ 'rule' => [ 'inList', [
ResolutionModeEnum::External,
ResolutionModeEnum::Interactive
] ] ]
[ 'rule' => [ 'inList', ResolutionModeEnum::getConstValues() ] ]
);
$validator->notEmptyString('resolution_mode');

3 changes: 1 addition & 2 deletions app/templates/Matchgrids/reconcile.php
@@ -58,7 +58,6 @@
// String: attribute field name | Array of arrays: [attribute value, candidate id, different?1:0]
$canAttr = array();
for($i = 0; $i < count($fieldNames); $i++) {

// Put the field name in the first column.
$canAttr[$i][0] = $fieldNames[$i];

@@ -68,7 +67,7 @@
foreach($c as $key => $val) {
if($key == $fieldNames[$i]) {
$id = $c['id'];
$diff = in_array($key,$vv_candidate_diff[$id]) ? 1 : 0;
$diff = (isset($vv_candidate_diff[$id]) && in_array($key,$vv_candidate_diff[$id])) ? 1 : 0;
$canAttr[$i][1][] = [$val, $id, $diff];
}
}
5 changes: 5 additions & 0 deletions app/templates/SystemsOfRecord/fields.inc
@@ -26,6 +26,7 @@
*/

use \App\Lib\Enum\ResolutionModeEnum;
use \App\Lib\Enum\TrustModeEnum;
?>
<script type="text/javascript">
// JS specific to these fields
@@ -51,6 +52,10 @@ use \App\Lib\Enum\ResolutionModeEnum;
if($action == 'add' || $action == 'edit') {
print $this->Field->control('label');

print $this->Field->control('trust_mode',
['empty' => true,
'default' => TrustModeEnum::Standard ]);

print $this->Field->control('resolution_mode',
['empty' => true,
'onChange' => 'fields_update_gadgets();']);

0 comments on commit de23813

Please sign in to comment.