Implement "match empty" (CO-2136)

COmanage · Aug 15, 2021 · fba23ec · fba23ec
1 parent 1ec6b4d
commit fba23ec
Show file tree

Hide file tree

Showing 10 changed files with 122 additions and 68 deletions.
diff --git a/app/config/schema/schema.json b/app/config/schema/schema.json
@@ -184,7 +184,8 @@
         "attribute_id": { "type": "integer", "foreignkey": { "table": "attributes", "column": "id" } },
         "crosscheck_attribute_id": { "type": "integer", "foreignkey": { "table": "attributes", "column": "id" } },
         "search_type": { "type": "string", "size": 2 },
-        "required": { "type": "boolean" }
+        "required": { "type": "boolean" },
+        "match_empty": { "type": "boolean" }
       },
       "indexes": {
         "rule_attributes_i1": {

diff --git a/app/src/Lib/Match/MatchService.php b/app/src/Lib/Match/MatchService.php
@@ -543,8 +543,10 @@ protected function search(string $mode,
             continue 2;
           }
 
-          Log::write('debug', $sor . "/" . $sorid . " No value found for optional attribute " . $ruleattr->attribute->name . ", ignoring it");
-          continue;
+          if(!$ruleattr->match_empty) {
+            Log::write('debug', $sor . "/" . $sorid . " No value found for optional attribute " . $ruleattr->attribute->name . ", ignoring it");
+            continue;
+          }
         }
 
         // From here, we use the Crosscheck Attribute if it is specified instead
@@ -559,68 +561,78 @@ protected function search(string $mode,
         // The column name
         $colclause = $attribute->name;
 
-        // If the attribute is case insensitive, we insert LOWER clauses
-        if(!$attribute->case_sensitive) {
-          $colclause = "LOWER(" . $colclause . ")";
-          $val = strtolower($val);
-        }
-
-        // If the attribute is alphanumeric only, we strip out non-alphanumeric characters
-        if($attribute->alphanumeric) {
-          $colclause = "REGEXP_REPLACE(" . $colclause . ", '[^A-Za-z0-9]', '', 'g')";
-          $val = preg_replace('/[^A-Za-z0-9]/', '', $val);
-        }
-
-  // XXX we only want search_type=E for canonical rules (should we enforce this here, or just at config time?)
-  // XXX complain if there are no Exact rules? Maybe in the UI during configuration?
-  // XXX document in wiki https://spaces.at.internet2.edu/display/COmanage/Match+Attributes
-  //     how we handle all this
-        switch($ruleattr->search_type) {
-          case SearchTypeEnum::Distance:
-            $maxdistance = (int)($attribute->search_distance)+1;
-            $attrclause = "LEVENSHTEIN_LESS_EQUAL("
-                        . $colclause 
-                        . ",?,"
-                        . $attribute->search_distance
-                        . ") < "
-                        . $maxdistance;
-            break;
-          case SearchTypeEnum::Exact:
-            $attrclause = $colclause . "=?";
-            break;
-          case SearchTypeEnum::Mapping:
-            $qclause = (!$attribute->case_sensitive ? "LOWER(query)" : "query");
-            $attrclause = "(" . $colclause . "
-                          IN (SELECT value
-                              FROM attribute_mappings
-                              WHERE attribute_map_id=" . $attribute->attribute_map_id ."
-                              AND " . $qclause . "=?)
-                          OR " . $colclause . "=?)";
-            // We need two copies of $val in the param list
-            $attrSql['vals'][$ruleattr->attribute->id][] = (!$attribute->case_sensitive ? strtolower($val) : $val);
-            break;
-          case SearchTypeEnum::Substring:
-            $attrclause = "SUBSTRING("
-                        . $colclause
-                        . " FROM "
-                        . $attribute->search_substr_from
-                        . " FOR "
-                        . $attribute->search_substr_for
-                        . ") = SUBSTRING(? FROM "
-                        . $attribute->search_substr_from
-                        . " FOR "
-                        . $attribute->search_substr_for
-                        . ")";
-            break;
-          default:
-            throw new LogicException(__('match.er.search_type', [$ruleattr->search_type]));
-            break;
+        if(!$val) {
+          // We skip all the column construction stuff before since we're
+          // comparing a NULL or empty value, so configurations like search
+          // distance or mapping don't apply or make sense.
+
+          $attrclause = $colclause . " IS NULL";
+        } else {
+          // If the attribute is case insensitive, we insert LOWER clauses
+          if(!$attribute->case_sensitive) {
+            $colclause = "LOWER(" . $colclause . ")";
+            $val = strtolower($val);
+          }
+
+          // If the attribute is alphanumeric only, we strip out non-alphanumeric characters
+          if($attribute->alphanumeric) {
+            $colclause = "REGEXP_REPLACE(" . $colclause . ", '[^A-Za-z0-9]', '', 'g')";
+            $val = preg_replace('/[^A-Za-z0-9]/', '', $val);
+          }
+
+    // XXX we only want search_type=E for canonical rules (should we enforce this here, or just at config time?)
+    // XXX complain if there are no Exact rules? Maybe in the UI during configuration?
+    // XXX document in wiki https://spaces.at.internet2.edu/display/COmanage/Match+Attributes
+    //     how we handle all this
+          switch($ruleattr->search_type) {
+            case SearchTypeEnum::Distance:
+              $maxdistance = (int)($attribute->search_distance)+1;
+              $attrclause = "LEVENSHTEIN_LESS_EQUAL("
+                          . $colclause 
+                          . ",?,"
+                          . $attribute->search_distance
+                          . ") < "
+                          . $maxdistance;
+              break;
+            case SearchTypeEnum::Exact:
+              $attrclause = $colclause . "=?";
+              break;
+            case SearchTypeEnum::Mapping:
+              $qclause = (!$attribute->case_sensitive ? "LOWER(query)" : "query");
+              $attrclause = "(" . $colclause . "
+                            IN (SELECT value
+                                FROM attribute_mappings
+                                WHERE attribute_map_id=" . $attribute->attribute_map_id ."
+                                AND " . $qclause . "=?)
+                            OR " . $colclause . "=?)";
+              // We need two copies of $val in the param list
+              $attrSql['vals'][$ruleattr->attribute->id][] = (!$attribute->case_sensitive ? strtolower($val) : $val);
+              break;
+            case SearchTypeEnum::Substring:
+              $attrclause = "SUBSTRING("
+                          . $colclause
+                          . " FROM "
+                          . $attribute->search_substr_from
+                          . " FOR "
+                          . $attribute->search_substr_for
+                          . ") = SUBSTRING(? FROM "
+                          . $attribute->search_substr_from
+                          . " FOR "
+                          . $attribute->search_substr_for
+                          . ")";
+              break;
+            default:
+              throw new LogicException(__('match.er.search_type', [$ruleattr->search_type]));
+              break;
+          }
         }
 
         // Note here we revert to using the original Attribute ID, since if
         // multiple configurations are specified for it we want to OR them together
         $attrSql['sql'][$ruleattr->attribute->id][] = $attrclause;
-        $attrSql['vals'][$ruleattr->attribute->id][] = $val;
+        if($val) {
+          $attrSql['vals'][$ruleattr->attribute->id][] = $val;
+        }
       }
 
       if(empty($attrSql['vals'])) {
@@ -650,7 +662,9 @@ protected function search(string $mode,
           $sql .= " AND " . $attrSql['sql'][$attrId][0];
         }
 
-        $vals = array_merge($vals, $attrSql['vals'][$attrId]);
+        if(!empty($attrSql['vals'][$attrId])) {
+          $vals = array_merge($vals, $attrSql['vals'][$attrId]);
+        }
       }
 
 LOG::write('debug', $sor . "/" . $sorid . " SQL: " . $sql);

diff --git a/app/src/Locale/en_US/default.po b/app/src/Locale/en_US/default.po
@@ -423,6 +423,9 @@ msgstr "Required"
 msgid "match.fd.resolution_mode"
 msgstr "Resolution Mode"
 
+msgid "match.fd.RuleAttributes.match_empty"
+msgstr "Match Empty Values"
+
 msgid "match.fd.search_distance"
 msgstr "Search Distance"
 

diff --git a/app/src/Model/Table/RuleAttributesTable.php b/app/src/Model/Table/RuleAttributesTable.php
@@ -132,7 +132,14 @@ public function validationDefault(Validator $validator) {
       'toggle',
       [ 'rule' => [ 'boolean' ] ]
     );
-    $validator->notEmpty('required');
+    $validator->allowEmpty('required');
+
+    $validator->add(
+      'match_empty',
+      'toggle',
+      [ 'rule' => [ 'boolean' ] ]
+    );
+    $validator->allowEmpty('match_empty');
 
     return $validator; 
   }

diff --git a/app/src/Template/Element/Flash/default.ctp b/app/src/Template/Element/Flash/default.ctp
@@ -16,7 +16,7 @@
 <?php
   if(!empty($message)) {
     // Strip tags then escape quotes before handing Flash message to noty.js
-    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_MAGIC_QUOTES);
+    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_ADD_SLASHES);
     // Replace all newlines with html breaks
     $filteredMessage = str_replace(array("\r", "\n"), '<br/>', $filteredMessage);
     print "<script>generateFlash('" . $filteredMessage . "', '" . $class . "');</script>";

diff --git a/app/src/Template/Element/Flash/error.ctp b/app/src/Template/Element/Flash/error.ctp
@@ -5,7 +5,7 @@
 
   if(!empty($message)) {
     // Strip tags then escape quotes before handing Flash message to noty.js
-    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_MAGIC_QUOTES);
+    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_ADD_SLASHES);
     // Replace all newlines with html breaks
     $filteredMessage = str_replace(array("\r", "\n"), '<br/>', $filteredMessage);
     print "<script>generateFlash('<span class=\"material-icons\">error</span>" . $filteredMessage . "', 'error');</script>";

diff --git a/app/src/Template/Element/Flash/information.ctp b/app/src/Template/Element/Flash/information.ctp
@@ -5,7 +5,7 @@
 
   if(!empty($message)) {
     // Strip tags then escape quotes before handing Flash message to noty.js
-    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_MAGIC_QUOTES);
+    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_ADD_SLASHES);
     // Replace all newlines with html breaks
     $filteredMessage = str_replace(array("\r", "\n"), '<br/>', $filteredMessage);
     print "<script>generateFlash('<span class=\"material-icons\">info</span>" . $filteredMessage . "', 'information');</script>";

diff --git a/app/src/Template/Element/Flash/success.ctp b/app/src/Template/Element/Flash/success.ctp
@@ -5,7 +5,7 @@
 
   if(!empty($message)) {
     // Strip tags then escape quotes before handing Flash message to noty.js
-    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_MAGIC_QUOTES);
+    $filteredMessage = filter_var(filter_var($message,FILTER_SANITIZE_STRING,FILTER_FLAG_NO_ENCODE_QUOTES),FILTER_SANITIZE_ADD_SLASHES);
     // Replace all newlines with html breaks
     $filteredMessage = str_replace(array("\r", "\n"), '<br/>', $filteredMessage);
     print "<script>generateFlash('<span class=\"material-icons\">check_circle</span>" . $filteredMessage . "', 'success');</script>";

diff --git a/app/src/Template/RuleAttributes/fields.inc b/app/src/Template/RuleAttributes/fields.inc
@@ -25,10 +25,39 @@
  * @license       Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)
  */
 
+use \App\Lib\Enum\ConfidenceModeEnum;
+use \App\Lib\Enum\SearchTypeEnum;
+?>
+<script type="text/javascript">
+  // JS specific to these fields
+
+  function fields_update_gadgets() {
+    // Hide or show gadgets according to current state
+
+    var req = document.getElementById('required').checked;
+
+    if(req) {
+      // If the RuleAttribute is Required, we can't Match Empty Values since the
+      // attribute isn't permitted to be empty
+
+      document.getElementById('match-empty').closest('li').hidden = true;
+      // XXX we want .show('fade') but jquery apparently not set up?
+      // or maybe not... instant update actually looks better
+    } else {
+      document.getElementById('match-empty').closest('li').hidden = false;
+    }
+  }
+
+  function js_local_onload() {
+    fields_update_gadgets();
+  }
+</script>
+<?php  
 // This view does not support read-only
 if($action == 'add' || $action == 'edit') {
   print $this->Field->control('attribute_id', ['empty' => true]);
   print $this->Field->control('crosscheck_attribute_id', ['empty' => true], false, __('match.fd.RuleAttributes.crosscheck_attribute_id'));
   print $this->Field->control('search_type', ['empty' => true]);
-  print $this->Field->control('required');
+  print $this->Field->control('required', ['onChange'=>'fields_update_gadgets();'], false);
+  print $this->Field->control('match_empty', [], false);
 }
diff --git a/app/src/Template/Rules/fields.inc b/app/src/Template/Rules/fields.inc
@@ -30,5 +30,5 @@ if($action == 'add' || $action == 'edit') {
   print $this->Field->control('name');
   print $this->Field->control('description', [], false);
   print $this->Field->control('confidence_mode', ['empty' => true]);
-  print $this->Field->control('ordr');
+  print $this->Field->control('ordr', [], false);
 }