Merge branch 'master' into postgres-test

TIER · Dec 4, 2019 · ced5869 · ced5869
2 parents 111d719 + 9a074d6
commit ced5869
Show file tree

Hide file tree

Showing 78 changed files with 939 additions and 675 deletions.
diff --git a/.gitignore b/.gitignore
@@ -390,4 +390,8 @@ pac4j-module/out/
 r
 
 #Local integration test run shell script wrapper
-rinteg
+rinteg
+
+#Local run with durable H2 shell script wrapper
+**/application-h2durable.properties
+rdurable
diff --git a/...2/tier/shibboleth/admin/ui/repository/envers/MetadataResolverEnversVersioningTests.groovy b/...2/tier/shibboleth/admin/ui/repository/envers/MetadataResolverEnversVersioningTests.groovy
@@ -54,7 +54,7 @@ class MetadataResolverEnversVersioningTests extends Specification {
 
         when:
         LocalDynamicMetadataResolver resolver = new LocalDynamicMetadataResolver(name: 'ldmr').with {
-            it.dynamicMetadataResolverAttributes = new DynamicMetadataResolverAttributes()
+            it.dynamicMetadataResolverAttributes = new DynamicMetadataResolverAttributes(refreshDelayFactor: 0.75)
             it
         }
         def resolverHistory = updateAndGetRevisionHistoryOfMetadataResolver(resolver,

diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/ShibbolethUiApplication.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/ShibbolethUiApplication.java
@@ -21,6 +21,8 @@
 import org.springframework.stereotype.Component;
 import org.springframework.transaction.annotation.Transactional;
 
+import javax.script.ScriptException;
+
 @SpringBootApplication
 @ComponentScan(excludeFilters = @ComponentScan.Filter(type = FilterType.REGEX, pattern = "edu.internet2.tier.shibboleth.admin.ui.configuration.auto.*"))
 @EntityScan(basePackages = {"edu.internet2.tier.shibboleth.admin.ui.domain", "edu.internet2.tier.shibboleth.admin.ui.envers", "edu.internet2.tier.shibboleth.admin.ui.security.model"})
@@ -69,7 +71,16 @@ public void initializeResolvers(ApplicationStartedEvent e) {
             metadataResolverRepository.findAll()
                     .forEach(it -> {
                         logger.info(String.format("Reloading filters for resolver [%s: %s]", it.getName(), it.getResourceId()));
-                        metadataResolverService.reloadFilters(it.getResourceId());
+                        try {
+                            metadataResolverService.reloadFilters(it.getResourceId());
+                        }
+                        catch (Throwable ex) {
+                            if(ex instanceof ScriptException) {
+                                logger.warn("Caught invalid script parsing error. Please fix the script data.", ex);
+                                return;
+                            }
+                            throw ex;
+                        }
                     });
         }
     }

diff --git a/...ain/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataFiltersController.java b/...ain/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataFiltersController.java
@@ -15,6 +15,7 @@
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.interceptor.TransactionAspectSupport;
 import org.springframework.web.bind.annotation.DeleteMapping;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -27,6 +28,7 @@
 import org.springframework.web.client.HttpClientErrorException;
 import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
 
+import javax.script.ScriptException;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.List;
@@ -57,7 +59,7 @@ public class MetadataFiltersController {
 
     @ExceptionHandler
     public ResponseEntity<?> notFoundHandler(HttpClientErrorException ex) {
-        if(ex.getStatusCode() == NOT_FOUND) {
+        if (ex.getStatusCode() == NOT_FOUND) {
             return ResponseEntity.notFound().build();
         }
         throw ex;
@@ -78,13 +80,14 @@ public ResponseEntity<?> getOne(@PathVariable String metadataResolverId, @PathVa
     }
 
     @PostMapping("/Filters")
+    @Transactional
     public ResponseEntity<?> create(@PathVariable String metadataResolverId, @RequestBody MetadataFilter createdFilter) {
         MetadataResolver metadataResolver = findResolverOrThrowHttp404(metadataResolverId);
         metadataResolver.addFilter(createdFilter);
         MetadataResolver persistedMr = repository.save(metadataResolver);
 
         // we reload the filters here after save
-        metadataResolverService.reloadFilters(persistedMr.getResourceId());
+        reloadFiltersAndHandleScriptException(persistedMr.getResourceId());
 
         MetadataFilter persistedFilter = newlyPersistedFilter(persistedMr.getMetadataFilters().stream(), createdFilter.getResourceId());
 
@@ -94,6 +97,7 @@ public ResponseEntity<?> create(@PathVariable String metadataResolverId, @Reques
     }
 
     @PutMapping("/Filters/{resourceId}")
+    @Transactional
     public ResponseEntity<?> update(@PathVariable String metadataResolverId,
                                     @PathVariable String resourceId,
                                     @RequestBody MetadataFilter updatedFilter) {
@@ -106,7 +110,7 @@ public ResponseEntity<?> update(@PathVariable String metadataResolverId,
                 .stream()
                 .filter(it -> it.getResourceId().equals(resourceId))
                 .findFirst();
-        if(!filterTobeUpdatedOptional.isPresent()) {
+        if (!filterTobeUpdatedOptional.isPresent()) {
             return ResponseEntity.notFound().build();
         }
         MetadataFilter filterTobeUpdated = filterTobeUpdatedOptional.get();
@@ -129,8 +133,8 @@ public ResponseEntity<?> update(@PathVariable String metadataResolverId,
         metadataResolver.markAsModified();
         repository.save(metadataResolver);
 
-        // TODO: this is wrong
-        metadataResolverService.reloadFilters(metadataResolver.getResourceId());
+        // TODO: do we need to reload filters here?
+        reloadFiltersAndHandleScriptException(metadataResolver.getResourceId());
 
         return ResponseEntity.ok().body(persistedFilter);
     }
@@ -149,7 +153,7 @@ public ResponseEntity<?> delete(@PathVariable String metadataResolverId,
         //change that we need to make in the entire code base
         List<MetadataFilter> updatedFilters = new ArrayList<>(resolver.getMetadataFilters());
         boolean removed = updatedFilters.removeIf(f -> f.getResourceId().equals(resourceId));
-        if(!removed) {
+        if (!removed) {
             throw HTTP_404_CLIENT_ERROR_EXCEPTION.get();
         }
         resolver.setMetadataFilters(updatedFilters);
@@ -164,17 +168,31 @@ public ResponseEntity<?> delete(@PathVariable String metadataResolverId,
         return ResponseEntity.noContent().build();
     }
 
+    private void reloadFiltersAndHandleScriptException(String resolverResourceId) {
+        try {
+            metadataResolverService.reloadFilters(resolverResourceId);
+        } catch (Throwable ex) {
+            //explicitly mark transaction for rollback when we get ScriptException as we call reloadFilters
+            //after persistence call. Then re-throw the exception
+            //to let RestControllerSupport advice return proper 400 error message
+            if (ex instanceof ScriptException) {
+                TransactionAspectSupport.currentTransactionStatus().setRollbackOnly();
+                throw ex;
+            }
+        }
+    }
+
     private MetadataResolver findResolverOrThrowHttp404(String resolverResourceId) {
         MetadataResolver resolver = repository.findByResourceId(resolverResourceId);
-        if(resolver == null) {
+        if (resolver == null) {
             throw HTTP_404_CLIENT_ERROR_EXCEPTION.get();
         }
         return resolver;
     }
 
     private MetadataFilter findFilterOrThrowHttp404(String filterResourceId) {
         MetadataFilter filter = filterRepository.findByResourceId(filterResourceId);
-        if(filter == null) {
+        if (filter == null) {
             throw HTTP_404_CLIENT_ERROR_EXCEPTION.get();
         }
         return filter;
@@ -189,28 +207,25 @@ private MetadataFilter newlyPersistedFilter(Stream<MetadataFilter> filters, fina
     }
 
     /**
-     *
      * Add else if instanceof block here for each concrete filter types we add in the future
      */
     private void updateConcreteFilterTypeData(MetadataFilter filterToBeUpdated, MetadataFilter filterWithUpdatedData) {
         //TODO: Could we maybe use Dozer here before things get out of control? https://dozermapper.github.io
         // Mapper mapper = new net.sf.dozer.Mapper(); // or autowire one
         // mapper.map(fromFilter, toFilter);
-        if(filterWithUpdatedData instanceof EntityAttributesFilter) {
+        if (filterWithUpdatedData instanceof EntityAttributesFilter) {
             EntityAttributesFilter toFilter = EntityAttributesFilter.class.cast(filterToBeUpdated);
             EntityAttributesFilter fromFilter = EntityAttributesFilter.class.cast(filterWithUpdatedData);
             toFilter.setEntityAttributesFilterTarget(fromFilter.getEntityAttributesFilterTarget());
             toFilter.setRelyingPartyOverrides(fromFilter.getRelyingPartyOverrides());
             toFilter.setAttributeRelease(fromFilter.getAttributeRelease());
-        }
-        else if(filterWithUpdatedData instanceof EntityRoleWhiteListFilter) {
+        } else if (filterWithUpdatedData instanceof EntityRoleWhiteListFilter) {
             EntityRoleWhiteListFilter toFilter = EntityRoleWhiteListFilter.class.cast(filterToBeUpdated);
             EntityRoleWhiteListFilter fromFilter = EntityRoleWhiteListFilter.class.cast(filterWithUpdatedData);
             toFilter.setRemoveEmptyEntitiesDescriptors(fromFilter.getRemoveEmptyEntitiesDescriptors());
             toFilter.setRemoveRolelessEntityDescriptors(fromFilter.getRemoveRolelessEntityDescriptors());
             toFilter.setRetainedRoles(fromFilter.getRetainedRoles());
-        }
-        else if (filterWithUpdatedData instanceof SignatureValidationFilter) {
+        } else if (filterWithUpdatedData instanceof SignatureValidationFilter) {
             SignatureValidationFilter toFilter = SignatureValidationFilter.class.cast(filterToBeUpdated);
             SignatureValidationFilter fromFilter = SignatureValidationFilter.class.cast(filterWithUpdatedData);
             toFilter.setRequireSignedRoot(fromFilter.getRequireSignedRoot());
@@ -220,13 +235,11 @@ else if (filterWithUpdatedData instanceof SignatureValidationFilter) {
             toFilter.setDynamicTrustedNamesStrategyRef(fromFilter.getDynamicTrustedNamesStrategyRef());
             toFilter.setTrustEngineRef(fromFilter.getTrustEngineRef());
             toFilter.setPublicKey(fromFilter.getPublicKey());
-        }
-        else if(filterWithUpdatedData instanceof RequiredValidUntilFilter) {
+        } else if (filterWithUpdatedData instanceof RequiredValidUntilFilter) {
             RequiredValidUntilFilter toFilter = RequiredValidUntilFilter.class.cast(filterToBeUpdated);
             RequiredValidUntilFilter fromFilter = RequiredValidUntilFilter.class.cast(filterWithUpdatedData);
             toFilter.setMaxValidityInterval(fromFilter.getMaxValidityInterval());
-        }
-        else if (filterWithUpdatedData instanceof NameIdFormatFilter) {
+        } else if (filterWithUpdatedData instanceof NameIdFormatFilter) {
             NameIdFormatFilter toFilter = NameIdFormatFilter.class.cast(filterToBeUpdated);
             NameIdFormatFilter fromFilter = NameIdFormatFilter.class.cast(filterWithUpdatedData);
             toFilter.setRemoveExistingFormats(fromFilter.getRemoveExistingFormats());

diff --git a/...a/edu/internet2/tier/shibboleth/admin/ui/domain/filters/EntityAttributesFilterTarget.java b/...a/edu/internet2/tier/shibboleth/admin/ui/domain/filters/EntityAttributesFilterTarget.java
@@ -8,6 +8,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.persistence.Column;
 import javax.persistence.ElementCollection;
 import javax.persistence.Entity;
 import javax.persistence.FetchType;
@@ -31,6 +32,7 @@ public enum EntityAttributesFilterTargetType {
 
     @ElementCollection
     @OrderColumn
+    @Column(length = 4000)
     private List<String> value;
 
     public EntityAttributesFilterTargetType getEntityAttributesFilterTargetType() {

diff --git a/...nternet2/tier/shibboleth/admin/ui/domain/resolvers/DynamicMetadataResolverAttributes.java b/...nternet2/tier/shibboleth/admin/ui/domain/resolvers/DynamicMetadataResolverAttributes.java
@@ -20,17 +20,17 @@ public class DynamicMetadataResolverAttributes {
 
     private String taskTimerRef;
 
-    private Float refreshDelayFactor = 0.75F;
+    private Float refreshDelayFactor;
 
-    private String minCacheDuration = "PT10M";
+    private String minCacheDuration;
 
-    private String maxCacheDuration = "PT8H";
+    private String maxCacheDuration;
 
-    private String maxIdleEntityData = "PT8H";
+    private String maxIdleEntityData;
 
     private Boolean removeIdleEntityData;
 
-    private String cleanupTaskInterval = "PT30M";
+    private String cleanupTaskInterval;
 
     private String persistentCacheManagerRef;
 
@@ -40,7 +40,7 @@ public class DynamicMetadataResolverAttributes {
 
     private Boolean initializeFromPersistentCacheInBackground = true;
 
-    private String backgroundInitializationFromCacheDelay = "PT2S";
+    private String backgroundInitializationFromCacheDelay;
 
     private String initializationFromCachePredicateRef;
 

diff --git a/...shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFileBackedHTTPMetadataResolver.java b/...shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFileBackedHTTPMetadataResolver.java
@@ -15,6 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
 import static edu.internet2.tier.shibboleth.admin.util.DurationUtility.toMillis;
@@ -83,6 +84,15 @@ protected void initMetadataResolver() throws ComponentInitializationException {
                                                        indexWriter);
     }
 
+    @Nonnull
+    @Override
+    protected BatchEntityBackingStore getBackingStore() {
+        if (super.getBackingStore() == null) {
+            super.setBackingStore(super.createNewBackingStore());
+        }
+        return super.getBackingStore();
+    }
+
     /**
      * {@inheritDoc}
      */

diff --git a/...ier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFilesystemMetadataResolver.java b/...ier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFilesystemMetadataResolver.java
@@ -11,6 +11,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.File;
 
@@ -51,15 +52,31 @@ public DateTime getLastRefresh() {
 
     @Override
     protected void initMetadataResolver() throws ComponentInitializationException {
+        //Necessary to make sure backing store is initialized by the super class to avoid NPE during re-filtering
+        try {
+            setBackingStore(createNewBackingStore());
+        }
+        catch(Throwable e) {
+            logger.warn("Error caught and ignored during initialization necessary to init backingStore", e);
+        }
+
         if (this.sourceResolver.getDoInitialization()) {
             super.initMetadataResolver();
-
             delegate.addIndexedDescriptorsFromBackingStore(this.getBackingStore(),
                     this.sourceResolver.getResourceId(),
                     indexWriter);
         }
     }
 
+    @Nonnull
+    @Override
+    protected BatchEntityBackingStore getBackingStore() {
+        if (super.getBackingStore() == null) {
+            super.setBackingStore(super.createNewBackingStore());
+        }
+        return super.getBackingStore();
+    }
+
     /**
      * {@inheritDoc}
      */

diff --git a/...shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlResourceBackedMetadataResolver.java b/...shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlResourceBackedMetadataResolver.java
@@ -11,6 +11,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import java.io.IOException;
 
@@ -59,6 +60,15 @@ protected void initMetadataResolver() throws ComponentInitializationException {
                                                        indexWriter);
     }
 
+    @Nonnull
+    @Override
+    protected BatchEntityBackingStore getBackingStore() {
+        if (super.getBackingStore() == null) {
+            super.setBackingStore(super.createNewBackingStore());
+        }
+        return super.getBackingStore();
+    }
+
     /**
      * {@inheritDoc}
      */