Merged in NOJIRA-lucene-search-fix (pull request #185)

NOJIRA lucene search fix

Approved-by: Shibui Jenkins <shibui.jenkins@gmail.com>

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/SearchConfiguration.java

@@ -70,6 +70,7 @@ public IndexWriterService indexWriterService() {
             IndexWriter indexWriter = indexWriterMap.get(resourceId);
             if (indexWriter == null) {
                 indexWriter = createIndexWriter(directoryService.getDirectory(resourceId));
+                indexWriter.commit();
                 indexWriterMap.put(resourceId, indexWriter);
             }
             return indexWriter;

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataFiltersController.java

@@ -6,10 +6,16 @@
 import edu.internet2.tier.shibboleth.admin.ui.domain.filters.RequiredValidUntilFilter;
 import edu.internet2.tier.shibboleth.admin.ui.domain.filters.SignatureValidationFilter;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
+import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.opensaml.OpenSamlFunctionDrivenDynamicHTTPMetadataResolver;
+import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.opensaml.OpenSamlLocalDynamicMetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.repository.FilterRepository;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
 
+import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
+import net.shibboleth.utilities.java.support.resolver.ResolverException;
+import org.opensaml.saml.metadata.resolver.ChainingMetadataResolver;
+import org.opensaml.saml.metadata.resolver.RefreshableMetadataResolver;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -48,6 +54,9 @@ public class MetadataFiltersController {
     @Autowired
     private FilterRepository filterRepository;
 
+    @Autowired
+    org.opensaml.saml.metadata.resolver.MetadataResolver chainingMetadataResolver;
+
     private static final Supplier<HttpClientErrorException> HTTP_404_CLIENT_ERROR_EXCEPTION = () -> new HttpClientErrorException(NOT_FOUND);
 
     @ExceptionHandler
@@ -80,6 +89,7 @@ public ResponseEntity<?> create(@PathVariable String metadataResolverId, @Reques
 
         // we reload the filters here after save
         metadataResolverService.reloadFilters(persistedMr.getName());
+        refreshOrInitResolver(metadataResolver);
 
         MetadataFilter persistedFilter = newlyPersistedFilter(persistedMr.getMetadataFilters().stream(), createdFilter.getResourceId());
 
@@ -88,6 +98,33 @@ public ResponseEntity<?> create(@PathVariable String metadataResolverId, @Reques
                 .body(persistedFilter);
     }
 
+    private void refreshOrInitResolver(MetadataResolver resolver) {
+        List<org.opensaml.saml.metadata.resolver.MetadataResolver> resolvers = ((ChainingMetadataResolver) chainingMetadataResolver).getResolvers();
+        resolvers.stream().filter(it -> it.getId().equals(resolver.getResourceId())).forEach(it -> {
+            if (it instanceof RefreshableMetadataResolver) {
+                try {
+                    ((RefreshableMetadataResolver) it).refresh();
+                } catch (ResolverException e) {
+                    //TODO what should we do if we can't refresh?
+                }
+            } else if (it instanceof OpenSamlFunctionDrivenDynamicHTTPMetadataResolver) {
+                try {
+                    ((OpenSamlFunctionDrivenDynamicHTTPMetadataResolver) it).refresh();
+                } catch (ComponentInitializationException e) {
+                    //TODO what should we do if we can't refresh?
+                }
+            } else if (it instanceof OpenSamlLocalDynamicMetadataResolver) {
+                try {
+                    ((OpenSamlLocalDynamicMetadataResolver) it).refresh();
+                } catch (ComponentInitializationException e) {
+                    //TODO what should we do if we can't refresh?
+                }
+            } else {
+                //TODO we shouldn't get here, but if we do... throw exception?
+            }
+        });
+    }
+
     @PutMapping("/Filters/{resourceId}")
     public ResponseEntity<?> update(@PathVariable String metadataResolverId,
                                     @PathVariable String resourceId,
@@ -122,6 +159,7 @@ public ResponseEntity<?> update(@PathVariable String metadataResolverId,
 
         // TODO: this is wrong
         metadataResolverService.reloadFilters(metadataResolver.getName());
+        refreshOrInitResolver(metadataResolver);
 
         return ResponseEntity.ok().body(persistedFilter);
     }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversController.java

@@ -11,10 +11,6 @@
 import lombok.extern.slf4j.Slf4j;
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
 import net.shibboleth.utilities.java.support.resolver.ResolverException;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
-import org.apache.lucene.index.IndexWriter;
 import org.opensaml.saml.metadata.resolver.ChainingMetadataResolver;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
@@ -109,7 +105,7 @@ public ResponseEntity<?> getOne(@PathVariable String resourceId) {
 
     @PostMapping("/MetadataResolvers")
     @Transactional
-    public ResponseEntity<?> create(@RequestBody MetadataResolver newResolver) throws IOException, ResolverException {
+    public ResponseEntity<?> create(@RequestBody MetadataResolver newResolver) throws IOException, ResolverException, ComponentInitializationException {
         if (resolverRepository.findByName(newResolver.getName()) != null) {
             return ResponseEntity.status(HttpStatus.CONFLICT).build();
         }
@@ -127,21 +123,17 @@ public ResponseEntity<?> create(@RequestBody MetadataResolver newResolver) throw
         return ResponseEntity.created(getResourceUriFor(persistedResolver)).body(persistedResolver);
     }
 
-    private void updateChainingMetadataResolver(MetadataResolver persistedResolver) throws IOException, ResolverException {
+    private void updateChainingMetadataResolver(MetadataResolver persistedResolver) throws IOException, ResolverException, ComponentInitializationException {
         org.opensaml.saml.metadata.resolver.MetadataResolver openSamlResolver = metadataResolverConverterService.convertToOpenSamlRepresentation(persistedResolver);
         List<org.opensaml.saml.metadata.resolver.MetadataResolver> resolverList = new ArrayList<>(((ChainingMetadataResolver) chainingMetadataResolver).getResolvers());
-        for (org.opensaml.saml.metadata.resolver.MetadataResolver resolver : resolverList) {
-            if (resolver.getId().equals(persistedResolver.getResourceId())) {
-                resolverList.remove(resolver);
-            }
-        }
+        resolverList.removeIf(resolver -> resolver.getId().equals(persistedResolver.getResourceId()));
         resolverList.add(openSamlResolver);
         ((ChainingMetadataResolver) chainingMetadataResolver).setResolvers(resolverList);
     }
 
     @PutMapping("/MetadataResolvers/{resourceId}")
     @Transactional
-    public ResponseEntity<?> update(@PathVariable String resourceId, @RequestBody MetadataResolver updatedResolver) throws IOException, ResolverException {
+    public ResponseEntity<?> update(@PathVariable String resourceId, @RequestBody MetadataResolver updatedResolver) throws IOException, ResolverException, ComponentInitializationException {
         MetadataResolver existingResolver = resolverRepository.findByResourceId(resourceId);
         if (existingResolver == null) {
             return ResponseEntity.notFound().build();

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/AbstractXMLObject.java

@@ -3,9 +3,12 @@
 import lombok.EqualsAndHashCode;
 import net.shibboleth.utilities.java.support.collection.LockableClassToInstanceMultiMap;
 import net.shibboleth.utilities.java.support.xml.QNameSupport;
+import org.opensaml.core.config.ConfigurationService;
 import org.opensaml.core.xml.Namespace;
 import org.opensaml.core.xml.NamespaceManager;
 import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.config.XMLObjectProviderRegistry;
+import org.opensaml.core.xml.io.MarshallingException;
 import org.opensaml.core.xml.schema.XSBooleanValue;
 import org.opensaml.core.xml.util.IDIndex;
 import org.w3c.dom.Element;
@@ -43,9 +46,17 @@ public void detach() {
 
     }
 
+    @Transient
+    private transient Element dom;
+
     @Nullable
     public Element getDOM() {
-        return null; //convert this class using opensaml stuff
+        return this.dom;
+    }
+
+    @Override
+    public void setDOM(@Nullable Element dom) {
+        this.dom = dom;
     }
 
     public String getNamespaceURI() {
@@ -105,9 +116,19 @@ public List<XMLObject> getOrderedChildren() {
         return null;
     }
 
+    @Transient
+    private transient XMLObject parent;
     @Nullable
     public XMLObject getParent() {
-        return null;
+        return parent;
+    }
+
+    public void setParent(@Nullable XMLObject xmlObject) {
+        parent = xmlObject;
+    }
+
+    public boolean hasParent() {
+        return getParent() != null;
     }
 
     @Nullable
@@ -132,23 +153,34 @@ public void setSchemaType(QName schemaType) {
     }
 
     public boolean hasChildren() {
-        return false;
-    }
-
-    public boolean hasParent() {
-        return false;
+        List children = getOrderedChildren();
+        return children != null && children.size() > 0;
     }
 
     public void releaseChildrenDOM(boolean b) {
-
+        if (getOrderedChildren() != null) {
+            for (XMLObject child : getOrderedChildren()) {
+                if (child != null) {
+                    child.releaseDOM();
+                    if (b) {
+                        child.releaseChildrenDOM(b);
+                    }
+                }
+            }
+        }
     }
 
     public void releaseDOM() {
-
+        this.setDOM(null);
     }
 
     public void releaseParentDOM(boolean b) {
-
+        if (hasParent()) {
+            getParent().releaseDOM();
+            if (b) {
+                getParent().releaseParentDOM(b);
+            }
+        }
     }
 
     @Nullable
@@ -161,18 +193,10 @@ public XMLObject resolveIDFromRoot(@Nonnull String s) {
         return null;
     }
 
-    public void setDOM(@Nullable Element element) {
-
-    }
-
     public void setNoNamespaceSchemaLocation(@Nullable String s) {
 
     }
 
-    public void setParent(@Nullable XMLObject xmlObject) {
-
-    }
-
     public void setSchemaLocation(@Nullable String s) {
 
     }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/EntityDescriptor.java

@@ -4,8 +4,12 @@
 import com.google.common.collect.Lists;
 
 import lombok.EqualsAndHashCode;
+import org.opensaml.core.config.ConfigurationService;
 import org.opensaml.core.xml.XMLObject;
+import org.opensaml.core.xml.config.XMLObjectProviderRegistry;
+import org.opensaml.core.xml.io.MarshallingException;
 import org.springframework.util.StringUtils;
+import org.w3c.dom.Element;
 
 import javax.annotation.Nullable;
 

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlChainingMetadataResolver.java

@@ -7,6 +7,7 @@
 import net.shibboleth.utilities.java.support.resolver.ResolverException;
 import org.opensaml.saml.metadata.resolver.ChainingMetadataResolver;
 import org.opensaml.saml.metadata.resolver.MetadataResolver;
+import org.opensaml.saml.metadata.resolver.RefreshableMetadataResolver;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -32,8 +33,7 @@ public OpenSamlChainingMetadataResolver(@Nonnull List<MetadataResolver> mutableR
     }
 
     @Override
-    public void setResolvers(@Nonnull @NonnullElements final List<? extends MetadataResolver> newResolvers)
-            throws ResolverException {
+    public void setResolvers(@Nonnull @NonnullElements final List<? extends MetadataResolver> newResolvers) {
         if (newResolvers == null || newResolvers.isEmpty()) {
             mutableResolvers = Collections.emptyList();
             return;
@@ -57,4 +57,13 @@ protected void doInitialize() throws ComponentInitializationException {
             mutableResolvers = Collections.emptyList();
         }
     }
+
+    @Override
+    public void refresh() throws ResolverException {
+        for (final MetadataResolver resolver : this.mutableResolvers) {
+            if (resolver instanceof RefreshableMetadataResolver) {
+                ((RefreshableMetadataResolver) resolver).refresh();
+            }
+        }
+    }
 }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFileBackedHTTPMetadataResolver.java

@@ -3,6 +3,7 @@
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.FileBackedHttpMetadataResolver;
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
 import net.shibboleth.utilities.java.support.resolver.ResolverException;
+import net.shibboleth.utilities.java.support.xml.ParserPool;
 import org.apache.http.HttpResponse;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.index.IndexWriter;
@@ -22,7 +23,8 @@ public class OpenSamlFileBackedHTTPMetadataResolver extends FileBackedHTTPMetada
 
     private OpenSamlMetadataResolverDelegate delegate;
 
-    public OpenSamlFileBackedHTTPMetadataResolver(IndexWriter indexWriter,
+    public OpenSamlFileBackedHTTPMetadataResolver(ParserPool parserPool,
+                                                  IndexWriter indexWriter,
                                                   FileBackedHttpMetadataResolver sourceResolver) throws ResolverException {
         super(HttpClients.createMinimal(), sourceResolver.getMetadataURL(), sourceResolver.getBackingFile());
         this.indexWriter = indexWriter;
@@ -34,7 +36,7 @@ public OpenSamlFileBackedHTTPMetadataResolver(IndexWriter indexWriter,
         OpenSamlMetadataResolverConstructorHelper.updateOpenSamlMetadataResolverFromHttpMetadataResolverAttributes(
                 this, sourceResolver.getHttpMetadataResolverAttributes());
         OpenSamlMetadataResolverConstructorHelper.updateOpenSamlMetadataResolverFromReloadableMetadataResolverAttributes(
-                this, sourceResolver.getReloadableMetadataResolverAttributes());
+                this, sourceResolver.getReloadableMetadataResolverAttributes(), parserPool);
 
         this.setBackupFile(sourceResolver.getBackingFile());
         this.setBackupFileInitNextRefreshDelay(toMillis(sourceResolver.getBackupFileInitNextRefreshDelay()));

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFilesystemMetadataResolver.java

@@ -2,6 +2,7 @@
 
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
 import net.shibboleth.utilities.java.support.resolver.ResolverException;
+import net.shibboleth.utilities.java.support.xml.ParserPool;
 import org.apache.lucene.index.IndexWriter;
 import org.joda.time.DateTime;
 import org.opensaml.saml.metadata.resolver.impl.FilesystemMetadataResolver;
@@ -17,9 +18,10 @@ public class OpenSamlFilesystemMetadataResolver extends FilesystemMetadataResolv
     private edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.FilesystemMetadataResolver sourceResolver;
     private OpenSamlMetadataResolverDelegate delegate;
 
-    public OpenSamlFilesystemMetadataResolver(File metadataFile,
+    public OpenSamlFilesystemMetadataResolver(ParserPool parserPool,
                                               IndexWriter indexWriter,
-                                              edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.FilesystemMetadataResolver sourceResolver) throws ResolverException {
+                                              edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.FilesystemMetadataResolver sourceResolver,
+                                              File metadataFile) throws ResolverException {
         super(metadataFile);
         this.indexWriter = indexWriter;
         this.sourceResolver = sourceResolver;
@@ -28,7 +30,7 @@ public OpenSamlFilesystemMetadataResolver(File metadataFile,
         this.setId(sourceResolver.getResourceId());
 
         OpenSamlMetadataResolverConstructorHelper.updateOpenSamlMetadataResolverFromReloadableMetadataResolverAttributes(
-                this, sourceResolver.getReloadableMetadataResolverAttributes());
+                this, sourceResolver.getReloadableMetadataResolverAttributes(), parserPool);
     }
 
     // TODO: this is still probably not the best way to do this?

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/resolvers/opensaml/OpenSamlFunctionDrivenDynamicHTTPMetadataResolver.java

@@ -2,6 +2,7 @@
 
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.DynamicHttpMetadataResolver;
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
+import net.shibboleth.utilities.java.support.xml.ParserPool;
 import org.apache.http.impl.client.HttpClients;
 import org.apache.lucene.index.IndexWriter;
 import org.opensaml.saml.metadata.resolver.impl.FunctionDrivenDynamicHTTPMetadataResolver;
@@ -14,7 +15,8 @@ public class OpenSamlFunctionDrivenDynamicHTTPMetadataResolver extends FunctionD
     private DynamicHttpMetadataResolver sourceResolver;
     private OpenSamlMetadataResolverDelegate delegate;
 
-    public OpenSamlFunctionDrivenDynamicHTTPMetadataResolver(IndexWriter indexWriter,
+    public OpenSamlFunctionDrivenDynamicHTTPMetadataResolver(ParserPool parserPool,
+                                                             IndexWriter indexWriter,
                                                              DynamicHttpMetadataResolver sourceResolver) {
         super(HttpClients.createMinimal());
         this.indexWriter = indexWriter;
@@ -24,7 +26,7 @@ public OpenSamlFunctionDrivenDynamicHTTPMetadataResolver(IndexWriter indexWriter
         this.setId(sourceResolver.getResourceId());
 
         OpenSamlMetadataResolverConstructorHelper.updateOpenSamlMetadataResolverFromDynamicMetadataResolverAttributes(
-                this, sourceResolver.getDynamicMetadataResolverAttributes());
+                this, sourceResolver.getDynamicMetadataResolverAttributes(), parserPool);
 
         OpenSamlMetadataResolverConstructorHelper.updateOpenSamlMetadataResolverFromHttpMetadataResolverAttributes(
                 this, sourceResolver.getHttpMetadataResolverAttributes());
@@ -45,4 +47,10 @@ protected void initMetadataResolver() throws ComponentInitializationException {
                                                        this.sourceResolver.getResourceId(),
                                                        indexWriter);
     }
+
+    public void refresh() throws ComponentInitializationException {
+        delegate.addIndexedDescriptorsFromBackingStore(this.getBackingStore(),
+                                                       this.sourceResolver.getResourceId(),
+                                                       indexWriter);
+    }
 }