Merge branch 'SHIBUI-660' into SHIBUI-723

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java

@@ -9,12 +9,14 @@
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.queryparser.classic.QueryParser;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -32,7 +34,9 @@
 import javax.servlet.http.HttpServletRequest;
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 @Configuration
 public class CoreShibUiConfiguration {
@@ -80,7 +84,7 @@ public AttributeUtility attributeUtility() {
     Analyzer fullTokenAnalyzer;
 
     @Autowired
-    Directory directory;
+    DirectoryService directoryService;
 
     @Autowired
     LocaleResolver localeResolver;
@@ -95,8 +99,9 @@ public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(E
 
     @Bean
     public EntityIdsSearchService entityIdsSearchService() {
-        return (term, limit) -> {
+        return (resourceId, term, limit) -> {
             List<String> entityIds = new ArrayList<>();
+            Directory directory = directoryService.getDirectory(resourceId);
             try {
                 IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
                 QueryParser parser = new QueryParser("content", fullTokenAnalyzer);

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/MetadataResolverConfiguration.java

@@ -2,6 +2,7 @@
 
 import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
+import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
 import net.shibboleth.utilities.java.support.resolver.ResolverException;
 import org.apache.http.HttpResponse;
@@ -38,7 +39,7 @@ public class MetadataResolverConfiguration {
     OpenSamlObjects openSamlObjects;
 
     @Autowired
-    IndexWriter indexWriter;
+    IndexWriterService indexWriterService;
 
     @Autowired
     MetadataResolverRepository metadataResolverRepository;
@@ -50,13 +51,22 @@ public MetadataResolver metadataResolver() throws ResolverException, ComponentIn
 
         List<MetadataResolver> resolvers = new ArrayList<>();
 
+        String incommonMRId = "incommonmd";
         // TODO: remove this later when we allow for creation of arbitrary metadata resolvers
         FileBackedHTTPMetadataResolver incommonMR = new FileBackedHTTPMetadataResolver(HttpClients.createMinimal(), "http://md.incommon.org/InCommon/InCommon-metadata.xml", "/tmp/incommonmd.xml"){
             @Override
             protected void initMetadataResolver() throws ComponentInitializationException {
                 super.initMetadataResolver();
 
+                IndexWriter indexWriter;
+                try {
+                    indexWriter = indexWriterService.getIndexWriter(incommonMRId);
+                } catch (IOException e) {
+                    throw new ComponentInitializationException(e);
+                }
+
                 for (String entityId: this.getBackingStore().getIndexedDescriptors().keySet()) {
+
                     Document document = new Document();
                     document.add(new StringField("id", entityId, Field.Store.YES));
                     document.add(new TextField("content", entityId, Field.Store.YES)); // TODO: change entityId to be content of entity descriptor block
@@ -86,7 +96,7 @@ protected void processConditionalRetrievalHeaders(HttpResponse response) {
                 // let's do nothing 'cause we want to allow a refresh
             }
         };
-        incommonMR.setId("incommonmd");
+        incommonMR.setId(incommonMRId);
         incommonMR.setParserPool(openSamlObjects.getParserPool());
         incommonMR.setMetadataFilter(new MetadataFilterChain());
         incommonMR.initialize();

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/SearchConfiguration.java

@@ -1,5 +1,7 @@
 package edu.internet2.tier.shibboleth.admin.ui.configuration;
 
+import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
+import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.LowerCaseFilter;
 import org.apache.lucene.analysis.StopFilter;
@@ -11,17 +13,18 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 @Configuration
 public class SearchConfiguration {
-    @Bean
-    Directory directory() {
-        return new RAMDirectory();
-    }
+    @Autowired
+    DirectoryService directoryService;
 
     @Bean
     Analyzer analyzer() {
@@ -54,10 +57,37 @@ protected TokenStreamComponents createComponents(String fieldName) {
         };
     }
 
-    @Bean
-    IndexWriter indexWriter() throws IOException {
+    private IndexWriter createIndexWriter(Directory directory) throws IOException {
         IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer());
         indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
-        return new IndexWriter(directory(), indexWriterConfig);
+        return new IndexWriter(directory, indexWriterConfig);
+    }
+
+    @Bean
+    public IndexWriterService indexWriterService() {
+        Map<String, IndexWriter> indexWriterMap = new HashMap<>();
+
+        return resourceId -> {
+            IndexWriter indexWriter = indexWriterMap.get(resourceId);
+            if (indexWriter == null) {
+                indexWriter = createIndexWriter(directoryService.getDirectory(resourceId));
+                indexWriterMap.put(resourceId, indexWriter);
+            }
+            return indexWriter;
+        };
+    }
+
+    @Bean
+    public DirectoryService directoryService() {
+        Map<String, Directory> directoryMap = new HashMap<>();
+
+        return resourceId -> {
+            Directory directory = directoryMap.get(resourceId);
+            if (directory == null) {
+                directory = new RAMDirectory();
+                directoryMap.put(resourceId, directory);
+            }
+            return directory;
+        };
     }
 }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/EntityIdsSearchController.java

@@ -18,10 +18,13 @@ public EntityIdsSearchController(EntityIdsSearchService entityIdsSearchService)
         this.entityIdsSearchService = entityIdsSearchService;
     }
 
+    // TODO Change this to include a metadata resolver id so we can pull out the right index to search
     @GetMapping
-    ResponseEntity<?> search(@RequestParam String term, @RequestParam(required = false) Integer limit) {
+    ResponseEntity<?> search(@RequestParam String resourceId,
+                             @RequestParam String term,
+                             @RequestParam(required = false) Integer limit) {
         //Zero indicates no-limit
         final int resultLimit = (limit != null ? limit : 10);
-        return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(term, resultLimit));
+        return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(resourceId, term, resultLimit));
     }
 }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversController.java

@@ -4,8 +4,14 @@
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolverValidationService;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
+import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
 import lombok.extern.slf4j.Slf4j;
+import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.IndexWriter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
@@ -47,6 +53,9 @@ public class MetadataResolversController {
     @Autowired
     MetadataResolverService metadataResolverService;
 
+    @Autowired
+    IndexWriterService indexWriterService;
+
     @ExceptionHandler({InvalidTypeIdException.class, IOException.class, HttpMessageNotReadableException.class})
     public ResponseEntity<?> unableToParseJson(Exception ex) {
         return ResponseEntity.badRequest().body(new ErrorResponse(HttpStatus.BAD_REQUEST.toString(), ex.getMessage()));
@@ -150,4 +159,45 @@ private static URI getResourceUriFor(MetadataResolver resolver) {
                 .build()
                 .toUri();
     }
+
+    private void updateLucene(MetadataResolver resolver) throws ComponentInitializationException {
+        IndexWriter indexWriter = null;
+        try {
+            indexWriter = indexWriterService.getIndexWriter(resolver.getResourceId());
+        } catch (IOException e) {
+            throw new ComponentInitializationException(e);
+        }
+
+        // add documents to indexWriter .. for each what?
+        /*
+        for () {
+            Document document = new Document();
+            document.add(new StringField("id", entityId, Field.Store.YES));
+            document.add(new TextField("content", entityId, Field.Store.YES)); // TODO: change entityId to be content of entity descriptor block
+            try {
+                indexWriter.addDocument(document);
+            } catch (IOException e) {
+                logger.error(e.getMessage(), e);
+            }
+        }
+        */
+
+        // if document exists
+        // indexWriter.updateDocument(term, document) <-- what's the term?
+        // else, create...
+        Document document = new Document();
+        document.add(new StringField("id", resolver.getResourceId(), Field.Store.YES));
+
+        try {
+            indexWriter.addDocument(document);
+        } catch (IOException e) {
+            throw new ComponentInitializationException(e);
+        }
+
+        try {
+            indexWriter.commit();
+        } catch (IOException e) {
+            throw new ComponentInitializationException(e);
+        }
+    }
 }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/DirectoryService.java

@@ -0,0 +1,16 @@
+package edu.internet2.tier.shibboleth.admin.ui.service;
+
+import org.apache.lucene.store.Directory;
+
+/**
+ * API component responsible for entity ids search.
+ */
+@FunctionalInterface
+public interface DirectoryService {
+    /**
+     * Return a Directory for a given resource id. If one is not found, it will be created.
+     * @param resourceId the resource to get the Directory for
+     * @return Directory
+     */
+    Directory getDirectory(String resourceId);
+}

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchService.java

@@ -15,9 +15,10 @@ public interface EntityIdsSearchService {
 
     /**
      * Find a list of entity ids
+     * @param resourceId the id of the resource to search within
      * @param searchTerm for the query
      * @param limit optional limit of query results to return. Zero or less value indicates no limit.
      * @return EntityIdsSearchResultRepresentation
      */
-    EntityIdsSearchResultRepresentation findBySearchTermAndOptionalLimit(String searchTerm, int limit);
+    EntityIdsSearchResultRepresentation findBySearchTermAndOptionalLimit(String resourceId, String searchTerm, int limit);
 }

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/IndexWriterService.java

@@ -0,0 +1,18 @@
+package edu.internet2.tier.shibboleth.admin.ui.service;
+
+import org.apache.lucene.index.IndexWriter;
+
+import java.io.IOException;
+
+/**
+ * API component responsible for entity ids search.
+ */
+@FunctionalInterface
+public interface IndexWriterService {
+    /**
+     * Return a (possibly cached) index writer for a given resource id.
+     * @param resourceId the resource to create the IndexWriter for
+     * @return IndexWriter
+     */
+    IndexWriter getIndexWriter(String resourceId) throws IOException;
+}

backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/configuration/TestConfiguration.groovy

@@ -2,6 +2,7 @@ package edu.internet2.tier.shibboleth.admin.ui.configuration
 
 import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository
+import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService
 import net.shibboleth.ext.spring.resource.ResourceHelper
 import net.shibboleth.utilities.java.support.component.ComponentInitializationException
 import org.apache.lucene.document.Document
@@ -14,27 +15,32 @@ import org.opensaml.saml.metadata.resolver.MetadataResolver
 import org.opensaml.saml.metadata.resolver.impl.ResourceBackedMetadataResolver
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Bean
 import org.springframework.context.annotation.Configuration
 import org.springframework.core.io.ClassPathResource
 
 @Configuration
 class TestConfiguration {
+    @Autowired
+    IndexWriterService indexWriterService
+
     final OpenSamlObjects openSamlObjects
-    final IndexWriter indexWriter
     final MetadataResolverRepository metadataResolverRepository
     final Logger logger = LoggerFactory.getLogger(TestConfiguration.class);
 
-    TestConfiguration(final OpenSamlObjects openSamlObjects, final IndexWriter indexWriter, final MetadataResolverRepository metadataResolverRepository) {
+    TestConfiguration(final OpenSamlObjects openSamlObjects, final MetadataResolverRepository metadataResolverRepository) {
         this.openSamlObjects =openSamlObjects
-        this.indexWriter = indexWriter
         this.metadataResolverRepository = metadataResolverRepository
     }
 
     @Bean
     MetadataResolver metadataResolver() {
         ChainingMetadataResolver metadataResolver = new ChainingMetadataResolver()
         metadataResolver.setId("chain")
+        String resolverId = "test"
+
+        IndexWriter indexWriter = indexWriterService.getIndexWriter(resolverId)
 
         def shortIncommon = new ResourceBackedMetadataResolver(ResourceHelper.of(new ClassPathResource('/metadata/incommon-short.xml'))){
             @Override
@@ -58,7 +64,7 @@ class TestConfiguration {
                 }
             }
         }.with {
-            it.id = 'test'
+            it.id = resolverId
             TestConfiguration p = owner
             it.parserPool = p.openSamlObjects.parserPool
             it.initialize()