From 34b084e1e96cfcbedea58c8a13e04ce6f7aef478 Mon Sep 17 00:00:00 2001 From: Bill Smith Date: Thu, 23 Aug 2018 09:59:04 -0700 Subject: [PATCH 1/4] [NOJIRA] Replaced /tmp with /groovytest --- ...adataResolversControllerIntegrationTests.groovy | 14 ++++++++++++++ .../admin/ui/util/TestObjectGenerator.groovy | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy index f90b85146..f88df2d1b 100644 --- a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy @@ -160,6 +160,13 @@ class MetadataResolversControllerIntegrationTests extends Specification { result.statusCodeValue == 201 result.headers.Location[0].contains(BASE_URI) + cleanup: + def userHome = System.getProperty("user.home") + def tmpDirectory = new File(userHome + File.separator + 'groovytest') + if (tmpDirectory.exists()) { + tmpDirectory.deleteDir() + } + where: resolverType | _ 'DynamicHttp' | _ @@ -196,6 +203,13 @@ class MetadataResolversControllerIntegrationTests extends Specification { then: updatedResolverMap.name == 'Updated DynamicHttpMetadataResolver' + cleanup: + def userHome = System.getProperty("user.home") + def tmpDirectory = new File(userHome + File.separator + 'groovytest') + if (tmpDirectory.exists()) { + tmpDirectory.deleteDir() + } + where: resolverType | _ 'DynamicHttp' | _ diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy index eea2bbbec..187ec50f8 100644 --- a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy @@ -483,10 +483,15 @@ class TestObjectGenerator { } LocalDynamicMetadataResolver localDynamicMetadataResolver() { + def userHome = System.getProperty("user.home") + def tmpDirectory = new File(userHome + File.separator + 'groovytest') + if (!tmpDirectory.exists()) { + tmpDirectory.mkdir() + } new LocalDynamicMetadataResolver().with { it.name = 'LocalDynamic' it.xmlId = 'LocalDynamic' - it.sourceDirectory = '/tmp' + it.sourceDirectory = userHome + File.separator + 'groovytest' it.dynamicMetadataResolverAttributes = new DynamicMetadataResolverAttributes().with { it } From 4f29233a51f46d44be1a0693f2ff590b25e8d2c6 Mon Sep 17 00:00:00 2001 From: Bill Smith Date: Thu, 23 Aug 2018 11:19:34 -0700 Subject: [PATCH 2/4] [NOJIRA] Replaced sourceDirectory /groovytest with Files.createTempDirectory --- ...ResolversControllerIntegrationTests.groovy | 27 +++++++++++++------ .../admin/ui/util/TestObjectGenerator.groovy | 9 +++---- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy index f88df2d1b..9775160bd 100644 --- a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversControllerIntegrationTests.groovy @@ -6,6 +6,7 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule import edu.internet2.tier.shibboleth.admin.ui.domain.filters.EntityAttributesFilter import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.DynamicHttpMetadataResolver import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.FileBackedHttpMetadataResolver +import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.LocalDynamicMetadataResolver import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository import edu.internet2.tier.shibboleth.admin.ui.util.TestObjectGenerator import edu.internet2.tier.shibboleth.admin.util.AttributeUtility @@ -152,6 +153,10 @@ class MetadataResolversControllerIntegrationTests extends Specification { def "POST new concrete MetadataResolver of type #resolverType -> /api/MetadataResolvers"(String resolverType) { given: 'New MetadataResolver JSON representation' def resolver = generator.buildRandomMetadataResolverOfType(resolverType) + String sourceDirectory + if (resolverType.equals('Localdynamic')) { + sourceDirectory = ((LocalDynamicMetadataResolver) resolver).sourceDirectory + } when: 'POST request is made with new DynamicHttpMetadataResolver JSON representation' def result = this.restTemplate.postForEntity(BASE_URI, createRequestHttpEntityFor { mapper.writeValueAsString(resolver) }, String) @@ -161,10 +166,11 @@ class MetadataResolversControllerIntegrationTests extends Specification { result.headers.Location[0].contains(BASE_URI) cleanup: - def userHome = System.getProperty("user.home") - def tmpDirectory = new File(userHome + File.separator + 'groovytest') - if (tmpDirectory.exists()) { - tmpDirectory.deleteDir() + if (sourceDirectory != null) { + def tmpDirectory = new File(sourceDirectory) + if (tmpDirectory.exists()) { + tmpDirectory.deleteDir() + } } where: @@ -180,6 +186,10 @@ class MetadataResolversControllerIntegrationTests extends Specification { def "PUT concrete MetadataResolver of type #resolverType with updated changes -> /api/MetadataResolvers/{resourceId}"(String resolverType) { given: 'One resolver is available in data store' def resolver = generator.buildRandomMetadataResolverOfType(resolverType) + String sourceDirectory + if (resolverType.equals('Localdynamic')) { + sourceDirectory = ((LocalDynamicMetadataResolver) resolver).sourceDirectory + } def resolverResourceId = resolver.resourceId metadataResolverRepository.save(resolver) @@ -204,10 +214,11 @@ class MetadataResolversControllerIntegrationTests extends Specification { updatedResolverMap.name == 'Updated DynamicHttpMetadataResolver' cleanup: - def userHome = System.getProperty("user.home") - def tmpDirectory = new File(userHome + File.separator + 'groovytest') - if (tmpDirectory.exists()) { - tmpDirectory.deleteDir() + if (sourceDirectory != null) { + def tmpDirectory = new File(sourceDirectory) + if (tmpDirectory.exists()) { + tmpDirectory.deleteDir() + } } where: diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy index 187ec50f8..e7d227160 100644 --- a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestObjectGenerator.groovy @@ -10,6 +10,7 @@ import edu.internet2.tier.shibboleth.admin.util.AttributeUtility import edu.internet2.tier.shibboleth.admin.util.MDDCConstants import org.opensaml.saml.saml2.metadata.Organization +import java.nio.file.Files import java.util.function.Supplier /** @@ -483,15 +484,11 @@ class TestObjectGenerator { } LocalDynamicMetadataResolver localDynamicMetadataResolver() { - def userHome = System.getProperty("user.home") - def tmpDirectory = new File(userHome + File.separator + 'groovytest') - if (!tmpDirectory.exists()) { - tmpDirectory.mkdir() - } + def tmpDirectory = Files.createTempDirectory("groovy") new LocalDynamicMetadataResolver().with { it.name = 'LocalDynamic' it.xmlId = 'LocalDynamic' - it.sourceDirectory = userHome + File.separator + 'groovytest' + it.sourceDirectory = tmpDirectory it.dynamicMetadataResolverAttributes = new DynamicMetadataResolverAttributes().with { it } From b6105f857950d7521a4a3613562d7dd3e0f339ba Mon Sep 17 00:00:00 2001 From: Bill Smith Date: Thu, 23 Aug 2018 11:39:00 -0700 Subject: [PATCH 3/4] [NOJIRA] Added an attribute filter to force DiffBuilder to ignore sourceDirectory --- .../internet2/tier/shibboleth/admin/ui/util/TestHelpers.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestHelpers.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestHelpers.groovy index 04066c581..29db1fb1d 100644 --- a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestHelpers.groovy +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/util/TestHelpers.groovy @@ -29,6 +29,7 @@ class TestHelpers { static void generatedXmlIsTheSameAsExpectedXml(String expectedXmlResource, Document generatedXml) { assert !DiffBuilder.compare(Input.fromStream(TestHelpers.getResourceAsStream(expectedXmlResource))) .withTest(Input.fromDocument(generatedXml)) + .withAttributeFilter({attribute -> !attribute.name.equals("sourceDirectory")}) .ignoreComments() .ignoreWhitespace() .build() From 77b208b659924a4b4c9b813025a2dc04e1996d0b Mon Sep 17 00:00:00 2001 From: Bill Smith Date: Thu, 23 Aug 2018 14:29:34 -0700 Subject: [PATCH 4/4] [NOJIRA] Updated beans to try to fix circular dep issue. Fixed a couple conversion issues.. because oops. --- .../CoreShibUiConfiguration.java | 44 ++-------------- .../ui/service/DirectoryServiceImpl.java | 3 +- .../ui/service/EntityIdsSearchService.java | 2 +- .../service/EntityIdsSearchServiceImpl.java | 52 +++++++++++++++++++ .../shibboleth/admin/util/LuceneUtility.java | 10 ++-- 5 files changed, 66 insertions(+), 45 deletions(-) create mode 100644 backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceImpl.java diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java index 18a8a4953..c9b6493fa 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java @@ -1,6 +1,5 @@ package edu.internet2.tier.shibboleth.admin.ui.configuration; -import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityIdsSearchResultRepresentation; import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects; import edu.internet2.tier.shibboleth.admin.ui.repository.EntityDescriptorRepository; import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository; @@ -11,6 +10,7 @@ import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryServiceImpl; import edu.internet2.tier.shibboleth.admin.ui.service.EntityDescriptorService; import edu.internet2.tier.shibboleth.admin.ui.service.EntityIdsSearchService; +import edu.internet2.tier.shibboleth.admin.ui.service.EntityIdsSearchServiceImpl; import edu.internet2.tier.shibboleth.admin.ui.service.EntityService; import edu.internet2.tier.shibboleth.admin.ui.service.FilterService; import edu.internet2.tier.shibboleth.admin.ui.service.FilterTargetService; @@ -24,13 +24,6 @@ import edu.internet2.tier.shibboleth.admin.util.AttributeUtility; import edu.internet2.tier.shibboleth.admin.util.LuceneUtility; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.queryparser.classic.ParseException; -import org.apache.lucene.queryparser.classic.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -46,9 +39,6 @@ import org.springframework.web.util.UrlPathHelper; import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; @Configuration public class CoreShibUiConfiguration { @@ -92,44 +82,20 @@ public AttributeUtility attributeUtility() { return new AttributeUtility(openSamlObjects()); } - @Autowired - Analyzer fullTokenAnalyzer; - - @Autowired - DirectoryService directoryService; - @Autowired LocaleResolver localeResolver; @Autowired ResourceBundleMessageSource messageSource; - @Autowired - LuceneUtility luceneUtility; - @Bean public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(EntityDescriptorRepository entityDescriptorRepository) { return new EntityDescriptorFilesScheduledTasks(this.metadataDir, entityDescriptorRepository, openSamlObjects()); } @Bean - public EntityIdsSearchService entityIdsSearchService() { - return (resourceId, term, limit) -> { - List entityIds = new ArrayList<>(); - try { - IndexReader indexReader = luceneUtility.getIndexReader(resourceId); - IndexSearcher searcher = new IndexSearcher(indexReader); - QueryParser parser = new QueryParser("content", fullTokenAnalyzer); - TopDocs topDocs = searcher.search(parser.parse(term.trim()), limit); - for (ScoreDoc scoreDoc : topDocs.scoreDocs) { - Document document = searcher.doc(scoreDoc.doc); - entityIds.add(document.get("id")); - } - } catch (IOException | ParseException e) { - logger.error(e.getMessage(), e); - } - return new EntityIdsSearchResultRepresentation(entityIds); - }; + public EntityIdsSearchService entityIdsSearchService(LuceneUtility luceneUtility, Analyzer fullTokenAnalyzer) { + return new EntityIdsSearchServiceImpl(luceneUtility, fullTokenAnalyzer); } @Bean @@ -199,7 +165,7 @@ public DirectoryService directoryService() { } @Bean - public LuceneUtility luceneUtility() { - return new LuceneUtility(); + public LuceneUtility luceneUtility(DirectoryService directoryService) { + return new LuceneUtility(directoryService); } } diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/DirectoryServiceImpl.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/DirectoryServiceImpl.java index 553af3094..d19591b35 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/DirectoryServiceImpl.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/DirectoryServiceImpl.java @@ -3,6 +3,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -25,6 +26,6 @@ public Directory getDirectory(String resourceId) { @Override public List getDirectories() { - return (List) directoryMap.values(); + return new ArrayList<>(directoryMap.values()); } } diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchService.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchService.java index 6639d5b1a..2e1707e4e 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchService.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchService.java @@ -1,6 +1,7 @@ package edu.internet2.tier.shibboleth.admin.ui.service; import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityIdsSearchResultRepresentation; +import edu.internet2.tier.shibboleth.admin.util.LuceneUtility; import net.andreinc.mockneat.MockNeat; import java.util.ArrayList; @@ -10,7 +11,6 @@ /** * API component responsible for entity ids search. */ -@FunctionalInterface public interface EntityIdsSearchService { /** diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceImpl.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceImpl.java new file mode 100644 index 000000000..337b904b4 --- /dev/null +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceImpl.java @@ -0,0 +1,52 @@ +package edu.internet2.tier.shibboleth.admin.ui.service; + +import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityIdsSearchResultRepresentation; +import edu.internet2.tier.shibboleth.admin.util.LuceneUtility; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * @author Bill Smith (wsmith@unicon.net) + */ +public class EntityIdsSearchServiceImpl implements EntityIdsSearchService { + private static final Logger logger = LoggerFactory.getLogger(EntityIdsSearchServiceImpl.class); + private Analyzer fullTokenAnalyzer; + private LuceneUtility luceneUtility; + + public EntityIdsSearchServiceImpl(LuceneUtility luceneUtility, Analyzer fullTokenAnalyzer) { + this.luceneUtility = luceneUtility; + this.fullTokenAnalyzer = fullTokenAnalyzer; + } + + @Override + public EntityIdsSearchResultRepresentation findBySearchTermAndOptionalLimit(String resourceId, + String searchTerm, + int limit) { + List entityIds = new ArrayList<>(); + try { + IndexReader indexReader = luceneUtility.getIndexReader(resourceId); + IndexSearcher searcher = new IndexSearcher(indexReader); + QueryParser parser = new QueryParser("content", fullTokenAnalyzer); + TopDocs topDocs = searcher.search(parser.parse(searchTerm.trim()), limit); + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + Document document = searcher.doc(scoreDoc.doc); + entityIds.add(document.get("id")); + } + } catch (IOException | ParseException e) { + logger.error(e.getMessage(), e); + } + return new EntityIdsSearchResultRepresentation(entityIds); + } +} diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/util/LuceneUtility.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/util/LuceneUtility.java index 9b53ebdbb..57a96051e 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/util/LuceneUtility.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/util/LuceneUtility.java @@ -8,7 +8,6 @@ import org.apache.lucene.store.Directory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import java.io.IOException; import java.util.ArrayList; @@ -19,10 +18,12 @@ */ public class LuceneUtility { private static final Logger logger = LoggerFactory.getLogger(LuceneUtility.class); - - @Autowired private DirectoryService directoryService; + public LuceneUtility(DirectoryService directoryService) { + this.directoryService = directoryService; + } + public IndexReader getIndexReader(String resourceId) throws IOException { IndexReader indexReader; if (StringUtils.isBlank(resourceId)) { @@ -35,7 +36,8 @@ public IndexReader getIndexReader(String resourceId) throws IOException { logger.error(e.getMessage(), e); } }); - IndexReader[] indexReaders = (IndexReader[]) indexReaderList.toArray(); + IndexReader[] indexReaders = new IndexReader[indexReaderList.size()]; + indexReaders = indexReaderList.toArray(indexReaders); indexReader = new MultiReader(indexReaders, true); } else { indexReader = DirectoryReader.open(directoryService.getDirectory(resourceId));