diff --git a/backend/build.gradle b/backend/build.gradle index 12514b00c..ede094346 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -58,7 +58,13 @@ dependencies { ['starter-web', 'starter-data-jpa', 'starter-security', 'starter-actuator', 'devtools'].each { compile "org.springframework.boot:spring-boot-${it}" } - providedRuntime 'org.springframework.boot:spring-boot-starter-tomcat' + // TODO: figure out what this should really be + runtimeOnly 'org.springframework.boot:spring-boot-starter-tomcat' + + // lucene deps + ['core', 'analyzers-common', 'queryparser'].each { + compile "org.apache.lucene:lucene-${it}:${project.'lucene.version'}" + } compile "org.liquibase:liquibase-core" compile group: 'org.jadira.usertype', name: 'usertype.core', version: '6.0.1.GA' diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java index 34085e489..2aec22834 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java @@ -7,15 +7,29 @@ import edu.internet2.tier.shibboleth.admin.ui.service.EntityDescriptorService; import edu.internet2.tier.shibboleth.admin.ui.service.EntityIdsSearchService; import edu.internet2.tier.shibboleth.admin.ui.service.JPAEntityDescriptorServiceImpl; -import net.andreinc.mockneat.MockNeat; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.io.IOException; +import java.util.ArrayList; import java.util.List; @Configuration public class CoreShibUiConfiguration { + private static final Logger logger = LoggerFactory.getLogger(CoreShibUiConfiguration.class); @Value("${shibui.metadata-dir:/opt/shibboleth-idp/metadata/generated}") private String metadataDir; @@ -30,6 +44,12 @@ public EntityDescriptorService jpaEntityDescriptorService() { return new JPAEntityDescriptorServiceImpl(openSamlObjects()); } + @Autowired + Analyzer fullTokenAnalyzer; + + @Autowired + Directory directory; + @Bean public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(EntityDescriptorRepository entityDescriptorRepository) { @@ -38,12 +58,20 @@ public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(E @Bean public EntityIdsSearchService entityIdsSearchService() { - //TODO: replace with real data store implementation when ready return (term, limit) -> { - MockNeat m = MockNeat.threadLocal(); - // Just simulate returning 100 results for no-limit query - List simulatedEntityIds = limit > 0 ? m.urls().list(limit).val() : m.urls().list(100).val(); - return new EntityIdsSearchResultRepresentation(simulatedEntityIds); + List entityIds = new ArrayList<>(); + try { + IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory)); + QueryParser parser = new QueryParser("content", fullTokenAnalyzer); + TopDocs topDocs = searcher.search(parser.parse(term), limit); + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + Document document = searcher.doc(scoreDoc.doc); + entityIds.add(document.get("id")); + } + } catch (IOException | ParseException e) { + logger.error(e.getMessage(), e); + } + return new EntityIdsSearchResultRepresentation(entityIds); }; } } diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/MetadataResolverConfiguration.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/MetadataResolverConfiguration.java new file mode 100644 index 000000000..7158a6f9b --- /dev/null +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/MetadataResolverConfiguration.java @@ -0,0 +1,68 @@ +package edu.internet2.tier.shibboleth.admin.ui.configuration; + +import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects; +import net.shibboleth.utilities.java.support.component.ComponentInitializationException; +import net.shibboleth.utilities.java.support.resolver.ResolverException; +import org.apache.http.impl.client.HttpClients; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.opensaml.saml.metadata.resolver.ChainingMetadataResolver; +import org.opensaml.saml.metadata.resolver.MetadataResolver; +import org.opensaml.saml.metadata.resolver.impl.FileBackedHTTPMetadataResolver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; + +/** + * this is a temporary class until a better way of doing this is found. + */ +@Configuration +public class MetadataResolverConfiguration { + private static final Logger logger = LoggerFactory.getLogger(MetadataResolverConfiguration.class); + + @Autowired + OpenSamlObjects openSamlObjects; + + @Autowired + IndexWriter indexWriter; + + @Bean + public MetadataResolver metadataResolver() throws ResolverException, ComponentInitializationException { + MetadataResolver metadataResolver = new ChainingMetadataResolver(); + + FileBackedHTTPMetadataResolver incommonMR = new FileBackedHTTPMetadataResolver(HttpClients.createMinimal(), "http://md.incommon.org/InCommon/InCommon-metadata.xml", "/tmp/incommon.xml"){ + @Override + protected void initMetadataResolver() throws ComponentInitializationException { + super.initMetadataResolver(); + + for (String entityId: this.getBackingStore().getIndexedDescriptors().keySet()) { + Document document = new Document(); + document.add(new StringField("id", entityId, Field.Store.YES)); + document.add(new TextField("content", entityId, Field.Store.YES)); // TODO: change entityId to be content of entity descriptor block + try { + indexWriter.addDocument(document); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + try { + indexWriter.commit(); + } catch (IOException e) { + throw new ComponentInitializationException(e); + } + } + }; + incommonMR.setId("incommonmd"); + incommonMR.setParserPool(openSamlObjects.getParserPool()); + incommonMR.initialize(); + + return metadataResolver; + } +} diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/SearchConfiguration.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/SearchConfiguration.java new file mode 100644 index 000000000..ac5d1efb7 --- /dev/null +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/configuration/SearchConfiguration.java @@ -0,0 +1,63 @@ +package edu.internet2.tier.shibboleth.admin.ui.configuration; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseFilter; +import org.apache.lucene.analysis.StopFilter; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.ngram.NGramTokenFilter; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.IOException; + +@Configuration +public class SearchConfiguration { + @Bean + Directory directory() { + return new RAMDirectory(); + } + + @Bean + Analyzer analyzer() { + return new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final StandardTokenizer src = new StandardTokenizer(); + src.setMaxTokenLength(255); + TokenFilter tokenFilter; + tokenFilter = new NGramTokenFilter(src, 3, 10); + tokenFilter = new LowerCaseFilter(tokenFilter); + tokenFilter = new StopFilter(tokenFilter, StandardAnalyzer.STOP_WORDS_SET); + return new TokenStreamComponents(src, tokenFilter); + } + }; + } + + @Bean + Analyzer fullTokenAnalyzer() { + return new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + final StandardTokenizer src = new StandardTokenizer(); + src.setMaxTokenLength(255); + TokenFilter tokenFilter; + tokenFilter = new LowerCaseFilter(src); + tokenFilter = new StopFilter(tokenFilter, StandardAnalyzer.STOP_WORDS_SET); + return new TokenStreamComponents(src, tokenFilter); + } + }; + } + + @Bean + IndexWriter indexWriter() throws IOException { + IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer()); + indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + return new IndexWriter(directory(), indexWriterConfig); + } +} diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/EntityIdsSearchController.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/EntityIdsSearchController.java index f2df32b98..643e626f4 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/EntityIdsSearchController.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/EntityIdsSearchController.java @@ -21,7 +21,7 @@ public EntityIdsSearchController(EntityIdsSearchService entityIdsSearchService) @GetMapping ResponseEntity search(@RequestParam String term, @RequestParam(required = false) Integer limit) { //Zero indicates no-limit - final int resultLimit = (limit != null ? limit : 0); + final int resultLimit = (limit != null ? limit : 10); return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(term, resultLimit)); } } diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/AttributeAuthorityDescriptor.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/AttributeAuthorityDescriptor.java index 6cbd37fc7..9b130284d 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/AttributeAuthorityDescriptor.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/AttributeAuthorityDescriptor.java @@ -6,6 +6,7 @@ import javax.persistence.Entity; import javax.persistence.JoinColumn; import javax.persistence.OneToMany; +import java.util.ArrayList; import java.util.List; @@ -14,28 +15,28 @@ public class AttributeAuthorityDescriptor extends RoleDescriptor implements org. @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "attribauthdesc_attribserv_id") - private List attributeServices; + private List attributeServices = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "attribauthdesc_assertidreqservc_id") - private List assertionIDRequestServices; + private List assertionIDRequestServices = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "attribauthdesc_nameidfrmt_id") - private List nameIDFormats; + private List nameIDFormats = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "attribauthdesc_attribprofile_id") - private List attributeProfiles; + private List attributeProfiles = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "attribauthdesc_attrib_id") - private List attributes; + private List attributes = new ArrayList<>(); @Override public List getAttributeServices() { - return Lists.newArrayList(attributeServices); + return (List)(List)attributeServices; } public void setAttributeServices(List attributeServices) { @@ -44,7 +45,7 @@ public void setAttributeServices(List attributeServices) { @Override public List getAssertionIDRequestServices() { - return Lists.newArrayList(assertionIDRequestServices); + return (List)(List)assertionIDRequestServices; } public void setAssertionIDRequestServices(List assertionIDRequestServices) { @@ -53,7 +54,7 @@ public void setAssertionIDRequestServices(List assert @Override public List getNameIDFormats() { - return Lists.newArrayList(nameIDFormats); + return (List)(List)nameIDFormats; } public void setNameIDFormats(List nameIDFormats) { @@ -62,7 +63,7 @@ public void setNameIDFormats(List nameIDFormats) { @Override public List getAttributeProfiles() { - return Lists.newArrayList(attributeProfiles); + return (List)(List)attributeProfiles; } public void setAttributeProfiles(List attributeProfiles) { @@ -71,7 +72,7 @@ public void setAttributeProfiles(List attributeProfiles) { @Override public List getAttributes() { - return Lists.newArrayList(attributes); + return (List)(List)attributes; } public void setAttributes(List attributes) { diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/IDPSSODescriptor.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/IDPSSODescriptor.java index de1f7410e..a9efee5ad 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/IDPSSODescriptor.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/IDPSSODescriptor.java @@ -7,6 +7,7 @@ import javax.persistence.Entity; import javax.persistence.JoinColumn; import javax.persistence.OneToMany; +import java.util.ArrayList; import java.util.List; @@ -17,23 +18,23 @@ public class IDPSSODescriptor extends SSODescriptor implements org.opensaml.saml @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "idpssodesc_ssoserv_id") - private List singleSignOnServices; + private List singleSignOnServices = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "idpssodesc_nameidmapserv_id") - private List nameIDMappingServices; + private List nameIDMappingServices = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "idpssodesc_asseridreqserv_id") - private List assertionIDRequestServices; + private List assertionIDRequestServices = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "idpssodesc_attribprofile_id") - private List attributeProfiles; + private List attributeProfiles = new ArrayList<>(); @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "idpssodesc_attrib_id") - private List attributes; + private List attributes = new ArrayList<>(); @Override public Boolean getWantAuthnRequestsSigned() { @@ -57,7 +58,7 @@ public void setWantAuthnRequestsSigned(XSBooleanValue xsBooleanValue) { @Override public List getSingleSignOnServices() { - return Lists.newArrayList(singleSignOnServices); + return (List)(List)singleSignOnServices; } public void setSingleSignOnServices(List singleSignOnServices) { @@ -66,7 +67,7 @@ public void setSingleSignOnServices(List singleSignOnServic @Override public List getNameIDMappingServices() { - return Lists.newArrayList(nameIDMappingServices); + return (List)(List)nameIDMappingServices; } public void setNameIDMappingServices(List nameIDMappingServices) { @@ -75,7 +76,7 @@ public void setNameIDMappingServices(List nameIDMappingSer @Override public List getAssertionIDRequestServices() { - return Lists.newArrayList(assertionIDRequestServices); + return (List)(List)assertionIDRequestServices; } public void setAssertionIDRequestServices(List assertionIDRequestServices) { @@ -84,7 +85,7 @@ public void setAssertionIDRequestServices(List assert @Override public List getAttributeProfiles() { - return Lists.newArrayList(attributeProfiles); + return (List)(List)attributeProfiles; } public void setAttributeProfiles(List attributeProfiles) { @@ -93,7 +94,7 @@ public void setAttributeProfiles(List attributeProfiles) { @Override public List getAttributes() { - return Lists.newArrayList(attributes); + return (List)(List)attributes; } public void setAttributes(List attributes) { diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/KeyDescriptor.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/KeyDescriptor.java index d95d40f3b..fce55b916 100644 --- a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/KeyDescriptor.java +++ b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/KeyDescriptor.java @@ -1,6 +1,5 @@ package edu.internet2.tier.shibboleth.admin.ui.domain; -import com.google.common.collect.Lists; import org.opensaml.core.xml.XMLObject; import org.opensaml.security.credential.UsageType; import org.opensaml.xmlsec.signature.KeyInfo; @@ -22,7 +21,7 @@ public class KeyDescriptor extends AbstractXMLObject implements org.opensaml.sam @OneToMany(cascade = CascadeType.ALL) @JoinColumn(name = "keydesc_encryptionmethod_id") - private List encryptionMethods; + private List encryptionMethods = new ArrayList<>(); @Override public UsageType getUse() { @@ -61,7 +60,7 @@ public void setKeyInfo(KeyInfo keyInfo) { @Override public List getEncryptionMethods() { - return Lists.newArrayList(encryptionMethods); + return (List)(List) encryptionMethods; } public void setEncryptionMethods(List encryptionMethods) { diff --git a/backend/src/main/resources/jpa-saml2-metadata-config.xml b/backend/src/main/resources/jpa-saml2-metadata-config.xml index d2e17c2f8..91a2de51a 100644 --- a/backend/src/main/resources/jpa-saml2-metadata-config.xml +++ b/backend/src/main/resources/jpa-saml2-metadata-config.xml @@ -161,19 +161,17 @@ - diff --git a/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceTests.groovy b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceTests.groovy new file mode 100644 index 000000000..027f7e52c --- /dev/null +++ b/backend/src/test/groovy/edu/internet2/tier/shibboleth/admin/ui/service/EntityIdsSearchServiceTests.groovy @@ -0,0 +1,94 @@ +package edu.internet2.tier.shibboleth.admin.ui.service + +import edu.internet2.tier.shibboleth.admin.ui.configuration.CoreShibUiConfiguration +import edu.internet2.tier.shibboleth.admin.ui.configuration.MetadataResolverConfiguration +import edu.internet2.tier.shibboleth.admin.ui.configuration.SearchConfiguration +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.domain.EntityScan +import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest +import org.springframework.data.jpa.repository.config.EnableJpaRepositories +import org.springframework.test.context.ContextConfiguration +import spock.lang.Specification + +/** + * @author Bill Smith (wsmith@unicon.net) + */ +@DataJpaTest +@ContextConfiguration(classes=[CoreShibUiConfiguration, SearchConfiguration, MetadataResolverConfiguration]) +@EnableJpaRepositories(basePackages = ["edu.internet2.tier.shibboleth.admin.ui"]) +@EntityScan("edu.internet2.tier.shibboleth.admin.ui") +class EntityIdsSearchServiceTests extends Specification { + + @Autowired + EntityIdsSearchService entityIdsSearchService + + def "searching for carmen produces one result"() { + setup: + def searchTerm = "carmen" + def searchLimit = 10 + def expectedResultSize = 1 + def expectedResultItem = "https://carmenwiki.osu.edu/shibboleth" + + when: + def actualResults = entityIdsSearchService.findBySearchTermAndOptionalLimit(searchTerm, searchLimit) + + then: + expectedResultSize == actualResults.entityIds.size() + expectedResultItem == actualResults.entityIds.get(0) + } + + def "searching for unicon produces two results"() { + setup: + def searchTerm = "unicon" + def searchLimit = 10 + def expectedResultSize = 2 + def expectedResults = Arrays.asList(["http://unicon.instructure.com/saml2", "https://idp.unicon.net/idp/shibboleth"]) + + when: + def actualResults = entityIdsSearchService.findBySearchTermAndOptionalLimit(searchTerm, searchLimit) + + then: + expectedResultSize == actualResults.entityIds.size() + expectedResults.forEach { url -> actualResults.entityIds.contains(url) } + } + + def "searching for an empty string produces an empty result"() { + setup: + def searchTerm = "" + def searchLimit = 10 + def expectedResultSize = 0 + + when: + def actualResults = entityIdsSearchService.findBySearchTermAndOptionalLimit(searchTerm, searchLimit) + + then: + expectedResultSize == actualResults.entityIds.size() + } + + def "searching for unicon with a size limit of 1 produces one result"() { + setup: + def searchTerm = "unicon" + def searchLimit = 1 + def expectedResultSize = 1 + def expectedResults = Arrays.asList(["http://unicon.instructure.com/saml2"]) + + when: + def actualResults = entityIdsSearchService.findBySearchTermAndOptionalLimit(searchTerm, searchLimit) + + then: + expectedResultSize == actualResults.entityIds.size() + expectedResults.forEach { url -> actualResults.entityIds.contains(url) } + } + + def "searching for anything with a size limit of 0 produces an IllegalArgumentException"() { + setup: + def searchTerm = "anything" + def searchLimit = 0 + + when: + entityIdsSearchService.findBySearchTermAndOptionalLimit(searchTerm, searchLimit) + + then: + thrown IllegalArgumentException + } +} \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index 103d33b9a..ea90b9954 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,3 +9,5 @@ xmltooling.version=1.4.7-SNAPSHOT spring-boot.version=2.0.0.RELEASE hibernate.version=5.2.11.Final + +lucene.version=7.2.1 \ No newline at end of file