Skip to content

Commit

Permalink
Merged in SHIBUI-660 (pull request #169)
Browse files Browse the repository at this point in the history
SHIBUI-660

Approved-by: Shibui Jenkins <shibui.jenkins@gmail.com>
  • Loading branch information
Bill Smith authored and Jonathan Johnson committed Aug 22, 2018
2 parents bf5288e + b9259cf commit e937307
Show file tree
Hide file tree
Showing 14 changed files with 274 additions and 53 deletions.
Original file line number Diff line number Diff line change
@@ -1,23 +1,36 @@
package edu.internet2.tier.shibboleth.admin.ui.configuration;

import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityIdsSearchResultRepresentation;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolversPositionOrderContainer;
import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
import edu.internet2.tier.shibboleth.admin.ui.repository.EntityDescriptorRepository;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolversPositionOrderContainerRepository;
import edu.internet2.tier.shibboleth.admin.ui.scheduled.EntityDescriptorFilesScheduledTasks;
import edu.internet2.tier.shibboleth.admin.ui.service.*;
import edu.internet2.tier.shibboleth.admin.ui.service.DefaultMetadataResolversPositionOrderContainerService;
import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityDescriptorService;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityIdsSearchService;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityService;
import edu.internet2.tier.shibboleth.admin.ui.service.FilterService;
import edu.internet2.tier.shibboleth.admin.ui.service.FilterTargetService;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAEntityDescriptorServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAEntityServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAFilterServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAFilterTargetServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAMetadataResolverServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolversPositionOrderContainerService;
import edu.internet2.tier.shibboleth.admin.util.AttributeUtility;
import edu.internet2.tier.shibboleth.admin.util.LuceneUtility;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -83,25 +96,29 @@ public AttributeUtility attributeUtility() {
Analyzer fullTokenAnalyzer;

@Autowired
Directory directory;
DirectoryService directoryService;

@Autowired
LocaleResolver localeResolver;

@Autowired
ResourceBundleMessageSource messageSource;

@Autowired
LuceneUtility luceneUtility;

@Bean
public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(EntityDescriptorRepository entityDescriptorRepository) {
return new EntityDescriptorFilesScheduledTasks(this.metadataDir, entityDescriptorRepository, openSamlObjects());
}

@Bean
public EntityIdsSearchService entityIdsSearchService() {
return (term, limit) -> {
return (resourceId, term, limit) -> {
List<String> entityIds = new ArrayList<>();
try {
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
IndexReader indexReader = luceneUtility.getIndexReader(resourceId);
IndexSearcher searcher = new IndexSearcher(indexReader);
QueryParser parser = new QueryParser("content", fullTokenAnalyzer);
TopDocs topDocs = searcher.search(parser.parse(term.trim()), limit);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Expand Down Expand Up @@ -175,4 +192,14 @@ public void addInterceptors(InterceptorRegistry registry) {
return new DefaultMetadataResolversPositionOrderContainerService(positionOrderContainerRepository, resolverRepository);

}

@Bean
public DirectoryService directoryService() {
return new DirectoryServiceImpl();
}

@Bean
public LuceneUtility luceneUtility() {
return new LuceneUtility();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
import net.shibboleth.utilities.java.support.resolver.ResolverException;
import org.apache.http.HttpResponse;
Expand Down Expand Up @@ -38,7 +39,7 @@ public class MetadataResolverConfiguration {
OpenSamlObjects openSamlObjects;

@Autowired
IndexWriter indexWriter;
IndexWriterService indexWriterService;

@Autowired
MetadataResolverRepository metadataResolverRepository;
Expand All @@ -50,13 +51,22 @@ public MetadataResolver metadataResolver() throws ResolverException, ComponentIn

List<MetadataResolver> resolvers = new ArrayList<>();

String incommonMRId = "incommonmd";
// TODO: remove this later when we allow for creation of arbitrary metadata resolvers
FileBackedHTTPMetadataResolver incommonMR = new FileBackedHTTPMetadataResolver(HttpClients.createMinimal(), "http://md.incommon.org/InCommon/InCommon-metadata.xml", "/tmp/incommonmd.xml"){
@Override
protected void initMetadataResolver() throws ComponentInitializationException {
super.initMetadataResolver();

IndexWriter indexWriter;
try {
indexWriter = indexWriterService.getIndexWriter(incommonMRId);
} catch (IOException e) {
throw new ComponentInitializationException(e);
}

for (String entityId: this.getBackingStore().getIndexedDescriptors().keySet()) {

Document document = new Document();
document.add(new StringField("id", entityId, Field.Store.YES));
document.add(new TextField("content", entityId, Field.Store.YES)); // TODO: change entityId to be content of entity descriptor block
Expand Down Expand Up @@ -86,7 +96,7 @@ protected void processConditionalRetrievalHeaders(HttpResponse response) {
// let's do nothing 'cause we want to allow a refresh
}
};
incommonMR.setId("incommonmd");
incommonMR.setId(incommonMRId);
incommonMR.setParserPool(openSamlObjects.getParserPool());
incommonMR.setMetadataFilter(new MetadataFilterChain());
incommonMR.initialize();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package edu.internet2.tier.shibboleth.admin.ui.configuration;

import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
Expand All @@ -10,18 +12,18 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

@Configuration
public class SearchConfiguration {
@Bean
Directory directory() {
return new RAMDirectory();
}
@Autowired
DirectoryService directoryService;

@Bean
Analyzer analyzer() {
Expand Down Expand Up @@ -54,10 +56,23 @@ protected TokenStreamComponents createComponents(String fieldName) {
};
}

@Bean
IndexWriter indexWriter() throws IOException {
private IndexWriter createIndexWriter(Directory directory) throws IOException {
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer());
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(), indexWriterConfig);
return new IndexWriter(directory, indexWriterConfig);
}

@Bean
public IndexWriterService indexWriterService() {
Map<String, IndexWriter> indexWriterMap = new HashMap<>();

return resourceId -> {
IndexWriter indexWriter = indexWriterMap.get(resourceId);
if (indexWriter == null) {
indexWriter = createIndexWriter(directoryService.getDirectory(resourceId));
indexWriterMap.put(resourceId, indexWriter);
}
return indexWriter;
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ public EntityIdsSearchController(EntityIdsSearchService entityIdsSearchService)
}

@GetMapping
ResponseEntity<?> search(@RequestParam String term, @RequestParam(required = false) Integer limit) {
ResponseEntity<?> search(@RequestParam(required = false) String resourceId,
@RequestParam String term,
@RequestParam(required = false) Integer limit) {
//Zero indicates no-limit
final int resultLimit = (limit != null ? limit : 10);
return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(term, resultLimit));
return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(resourceId, term, resultLimit));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,15 @@
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolverValidationService;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolversPositionOrderContainerService;
import lombok.extern.slf4j.Slf4j;
import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -52,6 +58,9 @@ public class MetadataResolversController {
@Autowired
MetadataResolversPositionOrderContainerService positionOrderContainerService;

@Autowired
IndexWriterService indexWriterService;

@ExceptionHandler({InvalidTypeIdException.class, IOException.class, HttpMessageNotReadableException.class})
public ResponseEntity<?> unableToParseJson(Exception ex) {
return ResponseEntity.badRequest().body(new ErrorResponse(HttpStatus.BAD_REQUEST.toString(), ex.getMessage()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package edu.internet2.tier.shibboleth.admin.ui.service;

import org.apache.lucene.store.Directory;

import java.util.List;

/**
* API component responsible for entity ids search.
*/
public interface DirectoryService {
/**
* Return a Directory for a given resource id. If one is not found, it will be created.
* @param resourceId the resource to get the Directory for
* @return Directory
*/
Directory getDirectory(String resourceId);

List<Directory> getDirectories();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package edu.internet2.tier.shibboleth.admin.ui.service;

import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* @author Bill Smith (wsmith@unicon.net)
*/
public class DirectoryServiceImpl implements DirectoryService {
private Map<String, Directory> directoryMap = new HashMap<>();

@Override
public Directory getDirectory(String resourceId) {
Directory directory = directoryMap.get(resourceId);
if (directory == null) {
directory = new RAMDirectory();
directoryMap.put(resourceId, directory);
}
return directory;
}

@Override
public List<Directory> getDirectories() {
return (List<Directory>) directoryMap.values();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@ public interface EntityIdsSearchService {

/**
* Find a list of entity ids
* @param resourceId the id of the resource to search within
* @param searchTerm for the query
* @param limit optional limit of query results to return. Zero or less value indicates no limit.
* @return EntityIdsSearchResultRepresentation
*/
EntityIdsSearchResultRepresentation findBySearchTermAndOptionalLimit(String searchTerm, int limit);
EntityIdsSearchResultRepresentation findBySearchTermAndOptionalLimit(String resourceId, String searchTerm, int limit);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package edu.internet2.tier.shibboleth.admin.ui.service;

import org.apache.lucene.index.IndexWriter;

import java.io.IOException;

/**
* API component responsible for entity ids search.
*/
@FunctionalInterface
public interface IndexWriterService {
/**
* Return a (possibly cached) index writer for a given resource id.
* @param resourceId the resource to create the IndexWriter for
* @return IndexWriter
*/
IndexWriter getIndexWriter(String resourceId) throws IOException;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package edu.internet2.tier.shibboleth.admin.util;

import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.store.Directory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
* @author Bill Smith (wsmith@unicon.net)
*/
public class LuceneUtility {
private static final Logger logger = LoggerFactory.getLogger(LuceneUtility.class);

@Autowired
private DirectoryService directoryService;

public IndexReader getIndexReader(String resourceId) throws IOException {
IndexReader indexReader;
if (StringUtils.isBlank(resourceId)) {
List<Directory> directories = directoryService.getDirectories();
List<IndexReader> indexReaderList = new ArrayList<>();
directories.forEach(it -> {
try {
indexReaderList.add(DirectoryReader.open(it));
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
IndexReader[] indexReaders = (IndexReader[]) indexReaderList.toArray();
indexReader = new MultiReader(indexReaders, true);
} else {
indexReader = DirectoryReader.open(directoryService.getDirectory(resourceId));
}
return indexReader;
}
}
Loading

0 comments on commit e937307

Please sign in to comment.