Skip to content

Commit

Permalink
Merge branch 'master' into SHIBUI-661
Browse files Browse the repository at this point in the history
  • Loading branch information
Bill Smith committed Aug 23, 2018
2 parents 68a56b7 + 7885efc commit b908a03
Show file tree
Hide file tree
Showing 43 changed files with 1,187 additions and 197 deletions.
Original file line number Diff line number Diff line change
@@ -1,23 +1,36 @@
package edu.internet2.tier.shibboleth.admin.ui.configuration;

import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityIdsSearchResultRepresentation;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolversPositionOrderContainer;
import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
import edu.internet2.tier.shibboleth.admin.ui.repository.EntityDescriptorRepository;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolversPositionOrderContainerRepository;
import edu.internet2.tier.shibboleth.admin.ui.scheduled.EntityDescriptorFilesScheduledTasks;
import edu.internet2.tier.shibboleth.admin.ui.service.*;
import edu.internet2.tier.shibboleth.admin.ui.service.DefaultMetadataResolversPositionOrderContainerService;
import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityDescriptorService;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityIdsSearchService;
import edu.internet2.tier.shibboleth.admin.ui.service.EntityService;
import edu.internet2.tier.shibboleth.admin.ui.service.FilterService;
import edu.internet2.tier.shibboleth.admin.ui.service.FilterTargetService;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAEntityDescriptorServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAEntityServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAFilterServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAFilterTargetServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.JPAMetadataResolverServiceImpl;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolversPositionOrderContainerService;
import edu.internet2.tier.shibboleth.admin.util.AttributeUtility;
import edu.internet2.tier.shibboleth.admin.util.LuceneUtility;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -83,25 +96,29 @@ public AttributeUtility attributeUtility() {
Analyzer fullTokenAnalyzer;

@Autowired
Directory directory;
DirectoryService directoryService;

@Autowired
LocaleResolver localeResolver;

@Autowired
ResourceBundleMessageSource messageSource;

@Autowired
LuceneUtility luceneUtility;

@Bean
public EntityDescriptorFilesScheduledTasks entityDescriptorFilesScheduledTasks(EntityDescriptorRepository entityDescriptorRepository) {
return new EntityDescriptorFilesScheduledTasks(this.metadataDir, entityDescriptorRepository, openSamlObjects());
}

@Bean
public EntityIdsSearchService entityIdsSearchService() {
return (term, limit) -> {
return (resourceId, term, limit) -> {
List<String> entityIds = new ArrayList<>();
try {
IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(directory));
IndexReader indexReader = luceneUtility.getIndexReader(resourceId);
IndexSearcher searcher = new IndexSearcher(indexReader);
QueryParser parser = new QueryParser("content", fullTokenAnalyzer);
TopDocs topDocs = searcher.search(parser.parse(term.trim()), limit);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Expand Down Expand Up @@ -175,4 +192,14 @@ public void addInterceptors(InterceptorRegistry registry) {
return new DefaultMetadataResolversPositionOrderContainerService(positionOrderContainerRepository, resolverRepository);

}

@Bean
public DirectoryService directoryService() {
return new DirectoryServiceImpl();
}

@Bean
public LuceneUtility luceneUtility() {
return new LuceneUtility();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
import net.shibboleth.utilities.java.support.resolver.ResolverException;
import org.apache.http.HttpResponse;
Expand Down Expand Up @@ -38,7 +39,7 @@ public class MetadataResolverConfiguration {
OpenSamlObjects openSamlObjects;

@Autowired
IndexWriter indexWriter;
IndexWriterService indexWriterService;

@Autowired
MetadataResolverRepository metadataResolverRepository;
Expand All @@ -50,13 +51,22 @@ public MetadataResolver metadataResolver() throws ResolverException, ComponentIn

List<MetadataResolver> resolvers = new ArrayList<>();

String incommonMRId = "incommonmd";
// TODO: remove this later when we allow for creation of arbitrary metadata resolvers
FileBackedHTTPMetadataResolver incommonMR = new FileBackedHTTPMetadataResolver(HttpClients.createMinimal(), "http://md.incommon.org/InCommon/InCommon-metadata.xml", "/tmp/incommonmd.xml"){
@Override
protected void initMetadataResolver() throws ComponentInitializationException {
super.initMetadataResolver();

IndexWriter indexWriter;
try {
indexWriter = indexWriterService.getIndexWriter(incommonMRId);
} catch (IOException e) {
throw new ComponentInitializationException(e);
}

for (String entityId: this.getBackingStore().getIndexedDescriptors().keySet()) {

Document document = new Document();
document.add(new StringField("id", entityId, Field.Store.YES));
document.add(new TextField("content", entityId, Field.Store.YES)); // TODO: change entityId to be content of entity descriptor block
Expand Down Expand Up @@ -86,7 +96,7 @@ protected void processConditionalRetrievalHeaders(HttpResponse response) {
// let's do nothing 'cause we want to allow a refresh
}
};
incommonMR.setId("incommonmd");
incommonMR.setId(incommonMRId);
incommonMR.setParserPool(openSamlObjects.getParserPool());
incommonMR.setMetadataFilter(new MetadataFilterChain());
incommonMR.initialize();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package edu.internet2.tier.shibboleth.admin.ui.configuration;

import edu.internet2.tier.shibboleth.admin.ui.service.DirectoryService;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.StopFilter;
Expand All @@ -10,18 +12,18 @@
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

@Configuration
public class SearchConfiguration {
@Bean
Directory directory() {
return new RAMDirectory();
}
@Autowired
DirectoryService directoryService;

@Bean
Analyzer analyzer() {
Expand Down Expand Up @@ -54,10 +56,23 @@ protected TokenStreamComponents createComponents(String fieldName) {
};
}

@Bean
IndexWriter indexWriter() throws IOException {
private IndexWriter createIndexWriter(Directory directory) throws IOException {
IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer());
indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
return new IndexWriter(directory(), indexWriterConfig);
return new IndexWriter(directory, indexWriterConfig);
}

@Bean
public IndexWriterService indexWriterService() {
Map<String, IndexWriter> indexWriterMap = new HashMap<>();

return resourceId -> {
IndexWriter indexWriter = indexWriterMap.get(resourceId);
if (indexWriter == null) {
indexWriter = createIndexWriter(directoryService.getDirectory(resourceId));
indexWriterMap.put(resourceId, indexWriter);
}
return indexWriter;
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ public EntityIdsSearchController(EntityIdsSearchService entityIdsSearchService)
}

@GetMapping
ResponseEntity<?> search(@RequestParam String term, @RequestParam(required = false) Integer limit) {
ResponseEntity<?> search(@RequestParam(required = false) String resourceId,
@RequestParam String term,
@RequestParam(required = false) Integer limit) {
//Zero indicates no-limit
final int resultLimit = (limit != null ? limit : 10);
return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(term, resultLimit));
return ResponseEntity.ok(this.entityIdsSearchService.findBySearchTermAndOptionalLimit(resourceId, term, resultLimit));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package edu.internet2.tier.shibboleth.admin.ui.controller;

import edu.internet2.tier.shibboleth.admin.ui.controller.support.RestControllersSupport;
import edu.internet2.tier.shibboleth.admin.ui.domain.filters.MetadataFilter;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.stream.Collectors;

import static java.util.stream.Collectors.toList;

/**
* @author Dmitriy Kopylenko
*/
@RestController
@RequestMapping("/api/MetadataResolvers/{metadataResolverId}/FiltersPositionOrder")
public class MetadataFiltersPositionOrderController {

@Autowired
MetadataResolverRepository metadataResolverRepository;

@Autowired
RestControllersSupport restControllersSupport;

@PostMapping
@Transactional
public ResponseEntity<?> updateFiltersPositionOrder(@PathVariable String metadataResolverId,
@RequestBody List<String> filtersResourceIds) {

MetadataResolver resolver = restControllersSupport.findResolverOrThrowHttp404(metadataResolverId);
List<MetadataFilter> currentFilters = resolver.getMetadataFilters();

//Check for bad data upfront. We could avoid this check and take wrong size and/or filter ids and blindly pass to sort below.
//In that case, the sort operation will silently NOT do anything and leave original filters order,
//but we will not be able to indicate to calling clients HTTP 400 in that case.
if ((filtersResourceIds.size() != currentFilters.size()) ||
(!currentFilters.stream()
.map(MetadataFilter::getResourceId)
.collect(toList())
.containsAll(filtersResourceIds))) {

return ResponseEntity
.badRequest()
.body("Number of filters to reorder or filters resource ids do not match current filters");
}

//This is needed in order to set reference to persistent filters collection to be able to merge the persistent collection
//Otherwise if we manipulate the original collection directly and try to save, we'll get RDBMS constraint violation exception
List<MetadataFilter> reOrderedFilters = new ArrayList<>(currentFilters);

//Main re-ordering operation
reOrderedFilters.sort(Comparator.comparingInt(f -> filtersResourceIds.indexOf(f.getResourceId())));

//re-set the reference and save to DB
resolver.setMetadataFilters(reOrderedFilters);
metadataResolverRepository.save(resolver);

return ResponseEntity.noContent().build();
}

@GetMapping
@Transactional(readOnly = true)
public ResponseEntity<?> getFiltersPositionOrder(@PathVariable String metadataResolverId) {
MetadataResolver resolver = restControllersSupport.findResolverOrThrowHttp404(metadataResolverId);
List<String> resourceIds = resolver.getMetadataFilters().stream()
.map(MetadataFilter::getResourceId)
.collect(toList());

return ResponseEntity.ok(resourceIds);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,15 @@
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolverValidationService;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolversPositionOrderContainerService;
import lombok.extern.slf4j.Slf4j;
import net.shibboleth.utilities.java.support.component.ComponentInitializationException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
Expand Down Expand Up @@ -52,6 +58,9 @@ public class MetadataResolversController {
@Autowired
MetadataResolversPositionOrderContainerService positionOrderContainerService;

@Autowired
IndexWriterService indexWriterService;

@ExceptionHandler({InvalidTypeIdException.class, IOException.class, HttpMessageNotReadableException.class})
public ResponseEntity<?> unableToParseJson(Exception ex) {
return ResponseEntity.badRequest().body(new ErrorResponse(HttpStatus.BAD_REQUEST.toString(), ex.getMessage()));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package edu.internet2.tier.shibboleth.admin.ui.controller.support;

import com.google.common.collect.ImmutableMap;
import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.client.HttpClientErrorException;

import static org.springframework.http.HttpStatus.NOT_FOUND;

/**
* Common functionality for REST controllers.
*
* @author Dmitriy Kopylenko
*/
@RestControllerAdvice
public class RestControllersSupport {

@Autowired
MetadataResolverRepository resolverRepository;

public MetadataResolver findResolverOrThrowHttp404(String resolverResourceId) {
MetadataResolver resolver = resolverRepository.findByResourceId(resolverResourceId);
if(resolver == null) {
throw new HttpClientErrorException(NOT_FOUND, "Metadata resolver is not found");
}
return resolver;
}


@ExceptionHandler
public ResponseEntity<?> notFoundHandler(HttpClientErrorException ex) {
if(ex.getStatusCode() == NOT_FOUND) {
return ResponseEntity.status(NOT_FOUND).body(ex.getStatusText());
}
throw ex;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package edu.internet2.tier.shibboleth.admin.ui.service;

import org.apache.lucene.store.Directory;

import java.util.List;

/**
* API component responsible for entity ids search.
*/
public interface DirectoryService {
/**
* Return a Directory for a given resource id. If one is not found, it will be created.
* @param resourceId the resource to get the Directory for
* @return Directory
*/
Directory getDirectory(String resourceId);

List<Directory> getDirectories();
}
Loading

0 comments on commit b908a03

Please sign in to comment.