Merged in feature/beacon-enhanc (pull request #660)

* SHIBUI-2596: Adjustments to the cron for sending; Fixing unit tests and startup without a default value for beacon cron. Renamed property for beacon cron frmo code review feedback * SHIBUI-2584: Added additional beacon data collection to the output * SHIBUI-2586: Added logging output when data is sent to beacon
TIER · Jul 13, 2023 · eece9db · eece9db
2 parents 274218d + 00df9bd
commit eece9db
Show file tree

Hide file tree

Showing 25 changed files with 366 additions and 52 deletions.
diff --git a/backend/src/main/app-resources/default.yml b/backend/src/main/app-resources/default.yml
@@ -24,14 +24,14 @@
 ## Only set the installationID if you wish to define the ID used by the beacon. RECOMMENDED: ignore - a default random value will be used
 ## Only change the urls if instructed or if you wish to redirect for testing (if multiple, separate list with commas
 ## Only change the productName for testing purposes
-## Set the cron time to send the beacon data at a specific time - if unset, the system will send at a random time between 12AM and 4AM once per day
+## Set the cron time to send the beacon data at a specific time - if unset, the system will send at a random time between 12AM and 4AM once per day (random preferred)
 #  beacon:
 #    enabled: ture
 #    installationID: [user-defined value]
 #    urls: http://collector.testbed.tier.internet2.edu:5001
 #    productName: ShibUI
 #    send:
-#      cron: 0 4 * * * *
+#      cron: 0 59 3 * * ?
 
 #  pac4j-enabled: true
 #  pac4j:

diff --git a/...in/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java b/...in/java/edu/internet2/tier/shibboleth/admin/ui/configuration/CoreShibUiConfiguration.java
@@ -4,6 +4,7 @@
 import edu.internet2.tier.shibboleth.admin.ui.domain.BeaconConfiguration;
 import edu.internet2.tier.shibboleth.admin.ui.opensaml.OpenSamlObjects;
 import edu.internet2.tier.shibboleth.admin.ui.repository.BeaconConfigurationRepository;
+import edu.internet2.tier.shibboleth.admin.ui.repository.BeaconEventRepository;
 import edu.internet2.tier.shibboleth.admin.ui.repository.EntityDescriptorRepository;
 import edu.internet2.tier.shibboleth.admin.ui.repository.FilterRepository;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
@@ -44,9 +45,9 @@
 import edu.internet2.tier.shibboleth.admin.util.EntityDescriptorConversionUtils;
 import edu.internet2.tier.shibboleth.admin.util.LuceneUtility;
 import edu.internet2.tier.shibboleth.admin.util.ModelRepresentationConversions;
+import lombok.extern.slf4j.Slf4j;
 import net.javacrumbs.shedlock.core.LockProvider;
 import net.javacrumbs.shedlock.provider.jdbctemplate.JdbcTemplateLockProvider;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.lucene.analysis.Analyzer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -67,7 +68,6 @@
 import org.springframework.security.authentication.AuthenticationEventPublisher;
 import org.springframework.security.authentication.DefaultAuthenticationEventPublisher;
 import org.springframework.security.core.Authentication;
-import org.springframework.security.core.AuthenticationException;
 import org.springframework.web.servlet.LocaleResolver;
 import org.springframework.web.servlet.config.annotation.InterceptorRegistry;
 import org.springframework.web.servlet.config.annotation.PathMatchConfigurer;
@@ -83,11 +83,13 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
+import java.util.Random;
 
 @Configuration
 @Import(SearchConfiguration.class)
 @ComponentScan(basePackages = "{ edu.internet2.tier.shibboleth.admin.ui.service }")
 @EnableConfigurationProperties({CustomPropertiesConfiguration.class, ShibUIConfiguration.class})
+@Slf4j
 public class CoreShibUiConfiguration {
     @Bean
     public OpenSamlObjects openSamlObjects() {
@@ -275,18 +277,39 @@ public LockProvider lockProvider(DataSource dataSource) {
     }
 
     @Bean
-    public String getBeaconCronValue(BeaconConfigurationRepository repo)
+    public String getBeaconCronValue(BeaconConfigurationRepository repo, @Value("${shibui.beacon.send.cron:nodefault}") String valueFromConfig)
     {
-        Optional<BeaconConfiguration> bc = repo.findById(1);
-        return bc.isPresent() ? bc.get().getSendCron() : "0 3 * * * *";
+        Optional<BeaconConfiguration> obc = repo.findById(1);
+        BeaconConfiguration bc;
+        if (obc.isEmpty()) {
+            //set random cron in db
+            BeaconConfiguration newbc = new BeaconConfiguration();
+            Random rand = new Random();
+            String cron = "0 " + rand.nextInt(60) + " " + rand.nextInt(4) + " * * ?";
+            if (valueFromConfig.endsWith("?")) {
+                cron = valueFromConfig;
+            }
+            newbc.setSendCron(cron);
+            try {
+                repo.save(newbc);
+            }
+            catch(Exception e) {
+                log.debug("Error trying to create new BEACON CRON - should be ok");
+            }
+            bc = repo.findById(1).get();
+        } else {
+            bc = obc.get();
+        }
+        log.info("Scheduling beacon cron: {}", bc.getSendCron());
+        return bc.getSendCron();
     }
 
     @Bean
     public IBeaconDataService getBeaconDataService(@Value("${shibui.beacon.productName:ShibUi}") String productName, InfoEndpoint info, @Value("#{environment.TIERVERSION}") String tierVersion,
                                                    EntityDescriptorRepository entityDescriptorRepository, MetadataResolverRepository metadataResolverRepository, FilterRepository filterRepository,
                                                    GroupsRepository groupsRepository, RoleRepository roleRepository, BeaconConfigurationRepository beaconConfigurationRepository,
-                                                   UserService userService, HealthEndpoint healthEndpoint) {
-        BeaconDataServiceImpl result = new BeaconDataServiceImpl(productName, info, tierVersion, entityDescriptorRepository, metadataResolverRepository, filterRepository, groupsRepository, roleRepository, beaconConfigurationRepository, userService, healthEndpoint);
+                                                   UserService userService, HealthEndpoint healthEndpoint, BeaconEventRepository beaconEventRepository) {
+        BeaconDataServiceImpl result = new BeaconDataServiceImpl(productName, info, tierVersion, entityDescriptorRepository, metadataResolverRepository, filterRepository, groupsRepository, roleRepository, beaconConfigurationRepository, userService, healthEndpoint, beaconEventRepository);
         return result;
     }
 

diff --git a/...ain/java/edu/internet2/tier/shibboleth/admin/ui/configuration/auto/WebSecurityConfig.java b/...ain/java/edu/internet2/tier/shibboleth/admin/ui/configuration/auto/WebSecurityConfig.java
@@ -93,7 +93,7 @@ protected void configure(HttpSecurity http) throws Exception {
                         .csrf().csrfTokenRepository(CookieCsrfTokenRepository.withHttpOnlyFalse())
                         .and()
                         .authorizeRequests()
-                        .antMatchers("/unsecured/**/*","/entities/**/*", "/health").permitAll()
+                        .antMatchers("/unsecured/**/*","/entities/**/*", "/health", "/api/beacon/send").permitAll()
                         .anyRequest().hasAnyRole(acceptedAuthenticationRoles)
                         .and()
                         .exceptionHandling().accessDeniedHandler((request, response, accessDeniedException) -> response.sendRedirect("/unsecured/error.html"))

diff --git a/...d/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/ActivateController.java b/...d/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/ActivateController.java
@@ -1,8 +1,9 @@
 package edu.internet2.tier.shibboleth.admin.ui.controller;
 
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEvent;
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEventType;
 import edu.internet2.tier.shibboleth.admin.ui.domain.exceptions.MetadataFileNotFoundException;
 import edu.internet2.tier.shibboleth.admin.ui.domain.filters.MetadataFilter;
-import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.DynamicRegistrationRepresentation;
 import edu.internet2.tier.shibboleth.admin.ui.domain.frontend.EntityDescriptorRepresentation;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.exception.ForbiddenException;
@@ -12,6 +13,7 @@
 import edu.internet2.tier.shibboleth.admin.ui.service.DynamicRegistrationService;
 import edu.internet2.tier.shibboleth.admin.ui.service.EntityDescriptorService;
 import edu.internet2.tier.shibboleth.admin.ui.service.FilterService;
+import edu.internet2.tier.shibboleth.admin.ui.service.IBeaconDataService;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import io.swagger.v3.oas.annotations.tags.Tags;
@@ -41,6 +43,9 @@ public class ActivateController {
 
     @Autowired
     private MetadataResolverService metadataResolverService;
+
+    @Autowired
+    private IBeaconDataService beaconDataService;
 
     @PatchMapping(path = "/DynamicRegistration/{resourceId}/{mode}")
     @Transactional
@@ -70,6 +75,9 @@ public ResponseEntity<?> enableEntityDescriptor(@PathVariable String resourceId,
     public ResponseEntity<?> enableFilter(@PathVariable String metadataResolverId, @PathVariable String resourceId, @PathVariable String mode) throws PersistentEntityNotFound, ForbiddenException, ScriptException {
         boolean status = "enable".equalsIgnoreCase(mode);
         MetadataFilter persistedFilter = filterService.updateFilterEnabledStatus(metadataResolverId, resourceId, status);
+        if (status) {
+            beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_FILTER_ENABLED));
+        }
         return ResponseEntity.ok(persistedFilter);
     }    
 
@@ -78,6 +86,9 @@ public ResponseEntity<?> enableFilter(@PathVariable String metadataResolverId, @
     public ResponseEntity<?> enableProvider(@PathVariable String resourceId, @PathVariable String mode) throws PersistentEntityNotFound, ForbiddenException, MetadataFileNotFoundException, InitializationException {
         boolean status = "enable".equalsIgnoreCase(mode);
         MetadataResolver metadataResolver = metadataResolverService.updateMetadataResolverEnabledStatus(resourceId, status);
+        if (status) {
+            beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_PROVIDER_ENABLED));
+        }
         return ResponseEntity.ok(metadataResolver);
     }
 }
diff --git a/...end/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/BeaconController.java b/...end/src/main/java/edu/internet2/tier/shibboleth/admin/ui/controller/BeaconController.java
@@ -26,7 +26,8 @@ public ResponseEntity<?> getDetail() throws JsonProcessingException {
         return ResponseEntity.ok(service.getBeaconData());
     }
 
-    @PostMapping("/send")
+    // This should be a POST, but to make it easier to hit, this is a GET
+    @GetMapping("/send")
     public ResponseEntity<?> forceSendBeaconData() {
         beaconReporter.sendBeaconData();
         return ResponseEntity.ok("Manual push of beacon data completed");

diff --git a/...ain/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataFiltersController.java b/...ain/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataFiltersController.java
@@ -1,12 +1,15 @@
 package edu.internet2.tier.shibboleth.admin.ui.controller;
 
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEvent;
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEventType;
 import edu.internet2.tier.shibboleth.admin.ui.domain.filters.ITargetable;
 import edu.internet2.tier.shibboleth.admin.ui.domain.filters.MetadataFilter;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.repository.FilterRepository;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
 import edu.internet2.tier.shibboleth.admin.ui.security.service.IGroupService;
 import edu.internet2.tier.shibboleth.admin.ui.security.service.UserService;
+import edu.internet2.tier.shibboleth.admin.ui.service.IBeaconDataService;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import io.swagger.v3.oas.annotations.tags.Tags;
@@ -46,6 +49,9 @@ public class MetadataFiltersController {
     private static final Supplier<HttpClientErrorException> HTTP_400_BAD_REQUEST_EXCEPTION = () -> new HttpClientErrorException(BAD_REQUEST);
     private static final Supplier<HttpClientErrorException> HTTP_404_CLIENT_ERROR_EXCEPTION = () -> new HttpClientErrorException(NOT_FOUND);
 
+    @Autowired
+    private IBeaconDataService beaconDataService;
+
     @Autowired
     org.opensaml.saml.metadata.resolver.MetadataResolver chainingMetadataResolver;
 
@@ -77,16 +83,13 @@ public ResponseEntity<?> create(@PathVariable String metadataResolverId, @Reques
         reloadFiltersAndHandleScriptException(persistedMr.getResourceId());
 
         MetadataFilter persistedFilter = newlyPersistedFilter(persistedMr.getMetadataFilters().stream(), createdFilter.getResourceId());
-
-        return ResponseEntity
-                        .created(getResourceUriFor(persistedMr, createdFilter.getResourceId()))
-                        .body(persistedFilter);
+        beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_FILTER_CREATED));
+        return ResponseEntity.created(getResourceUriFor(persistedMr, createdFilter.getResourceId())).body(persistedFilter);
     }
 
     @DeleteMapping("/Filters/{resourceId}")
     @Transactional
-    public ResponseEntity<?> delete(@PathVariable String metadataResolverId,
-                    @PathVariable String resourceId) {
+    public ResponseEntity<?> delete(@PathVariable String metadataResolverId, @PathVariable String resourceId) {
 
         MetadataResolver resolver = findResolverOrThrowHttp404(metadataResolverId);
         MetadataFilter filterToDelete = findFilterOrThrowHttp404(resourceId);
@@ -109,6 +112,7 @@ public ResponseEntity<?> delete(@PathVariable String metadataResolverId,
         //TODO: do we need to reload filters here?!?
         //metadataResolverService.reloadFilters(persistedMr.getName());
 
+        beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_FILTER_REMOVED));
         return ResponseEntity.noContent().build();
     }
 
@@ -188,9 +192,7 @@ private void reloadFiltersAndHandleScriptException(String resolverResourceId) {
 
     @PutMapping("/Filters/{resourceId}")
     @Transactional
-    public ResponseEntity<?> update(@PathVariable String metadataResolverId,
-                    @PathVariable String resourceId,
-                    @RequestBody MetadataFilter updatedFilter) {
+    public ResponseEntity<?> update(@PathVariable String metadataResolverId, @PathVariable String resourceId, @RequestBody MetadataFilter updatedFilter) {
 
         MetadataResolver metadataResolver = findResolverOrThrowHttp404(metadataResolverId);
 
@@ -228,7 +230,7 @@ public ResponseEntity<?> update(@PathVariable String metadataResolverId,
 
         // TODO: do we need to reload filters here?
         reloadFiltersAndHandleScriptException(metadataResolver.getResourceId());
-
+        beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_FILTER_MODIFIED));
         return ResponseEntity.ok().body(persistedFilter);
     }
 

diff --git a/...n/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversController.java b/...n/java/edu/internet2/tier/shibboleth/admin/ui/controller/MetadataResolversController.java
@@ -1,12 +1,15 @@
 package edu.internet2.tier.shibboleth.admin.ui.controller;
 
 import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEvent;
+import edu.internet2.tier.shibboleth.admin.ui.domain.beacon.BeaconEventType;
 import edu.internet2.tier.shibboleth.admin.ui.domain.exceptions.MetadataFileNotFoundException;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.MetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.opensaml.OpenSamlChainingMetadataResolver;
 import edu.internet2.tier.shibboleth.admin.ui.domain.resolvers.validator.MetadataResolverValidationService;
 import edu.internet2.tier.shibboleth.admin.ui.domain.versioning.Version;
 import edu.internet2.tier.shibboleth.admin.ui.repository.MetadataResolverRepository;
+import edu.internet2.tier.shibboleth.admin.ui.service.IBeaconDataService;
 import edu.internet2.tier.shibboleth.admin.ui.service.IndexWriterService;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverConverterService;
 import edu.internet2.tier.shibboleth.admin.ui.service.MetadataResolverService;
@@ -52,27 +55,29 @@
 @Slf4j
 @Tags(value = {@Tag(name = "metadata resolvers")})
 public class MetadataResolversController {
+    @Autowired
+    private IBeaconDataService beaconDataService;
 
     @Autowired
-    MetadataResolverRepository resolverRepository;
+    org.opensaml.saml.metadata.resolver.MetadataResolver chainingMetadataResolver;
 
     @Autowired
-    MetadataResolverValidationService metadataResolverValidationService;
+    IndexWriterService indexWriterService;
 
     @Autowired
-    MetadataResolverService metadataResolverService;
+    MetadataResolverConverterService metadataResolverConverterService;
 
     @Autowired
-    MetadataResolversPositionOrderContainerService positionOrderContainerService;
+    MetadataResolverService metadataResolverService;
 
     @Autowired
-    IndexWriterService indexWriterService;
+    MetadataResolverValidationService metadataResolverValidationService;
 
     @Autowired
-    org.opensaml.saml.metadata.resolver.MetadataResolver chainingMetadataResolver;
+    MetadataResolversPositionOrderContainerService positionOrderContainerService;
 
     @Autowired
-    MetadataResolverConverterService metadataResolverConverterService;
+    MetadataResolverRepository resolverRepository;
 
     @Autowired
     MetadataResolverVersionService versionService;
@@ -156,7 +161,8 @@ public ResponseEntity<?> create(@RequestBody MetadataResolver newResolver) throw
         MetadataResolver persistedResolver = resolverRepository.save(newResolver);
         positionOrderContainerService.appendPositionOrderForNew(persistedResolver);
         doResolverInitialization(persistedResolver);
-
+        beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_PROVIDER_CREATED));
+        newResolver.getMetadataFilters().forEach(f -> beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_FILTER_CREATED)));
         return ResponseEntity.created(getResourceUriFor(persistedResolver)).body(persistedResolver);
     }
 
@@ -168,8 +174,7 @@ public ResponseEntity<?> update(@PathVariable String resourceId, @RequestBody Me
             return ResponseEntity.notFound().build();
         }
         if (existingResolver.getVersion() != updatedResolver.getVersion()) {
-            log.info("Metadata Resolver version conflict. Latest resolver in database version: {}. Resolver version sent from UI: {}",
-                    existingResolver.getVersion(), updatedResolver.getVersion());
+            log.info("Metadata Resolver version conflict. Latest resolver in database version: {}. Resolver version sent from UI: {}", existingResolver.getVersion(), updatedResolver.getVersion());
             return ResponseEntity.status(HttpStatus.CONFLICT).build();
         }
 
@@ -182,6 +187,7 @@ public ResponseEntity<?> update(@PathVariable String resourceId, @RequestBody Me
         MetadataResolver persistedResolver = resolverRepository.save(updatedResolver);
         doResolverInitialization(persistedResolver);
 
+        beaconDataService.addBeaconEvent(new BeaconEvent(BeaconEventType.METADATA_PROVIDER_MODIFIED));
         return ResponseEntity.ok(resolverRepository.findByResourceId(resourceId));
     }
 

diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/BeaconConfiguration.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/BeaconConfiguration.java
@@ -6,6 +6,7 @@
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Transient;
+import javax.persistence.Version;
 
 @Data
 @Entity
@@ -17,6 +18,9 @@ public class BeaconConfiguration {
 
     private String sendCron;
 
+    @Version
+    private Long version;
+
     // Comma separated list of the auth mechanisms used.
     @Setter
     @Transient

diff --git a/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/beacon/BeaconEvent.java b/backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/beacon/BeaconEvent.java
@@ -0,0 +1,30 @@
+package edu.internet2.tier.shibboleth.admin.ui.domain.beacon;
+
+import lombok.Data;
+
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import java.util.Date;
+
+@Entity
+@Data
+public class BeaconEvent {
+    @Id
+    @GeneratedValue(strategy = GenerationType.SEQUENCE)
+    private Long id;
+
+    private Date eventTime;
+
+    private String eventName;
+
+    public BeaconEvent() {
+        eventTime = new Date();
+    }
+
+    public BeaconEvent(BeaconEventType eventType) {
+        eventTime = new Date();
+        this.eventName = eventType.name();
+    }
+}
diff --git a/...d/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/beacon/BeaconEventType.java b/...d/src/main/java/edu/internet2/tier/shibboleth/admin/ui/domain/beacon/BeaconEventType.java
@@ -0,0 +1,7 @@
+package edu.internet2.tier.shibboleth.admin.ui.domain.beacon;
+
+public enum BeaconEventType {
+    METADATA_SOURCE_MODIFIED, METADATA_SOURCE_REMOVED, METADATA_SOURCE_ENABLED, METADATA_SOURCE_CREATED,
+    METADATA_PROVIDER_MODIFIED, METADATA_PROVIDER_ENABLED, METADATA_PROVIDER_CREATED,
+    METADATA_FILTER_MODIFIED, METADATA_FILTER_REMOVED, METADATA_FILTER_ENABLED, METADATA_FILTER_CREATED
+}