Merged in SHIBUI-1707-a (pull request #458)

SHIBUI-1707-a Beacon module implementation

Approved-by: Jonathan Johnson

backend/build.gradle

@@ -197,6 +197,9 @@ dependencies {
     //Pacj4 sub-project
     runtimeOnly project(':pac4j-module')
 
+    //Beacon
+    runtimeOnly project(':beacon:spring')
+
     enversTestCompile sourceSets.main.output
     enversTestCompile sourceSets.test.output
     enversTestCompile configurations.compile

backend/src/main/java/edu/internet2/tier/shibboleth/admin/ui/ShibbolethUiApplication.java

@@ -16,6 +16,7 @@
 import org.springframework.context.annotation.Profile;
 import org.springframework.context.event.EventListener;
 import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
+import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
 import org.springframework.stereotype.Component;
@@ -29,6 +30,7 @@
 @EnableJpaAuditing
 @EnableScheduling
 @EnableWebSecurity
+@EnableAsync
 public class ShibbolethUiApplication extends SpringBootServletInitializer {
 
     private static final Logger logger = LoggerFactory.getLogger(ShibbolethUiApplication.class);

backend/src/main/resources/application.properties

@@ -89,3 +89,8 @@ shibui.roles=ROLE_ADMIN,ROLE_USER,ROLE_NONE
 #This property must be set to true and pac4j properties configured. For sample pac4j properties, see application.yml
 #for an example pac4j configuration
 #shibui.pac4j-enabled=true
+
+#This property must be set to true in order to enable posting stats to beacon endpoint. Furthermore, appropriate
+#environment variables must be set for beacon publisher to be used (the ones that are set when running shib-ui in
+#docker container
+#shibui.beacon-enabled=true

beacon/build.gradle

@@ -0,0 +1,4 @@
+allprojects {
+    group = 'edu.internet2.tap.beacon'
+    version = '1.0.0-SNAPSHOT'
+}

beacon/core/build.gradle

@@ -0,0 +1,33 @@
+import org.springframework.boot.gradle.plugin.SpringBootPlugin
+
+plugins {
+    id 'org.springframework.boot' version '2.0.0.RELEASE' apply false
+    id 'io.spring.dependency-management' version '1.0.6.RELEASE'
+}
+
+apply plugin: 'groovy'
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+    jcenter()
+}
+
+dependencyManagement {
+    imports {
+        mavenBom SpringBootPlugin.BOM_COORDINATES
+    }
+}
+
+dependencies {
+    testCompile "org.springframework.boot:spring-boot-starter-test"
+    testCompile "org.spockframework:spock-core:1.1-groovy-2.4"
+    testCompile "org.spockframework:spock-spring:1.1-groovy-2.4"
+
+    testCompile 'org.junit.jupiter:junit-jupiter-api:5.5.2'
+    testCompile 'org.junit.jupiter:junit-jupiter-engine:5.5.2'
+}
+
+jar {
+    archiveName = "beacon-core-${version}.jar"
+}

beacon/core/src/main/java/edu/internet2/tap/beacon/Beacon.java

@@ -0,0 +1,22 @@
+package edu.internet2.tap.beacon;
+
+/**
+ * Exposes expected names of environment variables holding beacon config data.
+ */
+public final class Beacon {
+
+    private Beacon() {
+    }
+
+    public static final String LOG_HOST = "LOGHOST";
+
+    public static final String LOG_PORT = "LOGPORT";
+
+    public static final String IMAGE = "IMAGE";
+
+    public static final String VERSION = "VERSION";
+
+    public static final String TIERVERSION = "TIERVERSION";
+
+    public static final String MAINTAINER = "MAINTAINER";
+}

beacon/core/src/main/java/edu/internet2/tap/beacon/BeaconPublisher.java

@@ -0,0 +1,10 @@
+package edu.internet2.tap.beacon;
+
+/**
+ * Simple SPI allowing implementations to publish to beacon service utilizing Runnable API
+ * so that publishing code could run in separate threads of execution.
+ *
+ * @author Dmitriy Kopylenko
+ */
+public interface BeaconPublisher extends Runnable {
+}

beacon/core/src/main/java/edu/internet2/tap/beacon/DefaultBeaconPublisher.java

@@ -0,0 +1,90 @@
+package edu.internet2.tap.beacon;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static edu.internet2.tap.beacon.Beacon.IMAGE;
+import static edu.internet2.tap.beacon.Beacon.LOG_HOST;
+import static edu.internet2.tap.beacon.Beacon.LOG_PORT;
+import static edu.internet2.tap.beacon.Beacon.MAINTAINER;
+import static edu.internet2.tap.beacon.Beacon.TIERVERSION;
+import static edu.internet2.tap.beacon.Beacon.VERSION;
+
+/**
+ * Default implementation that knows the details about payload structure with its data and beacon endpoint details
+ * gathered by upstream components and passed to this implementation at object construction site.
+ *
+ * @author Dmitriy Kopylenko
+ */
+public class DefaultBeaconPublisher implements BeaconPublisher {
+
+    private final URL endpointUrl;
+
+    private final String jsonPayload;
+
+    public DefaultBeaconPublisher(Map<String, String> beaconDetails) {
+        //Do data validation checks here. If any of the necessary beacon data not available here, throw a Runtime exception
+        if (beaconDetails == null) {
+            throw new IllegalArgumentException("beaconDetails Map must not be null");
+        }
+        if (beaconDetails.get(LOG_HOST) == null
+                || beaconDetails.get(LOG_PORT) == null
+                || beaconDetails.get(IMAGE) == null
+                || beaconDetails.get(VERSION) == null
+                || beaconDetails.get(TIERVERSION) == null
+                || beaconDetails.get(MAINTAINER) == null) {
+            throw new IllegalArgumentException("Not all the necessary beacon data is available to be able to publish to beacon");
+        }
+        try {
+            this.endpointUrl = new URL(String.format("http://%s:%s", beaconDetails.get(LOG_HOST), beaconDetails.get(LOG_PORT)));
+        } catch (MalformedURLException ex) {
+            throw new IllegalArgumentException(ex.getMessage());
+        }
+
+        Map<String, String> dataToPublish = new HashMap<>();
+        dataToPublish.put("msgType", "TIERBEACON");
+        dataToPublish.put("msgName", "TIER");
+        dataToPublish.put("msgVersion", "1.0");
+        dataToPublish.put("tbProduct", beaconDetails.get(IMAGE));
+        dataToPublish.put("tbProductVersion", beaconDetails.get(VERSION));
+        dataToPublish.put("tbTIERRelease", beaconDetails.get(TIERVERSION));
+        dataToPublish.put("tbMaintainer", beaconDetails.get(MAINTAINER));
+
+        //Create JSON payload without any 3-rd party library
+        this.jsonPayload = "{" + dataToPublish.entrySet().stream()
+                .map(e -> "\"" + e.getKey() + "\"" + ":\"" + e.getValue() + "\"")
+                .collect(Collectors.joining(", ")) + "}";
+    }
+
+    @Override
+    public void run() {
+        try {
+            HttpURLConnection con = (HttpURLConnection) this.endpointUrl.openConnection();
+            con.setRequestMethod("POST");
+            con.setRequestProperty("Content-Type", "application/json; utf-8");
+            con.setRequestProperty("Accept", "application/json");
+            con.setDoOutput(true);
+            try(OutputStream os = con.getOutputStream()){
+                byte[] input = jsonPayload.getBytes("utf-8");
+                os.write(input, 0, input.length);
+            }
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    //getters used in unit tests and calling components for debugging purposes
+    public String getEndpointUri() {
+        return endpointUrl.toString();
+    }
+
+    public String getJsonPayload() {
+        return jsonPayload;
+    }
+}

beacon/core/src/test/groovy/edu/internet2/tap/beacon/DefaultBeaconPublisherTests.groovy

@@ -0,0 +1,45 @@
+package edu.internet2.tap.beacon
+
+import spock.lang.Specification
+import sun.security.x509.OtherName
+
+class DefaultBeaconPublisherTests extends Specification {
+
+    def "DefaultBeaconPublisher invariants are enforced during object creation - null Map is passed"() {
+        when:
+        new DefaultBeaconPublisher(null)
+
+        then:
+        thrown IllegalArgumentException
+
+    }
+
+    def "DefaultBeaconPublisher invariants are enforced during object creation - empty Map is passed"() {
+        when:
+        new DefaultBeaconPublisher([:])
+
+        then:
+        thrown IllegalArgumentException
+    }
+
+    def "DefaultBeaconPublisher invariants are enforced during object creation - valid Beacon data Map is passed"() {
+        when:
+        def expectedJsonPaylaod = """{"msgType":"TIERBEACON", "tbMaintainer":"unittest_maintainer", "msgName":"TIER", "tbProduct":"image", "msgVersion":"1.0", "tbProductVersion":"v1", "tbTIERRelease":"tv1"}"""
+
+        def configuredBeaconData = [LOGHOST   : 'collector.testbed.tier.internet2.edu',
+                                    LOGPORT   : '5001',
+                                    IMAGE      : 'image',
+                                    VERSION    : 'v1',
+                                    TIERVERSION: 'tv1',
+                                    MAINTAINER : 'unittest_maintainer']
+        def p = new DefaultBeaconPublisher(configuredBeaconData)
+        println p.jsonPayload
+
+        then:
+        noExceptionThrown()
+        p.endpointUri == 'http://collector.testbed.tier.internet2.edu:5001'
+        p.jsonPayload == expectedJsonPaylaod
+
+    }
+
+}

beacon/spring/build.gradle

@@ -0,0 +1,29 @@
+import org.springframework.boot.gradle.plugin.SpringBootPlugin
+
+plugins {
+    id 'org.springframework.boot' version '2.0.0.RELEASE' apply false
+    id 'io.spring.dependency-management' version '1.0.6.RELEASE'
+}
+
+apply plugin: 'java'
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+    jcenter()
+}
+
+jar {
+    archiveName = "beacon-spring-${version}.jar"
+}
+
+dependencyManagement {
+    imports {
+        mavenBom SpringBootPlugin.BOM_COORDINATES
+    }
+}
+
+dependencies {
+    compile project(':beacon:core')
+    compile "org.springframework.boot:spring-boot-starter"
+}

beacon/spring/src/main/java/edu/internet2/tap/beacon/configuration/BeaconPublishingConfiguration.java

@@ -0,0 +1,61 @@
+package edu.internet2.tap.beacon.configuration;
+
+import edu.internet2.tap.beacon.DefaultBeaconPublisher;
+import edu.internet2.tap.beacon.configuration.condition.ConditionalOnBeaconEnvironmentVariablesPresent;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.Environment;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.Scheduled;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static edu.internet2.tap.beacon.Beacon.IMAGE;
+import static edu.internet2.tap.beacon.Beacon.LOG_HOST;
+import static edu.internet2.tap.beacon.Beacon.LOG_PORT;
+import static edu.internet2.tap.beacon.Beacon.MAINTAINER;
+import static edu.internet2.tap.beacon.Beacon.TIERVERSION;
+import static edu.internet2.tap.beacon.Beacon.VERSION;
+
+@Configuration
+@ConditionalOnProperty(name = "shibui.beacon-enabled", havingValue = "true")
+public class BeaconPublishingConfiguration {
+
+    private static final Logger logger = LoggerFactory.getLogger(BeaconPublishingConfiguration.class);
+
+    @Bean
+    @ConditionalOnBeaconEnvironmentVariablesPresent
+    public BeaconPublishingTask beaconPublisher(Environment env) {
+        logger.debug("Creating BeaconPublishingTask...");
+        Map<String, String> beaconData = new HashMap<>();
+        beaconData.put(LOG_HOST, env.getProperty(LOG_HOST));
+        beaconData.put(LOG_PORT, env.getProperty(LOG_PORT));
+        beaconData.put(IMAGE, env.getProperty(IMAGE));
+        beaconData.put(VERSION, env.getProperty(VERSION));
+        beaconData.put(TIERVERSION, env.getProperty(TIERVERSION));
+        beaconData.put(MAINTAINER, env.getProperty(MAINTAINER));
+        return new BeaconPublishingTask(new DefaultBeaconPublisher(beaconData));
+    }
+
+    public static class BeaconPublishingTask {
+        private DefaultBeaconPublisher beaconPublisher;
+
+        public BeaconPublishingTask(DefaultBeaconPublisher beaconPublisher) {
+            this.beaconPublisher = beaconPublisher;
+        }
+
+        //Cron is based on the spec defined here: https://spaces.at.internet2.edu/display/TWGH/TIER+Instrumentation+-+The+TIER+Beacon
+        @Scheduled(cron = "0 ${random.int[0,59]} ${random.int[0,3]} ? * *}")
+        @Async
+        void publish() {
+            logger.debug("Publishing payload: {} to beacon endpoint: {}",
+                    beaconPublisher.getJsonPayload(),
+                    beaconPublisher.getEndpointUri());
+            beaconPublisher.run();
+        }
+    }
+}

beacon/spring/src/main/java/edu/internet2/tap/beacon/configuration/condition/BeaconEnvironmentVariablesCondition.java

@@ -0,0 +1,48 @@
+package edu.internet2.tap.beacon.configuration.condition;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.boot.autoconfigure.condition.ConditionOutcome;
+import org.springframework.boot.autoconfigure.condition.SpringBootCondition;
+import org.springframework.context.annotation.Condition;
+import org.springframework.context.annotation.ConditionContext;
+import org.springframework.core.env.Environment;
+import org.springframework.core.type.AnnotatedTypeMetadata;
+
+import static edu.internet2.tap.beacon.Beacon.IMAGE;
+import static edu.internet2.tap.beacon.Beacon.LOG_HOST;
+import static edu.internet2.tap.beacon.Beacon.LOG_PORT;
+import static edu.internet2.tap.beacon.Beacon.MAINTAINER;
+import static edu.internet2.tap.beacon.Beacon.TIERVERSION;
+import static edu.internet2.tap.beacon.Beacon.VERSION;
+
+/**
+ * {@link Condition} that checks for required beacon environment variables.
+ *
+ * @author Dmitriy Kopylenko
+ * @see ConditionalOnBeaconEnvironmentVariablesPresent
+ */
+public class BeaconEnvironmentVariablesCondition extends SpringBootCondition {
+
+    private static final String MATCHED_MSG = "Beacon properties are present. Beacon activation condition is matched.";
+
+    private static final String NOT_MATCHED_MSG = "Beacon properties are not present. Beacon activation condition is not matched.";
+
+    private static final Logger logger = LoggerFactory.getLogger(BeaconEnvironmentVariablesCondition.class);
+
+    @Override
+    public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata metadata) {
+        Environment env = context.getEnvironment();
+        if (env.containsProperty(LOG_HOST)
+                && env.containsProperty(LOG_PORT)
+                && env.containsProperty(IMAGE)
+                && env.containsProperty(VERSION)
+                && env.containsProperty(TIERVERSION)
+                && env.containsProperty(MAINTAINER)) {
+            logger.debug(MATCHED_MSG);
+            return ConditionOutcome.match(MATCHED_MSG);
+        }
+        logger.debug(NOT_MATCHED_MSG);
+        return ConditionOutcome.noMatch(NOT_MATCHED_MSG);
+    }
+}