From d0b9cc2e2c7495a26c43a1c34ce56ae226b5d03e Mon Sep 17 00:00:00 2001
From: Timothy Middelkoop <tmiddelkoop@internet2.edu>
Date: Tue, 11 May 2021 12:23:41 -0500
Subject: [PATCH] Merge singularity build and refactor controller setup

* Increase the size and add ssd to the controller node for builds
* Fix Spack build not to utilize slurm (did not speed up!).
* Move all builds to controller.
---
 keycloak/keycloak.yaml                        | 51 +++++++++++++++++++
 slurm-gcp/basic.tfvars                        |  6 +--
 slurm-gcp/scripts/controller.sh               | 29 ++---------
 .../scripts/singularity.sh                    |  6 +--
 slurm-gcp/scripts/spack.sh                    | 16 ++++++
 slurm-gcp/setup.sh                            |  3 ++
 6 files changed, 79 insertions(+), 32 deletions(-)
 create mode 100644 keycloak/keycloak.yaml
 rename install_singularity.sh => slurm-gcp/scripts/singularity.sh (93%)
 create mode 100644 slurm-gcp/scripts/spack.sh

diff --git a/keycloak/keycloak.yaml b/keycloak/keycloak.yaml
new file mode 100644
index 0000000..fd9b3f3
--- /dev/null
+++ b/keycloak/keycloak.yaml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: keycloak
+  labels:
+    app: keycloak
+spec:
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: keycloak
+  type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: keycloak
+  namespace: default
+  labels:
+    app: keycloak
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: keycloak
+  template:
+    metadata:
+      labels:
+        app: keycloak
+    spec:
+      containers:
+      - name: keycloak
+        image: quay.io/keycloak/keycloak:13.0.0
+        env:
+        - name: KEYCLOAK_USER
+          value: "admin"
+        - name: KEYCLOAK_PASSWORD
+          value: "classadmin"
+        - name: PROXY_ADDRESS_FORWARDING
+          value: "true"
+        ports:
+        - name: http
+          containerPort: 8080
+        - name: https
+          containerPort: 8443
+        readinessProbe:
+          httpGet:
+            path: /auth/realms/master
+            port: 8080
diff --git a/slurm-gcp/basic.tfvars b/slurm-gcp/basic.tfvars
index 41e8050..a031872 100644
--- a/slurm-gcp/basic.tfvars
+++ b/slurm-gcp/basic.tfvars
@@ -11,9 +11,9 @@ disable_login_public_ips      = false
 
 # suspend_time  = 300
 
-controller_machine_type = "n1-standard-2"
+controller_machine_type = "c2-standard-8"
 controller_image        = "projects/schedmd-slurm-public/global/images/family/schedmd-slurm-20-11-4-hpc-centos-7"
-controller_disk_type    = "pd-standard"
+controller_disk_type    = "pd-ssd"
 controller_disk_size_gb = 50
 # controller_labels = {
 #   key1 = "val1"
@@ -83,7 +83,7 @@ partitions = [
   { name                 = "debug"
     machine_type         = "c2-standard-4"
     static_node_count    = 0
-    max_node_count       = 5
+    max_node_count       = 4
     zone                 = "us-central1-c"
     image                = "projects/schedmd-slurm-public/global/images/family/schedmd-slurm-20-11-4-hpc-centos-7"
     image_hyperthreads   = false
diff --git a/slurm-gcp/scripts/controller.sh b/slurm-gcp/scripts/controller.sh
index a8eed2d..dd681d9 100644
--- a/slurm-gcp/scripts/controller.sh
+++ b/slurm-gcp/scripts/controller.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Run on class-controller node.
-# gcloud compute ssh class-controller -- < setup.sh
+# gcloud compute ssh class-controller -- < controller.sh
 
-echo "=== setup.sh $(hostname) $(date)"
+echo "=== controller.sh $(hostname) $(date)"
 
 # TODO: wait for slurm to configure
 # MOTD displays:
@@ -21,27 +21,4 @@ gcloud container clusters get-credentials --zone=$ZONE $K8S
 
 kubectl get nodes
 
-echo "+++ spack"
-sudo yum install -y epel-release
-sudo yum install -y git python3 ca-certificates procps curl unzip jq
-sudo yum install -y gcc gcc-c++ make patch file bzip2 xz
-
-sudo install -dv /apps/spack
-sudo chown $USER /apps/spack
-git clone --depth=1 https://github.com/spack/spack.git /apps/spack
-
-
-NODES=$(sinfo -p debug -h -Onodes)
-CPUS=$(sinfo -p debug -h -Ocpus)
-
-cat > build-spack.sh <<EOF
-#!/bin/bash
-echo "+++ \$(hostname) \$(date) \$SLURM_JOB_ID CPUS:\$SLURM_CPU_PER_TASK"
-. /apps/spack/share/spack/setup-env.sh
-srun -n $NODES --exclusive spack install lmod python py-pip miniconda3
-EOF
-sbatch -N $NODES --exclusive ./build-spack.sh
-
-# for I in class-compute-0-{0..7} ; do echo $I ; ssh -o stricthostkeychecking=no $I ps -fe \| grep spack ; done
-
-echo "=== setup.sh $(hostname) $(date) done"
+echo "=== controller.sh $(hostname) $(date) done"
diff --git a/install_singularity.sh b/slurm-gcp/scripts/singularity.sh
similarity index 93%
rename from install_singularity.sh
rename to slurm-gcp/scripts/singularity.sh
index 9164ec0..d7e2cfa 100644
--- a/install_singularity.sh
+++ b/slurm-gcp/scripts/singularity.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-yum  groupinstall -y 'Development Tools'
+yum groupinstall -y 'Development Tools'
 yum install -y \
      openssl-devel \
      libuuid-devel \
@@ -12,8 +12,8 @@ yum install -y \
 export GOLANG_VERSION=1.16.4
 export SINGULARITY_VERSION=3.7.0
 
-mkdir -p /opt/go/${GOLANG_VERSION}
-mkdir -p /opt/singularity/${SINGULARITY_VERSION}
+install -dvp /opt/go/${GOLANG_VERSION}
+install -dvp /opt/singularity/${SINGULARITY_VERSION}
 
 export OS=linux
 export ARCH=amd64
diff --git a/slurm-gcp/scripts/spack.sh b/slurm-gcp/scripts/spack.sh
new file mode 100644
index 0000000..666ebdf
--- /dev/null
+++ b/slurm-gcp/scripts/spack.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "=== spack.sh $(hostname) $(date)"
+
+sudo yum install -y epel-release
+sudo yum install -y git python3 ca-certificates procps curl unzip jq
+sudo yum install -y gcc gcc-c++ make patch file bzip2 xz
+
+sudo install -dv /apps/spack
+sudo chown $USER /apps/spack
+git clone --depth=1 https://github.com/spack/spack.git /apps/spack
+
+. /apps/spack/share/spack/setup-env.sh
+spack install lmod python py-pip miniconda3
+
+echo "=== spack.sh $(hostname) $(date) done"
diff --git a/slurm-gcp/setup.sh b/slurm-gcp/setup.sh
index 4803b4e..330adfa 100755
--- a/slurm-gcp/setup.sh
+++ b/slurm-gcp/setup.sh
@@ -1,3 +1,6 @@
 #!/bin/bash
 
 gcloud compute ssh class-controller -- < scripts/controller.sh
+
+gcloud compute ssh class-controller -- sudo -i < scripts/singularity.sh
+gcloud compute ssh class-controller -- < scripts/spack.sh