Merge singularity build and refactor controller setup

* Increase the size and add ssd to the controller node for builds
* Fix Spack build not to utilize slurm (did not speed up!).
* Move all builds to controller.

keycloak/keycloak.yaml

@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: keycloak
+  labels:
+    app: keycloak
+spec:
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+  selector:
+    app: keycloak
+  type: LoadBalancer
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: keycloak
+  namespace: default
+  labels:
+    app: keycloak
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: keycloak
+  template:
+    metadata:
+      labels:
+        app: keycloak
+    spec:
+      containers:
+      - name: keycloak
+        image: quay.io/keycloak/keycloak:13.0.0
+        env:
+        - name: KEYCLOAK_USER
+          value: "admin"
+        - name: KEYCLOAK_PASSWORD
+          value: "classadmin"
+        - name: PROXY_ADDRESS_FORWARDING
+          value: "true"
+        ports:
+        - name: http
+          containerPort: 8080
+        - name: https
+          containerPort: 8443
+        readinessProbe:
+          httpGet:
+            path: /auth/realms/master
+            port: 8080

slurm-gcp/basic.tfvars

@@ -11,9 +11,9 @@ disable_login_public_ips      = false
 
 # suspend_time  = 300
 
-controller_machine_type = "n1-standard-2"
+controller_machine_type = "c2-standard-8"
 controller_image        = "projects/schedmd-slurm-public/global/images/family/schedmd-slurm-20-11-4-hpc-centos-7"
-controller_disk_type    = "pd-standard"
+controller_disk_type    = "pd-ssd"
 controller_disk_size_gb = 50
 # controller_labels = {
 #   key1 = "val1"
@@ -83,7 +83,7 @@ partitions = [
   { name                 = "debug"
     machine_type         = "c2-standard-4"
     static_node_count    = 0
-    max_node_count       = 5
+    max_node_count       = 4
     zone                 = "us-central1-c"
     image                = "projects/schedmd-slurm-public/global/images/family/schedmd-slurm-20-11-4-hpc-centos-7"
     image_hyperthreads   = false

slurm-gcp/scripts/controller.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 # Run on class-controller node.
-# gcloud compute ssh class-controller -- < setup.sh
+# gcloud compute ssh class-controller -- < controller.sh
 
-echo "=== setup.sh $(hostname) $(date)"
+echo "=== controller.sh $(hostname) $(date)"
 
 # TODO: wait for slurm to configure
 # MOTD displays:
@@ -21,27 +21,4 @@ gcloud container clusters get-credentials --zone=$ZONE $K8S
 
 kubectl get nodes
 
-echo "+++ spack"
-sudo yum install -y epel-release
-sudo yum install -y git python3 ca-certificates procps curl unzip jq
-sudo yum install -y gcc gcc-c++ make patch file bzip2 xz
-
-sudo install -dv /apps/spack
-sudo chown $USER /apps/spack
-git clone --depth=1 https://github.com/spack/spack.git /apps/spack
-
-
-NODES=$(sinfo -p debug -h -Onodes)
-CPUS=$(sinfo -p debug -h -Ocpus)
-
-cat > build-spack.sh <<EOF
-#!/bin/bash
-echo "+++ \$(hostname) \$(date) \$SLURM_JOB_ID CPUS:\$SLURM_CPU_PER_TASK"
-. /apps/spack/share/spack/setup-env.sh
-srun -n $NODES --exclusive spack install lmod python py-pip miniconda3
-EOF
-sbatch -N $NODES --exclusive ./build-spack.sh
-
-# for I in class-compute-0-{0..7} ; do echo $I ; ssh -o stricthostkeychecking=no $I ps -fe \| grep spack ; done
-
-echo "=== setup.sh $(hostname) $(date) done"
+echo "=== controller.sh $(hostname) $(date) done"

install_singularity.sh → slurm-gcp/scripts/singularity.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-yum  groupinstall -y 'Development Tools'
+yum groupinstall -y 'Development Tools'
 yum install -y \
      openssl-devel \
      libuuid-devel \
@@ -12,8 +12,8 @@ yum install -y \
 export GOLANG_VERSION=1.16.4
 export SINGULARITY_VERSION=3.7.0
 
-mkdir -p /opt/go/${GOLANG_VERSION}
-mkdir -p /opt/singularity/${SINGULARITY_VERSION}
+install -dvp /opt/go/${GOLANG_VERSION}
+install -dvp /opt/singularity/${SINGULARITY_VERSION}
 
 export OS=linux
 export ARCH=amd64

slurm-gcp/scripts/spack.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "=== spack.sh $(hostname) $(date)"
+
+sudo yum install -y epel-release
+sudo yum install -y git python3 ca-certificates procps curl unzip jq
+sudo yum install -y gcc gcc-c++ make patch file bzip2 xz
+
+sudo install -dv /apps/spack
+sudo chown $USER /apps/spack
+git clone --depth=1 https://github.com/spack/spack.git /apps/spack
+
+. /apps/spack/share/spack/setup-env.sh
+spack install lmod python py-pip miniconda3
+
+echo "=== spack.sh $(hostname) $(date) done"

slurm-gcp/setup.sh

@@ -1,3 +1,6 @@
 #!/bin/bash
 
 gcloud compute ssh class-controller -- < scripts/controller.sh
+
+gcloud compute ssh class-controller -- sudo -i < scripts/singularity.sh
+gcloud compute ssh class-controller -- < scripts/spack.sh