Run GCP examples in a VM

tmiddelkoop · Nov 10, 2021 · 7ab2610 · 7ab2610
1 parent 774dd53
commit 7ab2610
Show file tree

Hide file tree

Showing 5 changed files with 104 additions and 48 deletions.
diff --git a/content/GCP/.gitignore b/content/GCP/.gitignore
@@ -0,0 +1 @@
+CLASS-Examples/
diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb
@@ -72,7 +72,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 1,
    "id": "05fadd30-4f35-4a93-b713-cc132131b948",
    "metadata": {},
    "outputs": [
@@ -83,18 +83,32 @@
       "Reading package lists... Done\n",
       "Building dependency tree       \n",
       "Reading state information... Done\n",
-      "git is already the newest version (1:2.20.1-2+deb10u3).\n",
-      "0 upgraded, 0 newly installed, 0 to remove and 7 not upgraded.\n"
+      "Suggested packages:\n",
+      "  git-daemon-run | git-daemon-sysvinit git-doc git-el git-email git-gui gitk\n",
+      "  gitweb git-cvs git-mediawiki git-svn\n",
+      "The following NEW packages will be installed:\n",
+      "  git\n",
+      "0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\n",
+      "Need to get 0 B/5633 kB of archives.\n",
+      "After this operation, 36.2 MB of additional disk space will be used.\n",
+      "debconf: unable to initialize frontend: Dialog\n",
+      "debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.)\n",
+      "debconf: falling back to frontend: Readline\n",
+      "Selecting previously unselected package git.\n",
+      "(Reading database ... 55447 files and directories currently installed.)\n",
+      "Preparing to unpack .../git_1%3a2.20.1-2+deb10u3_amd64.deb ...\n",
+      "Unpacking git (1:2.20.1-2+deb10u3) ...\n",
+      "Setting up git (1:2.20.1-2+deb10u3) ...\n"
      ]
     }
    ],
    "source": [
-    "sudo apt install --yes git"
+    "sudo apt-get install --yes git"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7",
    "metadata": {},
    "outputs": [
@@ -103,10 +117,10 @@
      "output_type": "stream",
      "text": [
       "Cloning into 'CLASS-Examples'...\n",
-      "remote: Enumerating objects: 16, done.\u001b[K\n",
-      "remote: Counting objects: 100% (16/16), done.\u001b[K\n",
-      "remote: Compressing objects: 100% (13/13), done.\u001b[K\n",
-      "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25\u001b[K\n",
+      "remote: Enumerating objects: 16, done.        \n",
+      "remote: Counting objects: 100% (16/16), done.        \n",
+      "remote: Compressing objects: 100% (13/13), done.        \n",
+      "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25        \n",
       "Unpacking objects: 100% (41/41), done.\n"
      ]
     }
@@ -117,7 +131,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 3,
    "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab",
    "metadata": {},
    "outputs": [],
@@ -127,7 +141,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 4,
    "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2",
    "metadata": {},
    "outputs": [
@@ -136,12 +150,12 @@
      "output_type": "stream",
      "text": [
       "total 24\n",
-      "-rw-r--r-- 1 student31 student31   72 Nov  9 20:18 clean.sh\n",
-      "-rw-r--r-- 1 student31 student31  256 Nov  9 20:18 download.sh\n",
-      "-rw-r--r-- 1 student31 student31  314 Nov  9 20:18 get-index.sh\n",
-      "-rw-r--r-- 1 student31 student31  841 Nov  9 20:18 ReadMe.md\n",
-      "-rw-r--r-- 1 student31 student31  110 Nov  9 20:18 search.json\n",
-      "-rw-r--r-- 1 student31 student31 1447 Nov  9 20:18 search.py\n"
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop  841 Nov  9 22:31 ReadMe.md\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop   72 Nov  9 22:31 clean.sh\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop  256 Nov  9 22:31 download.sh\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop  314 Nov  9 22:31 get-index.sh\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop  110 Nov  9 22:31 search.json\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 1447 Nov  9 22:31 search.py\n"
      ]
     }
    ],
@@ -161,7 +175,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd",
    "metadata": {},
    "outputs": [
@@ -198,7 +212,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e",
    "metadata": {},
    "outputs": [
@@ -216,7 +230,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 7,
    "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38",
    "metadata": {},
    "outputs": [
@@ -230,7 +244,7 @@
       "feature is enabled by default but requires that compiled crcmod be\n",
       "installed (see \"gsutil help crcmod\").\n",
       "\n",
-      "| [1 files][757.2 MiB/757.2 MiB]  737.1 KiB/s                                   \n",
+      "/ [1 files][757.2 MiB/757.2 MiB]                                                \n",
       "Operation completed over 1 objects/757.2 MiB.                                    \n"
      ]
     }
@@ -241,46 +255,32 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 8,
    "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "gzip: data/index.csv: No space left on device\n"
-     ]
-    },
-    {
-     "ename": "",
-     "evalue": "1",
-     "output_type": "error",
-     "traceback": []
-    }
-   ],
+   "outputs": [],
    "source": [
     "gzip -d data/index.csv.gz"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "b005876c-f9af-43d6-80c6-f04295413b9b",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total 2.6G\n",
+      "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 2.6G Nov  9 22:31 index.csv\n"
+     ]
+    }
+   ],
    "source": [
     "ls -lh data"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bbb26cfb-4b0d-4458-bc24-13a069556c3a",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

diff --git a/content/GCP/clean.sh b/content/GCP/clean.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "=== clean.sh"
+
+## Cleanup for 06_running_analysis
+sudo apt remove git
+rm -rf ./CLASS-Examples
diff --git a/scripts/gcp-dev.sh b/scripts/gcp-dev.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Options
+BRANCH="${1:-gcp-dev}" # checkout branch $1
+
+# Static Config - update gcp-*.sh files
+VM=essentials
+ZONE=us-west2-c
+PROJECT=CLASS-Essentials
+REPO="git@github.internet2.edu:CLASS/${PROJECT}.git"
+
+echo "=== gcp-dev.sh $PROJECT $BRANCH"
+
+echo "+++ creating VM"
+gcloud compute instances create --zone=$ZONE $VM --machine-type=e2-highcpu-4 --preemptible
+
+
+echo "+++ wait for boot"
+while ! gcloud compute ssh --zone=$ZONE $VM --command='hostname' ; do
+    sleep 1
+done
+
+echo "+++ configuring VM"
+
+gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-A' <<EOF
+sudo apt install --yes git python3-pip python3-venv
+ssh-keyscan github.internet2.edu > .ssh/known_hosts
+git config --global color.ui auto
+git config --global push.default simple
+git config --global pull.ff only
+git clone --branch $BRANCH $REPO
+cd $PROJECT
+git config user.name "$(git config user.name)"
+git config user.email "$(git config user.name)"
+EOF
+
+echo "+++ configure local ssh"
+gcloud compute config-ssh
+
+echo "+++ starting Jypter"
+gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-L 8080:localhost:8080 -L 8081:localhost:8081' --command="cd $PROJECT ; ./scripts/jupyter-lab.sh"
diff --git a/scripts/gcp-stop.sh b/scripts/gcp-stop.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+## User settings in gcp-dev.sh
+VM=essentials
+ZONE=us-west2-c
+
+gcloud --quiet compute instances delete --zone=$ZONE $VM