diff --git a/content/GCP/.gitignore b/content/GCP/.gitignore new file mode 100644 index 0000000..f71307d --- /dev/null +++ b/content/GCP/.gitignore @@ -0,0 +1 @@ +CLASS-Examples/ diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index 05428b8..29291a2 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "05fadd30-4f35-4a93-b713-cc132131b948", "metadata": {}, "outputs": [ @@ -83,18 +83,32 @@ "Reading package lists... Done\n", "Building dependency tree \n", "Reading state information... Done\n", - "git is already the newest version (1:2.20.1-2+deb10u3).\n", - "0 upgraded, 0 newly installed, 0 to remove and 7 not upgraded.\n" + "Suggested packages:\n", + " git-daemon-run | git-daemon-sysvinit git-doc git-el git-email git-gui gitk\n", + " gitweb git-cvs git-mediawiki git-svn\n", + "The following NEW packages will be installed:\n", + " git\n", + "0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\n", + "Need to get 0 B/5633 kB of archives.\n", + "After this operation, 36.2 MB of additional disk space will be used.\n", + "debconf: unable to initialize frontend: Dialog\n", + "debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.)\n", + "debconf: falling back to frontend: Readline\n", + "Selecting previously unselected package git.\n", + "(Reading database ... 55447 files and directories currently installed.)\n", + "Preparing to unpack .../git_1%3a2.20.1-2+deb10u3_amd64.deb ...\n", + "Unpacking git (1:2.20.1-2+deb10u3) ...\n", + "Setting up git (1:2.20.1-2+deb10u3) ...\n" ] } ], "source": [ - "sudo apt install --yes git" + "sudo apt-get install --yes git" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7", "metadata": {}, "outputs": [ @@ -103,10 +117,10 @@ "output_type": "stream", "text": [ "Cloning into 'CLASS-Examples'...\n", - "remote: Enumerating objects: 16, done.\u001b[K\n", - "remote: Counting objects: 100% (16/16), done.\u001b[K\n", - "remote: Compressing objects: 100% (13/13), done.\u001b[K\n", - "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25\u001b[K\n", + "remote: Enumerating objects: 16, done. \n", + "remote: Counting objects: 100% (16/16), done. \n", + "remote: Compressing objects: 100% (13/13), done. \n", + "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25 \n", "Unpacking objects: 100% (41/41), done.\n" ] } @@ -117,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab", "metadata": {}, "outputs": [], @@ -127,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2", "metadata": {}, "outputs": [ @@ -136,12 +150,12 @@ "output_type": "stream", "text": [ "total 24\n", - "-rw-r--r-- 1 student31 student31 72 Nov 9 20:18 clean.sh\n", - "-rw-r--r-- 1 student31 student31 256 Nov 9 20:18 download.sh\n", - "-rw-r--r-- 1 student31 student31 314 Nov 9 20:18 get-index.sh\n", - "-rw-r--r-- 1 student31 student31 841 Nov 9 20:18 ReadMe.md\n", - "-rw-r--r-- 1 student31 student31 110 Nov 9 20:18 search.json\n", - "-rw-r--r-- 1 student31 student31 1447 Nov 9 20:18 search.py\n" + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 841 Nov 9 22:31 ReadMe.md\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 72 Nov 9 22:31 clean.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 256 Nov 9 22:31 download.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 314 Nov 9 22:31 get-index.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 110 Nov 9 22:31 search.json\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 1447 Nov 9 22:31 search.py\n" ] } ], @@ -161,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd", "metadata": {}, "outputs": [ @@ -198,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e", "metadata": {}, "outputs": [ @@ -216,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38", "metadata": {}, "outputs": [ @@ -230,7 +244,7 @@ "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "| [1 files][757.2 MiB/757.2 MiB] 737.1 KiB/s \n", + "/ [1 files][757.2 MiB/757.2 MiB] \n", "Operation completed over 1 objects/757.2 MiB. \n" ] } @@ -241,46 +255,32 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "gzip: data/index.csv: No space left on device\n" - ] - }, - { - "ename": "", - "evalue": "1", - "output_type": "error", - "traceback": [] - } - ], + "outputs": [], "source": [ "gzip -d data/index.csv.gz" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "b005876c-f9af-43d6-80c6-f04295413b9b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 2.6G\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 2.6G Nov 9 22:31 index.csv\n" + ] + } + ], "source": [ "ls -lh data" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbb26cfb-4b0d-4458-bc24-13a069556c3a", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/content/GCP/clean.sh b/content/GCP/clean.sh new file mode 100755 index 0000000..454a326 --- /dev/null +++ b/content/GCP/clean.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "=== clean.sh" + +## Cleanup for 06_running_analysis +sudo apt remove git +rm -rf ./CLASS-Examples diff --git a/scripts/gcp-dev.sh b/scripts/gcp-dev.sh new file mode 100755 index 0000000..f64a96c --- /dev/null +++ b/scripts/gcp-dev.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Options +BRANCH="${1:-gcp-dev}" # checkout branch $1 + +# Static Config - update gcp-*.sh files +VM=essentials +ZONE=us-west2-c +PROJECT=CLASS-Essentials +REPO="git@github.internet2.edu:CLASS/${PROJECT}.git" + +echo "=== gcp-dev.sh $PROJECT $BRANCH" + +echo "+++ creating VM" +gcloud compute instances create --zone=$ZONE $VM --machine-type=e2-highcpu-4 --preemptible + + +echo "+++ wait for boot" +while ! gcloud compute ssh --zone=$ZONE $VM --command='hostname' ; do + sleep 1 +done + +echo "+++ configuring VM" + +gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-A' < .ssh/known_hosts +git config --global color.ui auto +git config --global push.default simple +git config --global pull.ff only +git clone --branch $BRANCH $REPO +cd $PROJECT +git config user.name "$(git config user.name)" +git config user.email "$(git config user.name)" +EOF + +echo "+++ configure local ssh" +gcloud compute config-ssh + +echo "+++ starting Jypter" +gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-L 8080:localhost:8080 -L 8081:localhost:8081' --command="cd $PROJECT ; ./scripts/jupyter-lab.sh" diff --git a/scripts/gcp-stop.sh b/scripts/gcp-stop.sh new file mode 100755 index 0000000..7237119 --- /dev/null +++ b/scripts/gcp-stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +## User settings in gcp-dev.sh +VM=essentials +ZONE=us-west2-c + +gcloud --quiet compute instances delete --zone=$ZONE $VM