From 10d6c3fc674a7977f64c4586dcd91bbd4cc418eb Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Mon, 8 Nov 2021 22:13:09 +0000 Subject: [PATCH 1/6] Fix heading levels (warnings) and TOC --- content/GCP/01_intro_to_cloud_console.ipynb | 21 ++++---- ..._part1.ipynb => 02_intro_to_compute.ipynb} | 19 ++++---- content/GCP/04_intro_to_cli.ipynb | 16 +++---- content/GCP/05_intro_to_cloud_storage.ipynb | 24 +++++----- content/GCP/06_running_analysis.ipynb | 48 ++++++------------- content/GCP/07_monitoring_costs.ipynb | 42 +++++++--------- content/GCP/08_cleaning_up_resources.ipynb | 48 ++++++------------- content/GCP/glossary.ipynb | 34 +++++++++++++ content/GCP/intro_to_GCP_Essentials.ipynb | 2 +- content/_toc.yml | 4 +- 10 files changed, 121 insertions(+), 137 deletions(-) rename content/GCP/{02_intro_to_compute_part1.ipynb => 02_intro_to_compute.ipynb} (97%) create mode 100644 content/GCP/glossary.ipynb diff --git a/content/GCP/01_intro_to_cloud_console.ipynb b/content/GCP/01_intro_to_cloud_console.ipynb index 2a50a63..b357627 100644 --- a/content/GCP/01_intro_to_cloud_console.ipynb +++ b/content/GCP/01_intro_to_cloud_console.ipynb @@ -7,7 +7,7 @@ "tags": [] }, "source": [ - "## Introduction to the GCP Cloud Console\n", + "# Introduction to the GCP Cloud Console\n", "\n", "Learner Questions\n", " * How do I find things in the web console?\n", @@ -22,9 +22,9 @@ "id": "3316c7f9-b0c8-4565-bdc6-7582c4f16127", "metadata": {}, "source": [ - "### The Who, What, and Where of the Cloud\n", + "## The Who, What, and Where of the Cloud\n", "\n", - "Before we get started we must first define, and find, a few key pieces of information about your environment that will be used throughout this lesson. This information is also important to ensure that this information is what you expect, otherwise you may use the wrong account or run or store things in the wrong place. It is important to be clear about these terms as they are often different in other cloud providers. We provide a [glossary of GCP terms](./glossary.ipynb) for your reference." + "Before we get started we must first define, and find, a few key pieces of information about your environment that will be used throughout this lesson. This information is also important to ensure that this information is what you expect, otherwise you may use the wrong account or run or store things in the wrong place. It is important to be clear about these terms as they are often different in other cloud providers. We provide a [glossary of GCP terms](glossary.ipynb) for your reference." ] }, { @@ -34,7 +34,7 @@ "tags": [] }, "source": [ - "### Accounts (Authentication and Identity)\n", + "## Accounts (Authentication and Identity)\n", "\n", "The (Google) **Account** is the identity used to access the Google Cloud Platform web console and is in the form of an email address and is associated with an individual (the *who* is *you*!). This is typically the account you use to access Google services associated with your institution and not your personal account. This is the \"Who\" of the cloud. We will now login to the Google Cloud Platform web console and verify our account.\n", "\n", @@ -54,7 +54,7 @@ "tags": [] }, "source": [ - "#### Exercise\n", + "### Exercise\n", "\n", "Take a few moments to ask yourself the following questions:\n", " * What account are you currently using in the Google Cloud web console?\n", @@ -69,7 +69,7 @@ "id": "5cfe3d13-c050-49ee-873b-be823fda5b10", "metadata": {}, "source": [ - "### Projects\n", + "## Projects\n", "\n", "Almost everything you will do within Google Cloud Platform must be associated with a **Project**. This is the \"Where\" of the cloud. You must have at least one Project and you can manage multiple Projects within a single (Google) Account. Always make sure you are working in the correct project!\n", "\n", @@ -87,7 +87,7 @@ "The active project is also shown next to the project icon (three hexagons) and clicking it brings up the project selection dialog. The current project has a check mark and and the active project can be changed by clicking on the project name or double clicking the row.\n", " ![select-project](img/select-project.png) \n", "\n", - "#### Exercise\n", + "### Exercise\n", "\n", "Take a few moments to ask yourself the following questions:\n", " * What is the *name* of the Project you are currently using in the Google Cloud Platform web console?\n", @@ -106,7 +106,7 @@ "tags": [] }, "source": [ - "### Services\n", + "## Services\n", "\n", "The web console is used to control and observe the cloud from the browser. It should only be used for simple and one-time tasks, exploring new services, accessing documentation, or for monitoring and debugging resources in the cloud. Programmatic control through the console, programming languages (for example Python), and other automation tools should be used for day to day activities to make the most out of the cloud and to help with the reproducibility of research and teaching.\n", "\n", @@ -126,11 +126,10 @@ "cell_type": "markdown", "id": "e955eb3a-4fdc-448b-aef8-ed1dbe2a73d9", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "### Everything Else\n", + "## Everything Else\n", "\n", "If you did not already notice, every page on the GCP web console has a blue navigation bar at the top of the page. This can be used to quickly navigate to the page you need.\n", "\n", @@ -159,7 +158,7 @@ "tags": [] }, "source": [ - "### Exercise\n", + "## Exercise\n", "\n", "Before we move on, do the following:\n", "* Double check that you are in the correct account and project that you will be using for this lesson.\n", diff --git a/content/GCP/02_intro_to_compute_part1.ipynb b/content/GCP/02_intro_to_compute.ipynb similarity index 97% rename from content/GCP/02_intro_to_compute_part1.ipynb rename to content/GCP/02_intro_to_compute.ipynb index 8ef2504..c8af305 100644 --- a/content/GCP/02_intro_to_compute_part1.ipynb +++ b/content/GCP/02_intro_to_compute.ipynb @@ -7,7 +7,7 @@ "tags": [] }, "source": [ - "## Introduction to Cloud Compute (Google Compute Engine)\n", + "# Introduction to Cloud Compute (Google Compute Engine)\n", "\n", "Learner Questions\n", " * How do I compute in the cloud?\n", @@ -36,7 +36,7 @@ "tags": [] }, "source": [ - "### Google Compute Engine\n", + "## Google Compute Engine\n", "\n", "To learn more about the Google Compute Engine service we will use the web console to create, explore, and destroy a Virtual Machine. The ability to quickly create, explore, and then discard resources in the web-console is a powerful tool to explore a service. After an initial exploration with the web-console, programmatic access using scripts or configuration management systems should be used.\n", "\n", @@ -52,7 +52,7 @@ "tags": [] }, "source": [ - "### Allocate a VM Instance\n", + "## Allocate a VM Instance\n", "\n", "We will first create an *VM instance* with defaults to explore the Google Compute Engine service.\n", "\n", @@ -77,7 +77,7 @@ "id": "41c63432-a614-4a1e-9967-f49b68f9069e", "metadata": {}, "source": [ - "### Security\n", + "## Security\n", "\n", "Everything in the cloud requires permission (authorization). Ordinary we would configure and check security first but in the case of exploring services it is often easier to do things out of order. We noted that the *VM instance* was created with the *Compute Engine default service account*, which by *default* has access to all the resources that your Google Account does.\n", "\n", @@ -97,7 +97,7 @@ "tags": [] }, "source": [ - "### Follow the Storage Allocation\n", + "## Follow the Storage Allocation\n", "\n", "Just as with security, we will audit (follow) the *VM instance* creation by examining at the project *activity*.\n", "\n", @@ -118,7 +118,7 @@ "tags": [] }, "source": [ - "### Enumerate the VM Instances\n", + "## Enumerate the VM Instances\n", "\n", "To see an overview of resources in the project navigate to the **dashboard** and you will now see at least **1 instance** in the *resources* card under *Compute Engine*.\n", "\n", @@ -135,7 +135,7 @@ "tags": [] }, "source": [ - "### Connect to the VM Instance\n", + "## Connect to the VM Instance\n", "\n", "To connect to the *VM Instance* we will use the cloud shell in the web console. Open up a shell by clicking on the **Activate Cloud Shell** icon in the top blue bar if it is not already open.\n", "\n", @@ -236,11 +236,10 @@ "cell_type": "markdown", "id": "0362e76c-3f03-4299-8217-24a2f39ee503", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "### Delete the Storage - Exercise \n", + "## Delete the Storage - Exercise \n", "\n", "We will now complete the resource life-cycle by deleting the *VM Instance* we just created. Think about the process we did for creating the bucket and follow a similar one. \n", "\n", @@ -254,7 +253,7 @@ "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", "metadata": {}, "source": [ - "### References\n", + "## References\n", " * Compute Engine Permissions (IAM): https://cloud.google.com/compute/docs/access/iam" ] } diff --git a/content/GCP/04_intro_to_cli.ipynb b/content/GCP/04_intro_to_cli.ipynb index f14e01e..9f8e713 100644 --- a/content/GCP/04_intro_to_cli.ipynb +++ b/content/GCP/04_intro_to_cli.ipynb @@ -5,7 +5,7 @@ "id": "5439e525-b985-495f-85a6-e4c8d7452956", "metadata": {}, "source": [ - "## Introduction to the gcloud CLI\n", + "# Introduction to the gcloud CLI\n", "\n", "Learner Questions\n", " * How do I use the Cloud CLI?\n", @@ -30,7 +30,7 @@ "id": "c4e1a4c5-7d28-483d-b27c-ef2afa4fb7e5", "metadata": {}, "source": [ - "### Cloud Automation\n", + "## Cloud Automation\n", "\n", "Now that Drew can create a bucket in the web console they wish to learn how to use tools to automate the process. Drew noticed that the web console could be used to upload and share objects but wishes to learn the command line so that they can automate the and document the process (it is much easier to document commands then describing and showing mouse clicks).\n", "\n", @@ -48,7 +48,7 @@ "id": "ce598878-1b1c-4e6b-9122-e679e84c7dbb", "metadata": {}, "source": [ - "### Open a Cloud Shell\n", + "## Open a Cloud Shell\n", "\n", "The Cloud Shell is an easy way to access Comand Line Interface (CLI) commands in a web browser. Under the hood Cloud Shell is Debian running in a container with 5GB of no-charge persistent storage with most of the Google uti and a lot of other useful tools (git, docker, jq, Docker, etc.) installed by default.\n", "\n", @@ -60,12 +60,12 @@ " * Note: You can open the Cloud Shell in a new tab by clicking on icon **Open in a New Window** on the far right of the Cloud Shell bar.\n", "\n", "\n", - "#### Opening Cloud Shell in a Tab - Optional\n", + "### Opening Cloud Shell in a Tab - Optional\n", "\n", " * Click the **Open in a New Window** icon on the far left of the \"Cloud Shell\" tab on the bottom to open up in a separate tab. Hover the mouse over the icons to see their name/function. **Note:** you may need to allow popups or enable popups for this website depending on the browser you use. You can also pull up the tab to increase the shell screen size.\n", " * Click the **\"Close editor\"** button (the pencil icon in the top right of the window).\n", "\n", - "#### Advanced call-out\n", + "### Advanced call-out\n", "You can access the cloud shell from your laptop via `gcloud cloud-shell ssh` if you have the [Google Cloud SDK installed](https://cloud.google.com/sdk/docs/install). Installing the Google Cloud SDK will also allow you to run `gcloud` and `gsutil` commands directly from your laptop, workstation, or cluster. *Pro-tip, you cannot use a service account to access the cloud shell.*" ] }, @@ -74,7 +74,7 @@ "id": "e4fa29a6-7d8e-4591-af3f-8539b94b3bef", "metadata": {}, "source": [ - "### Verify the Configuration (Who, What, Where)" + "## Verify the Configuration (Who, What, Where)" ] }, { @@ -555,7 +555,7 @@ "id": "270e0abb-d5e9-476a-af91-060cc0ea0349", "metadata": {}, "source": [ - "### Exercise\n", + "## Exercise\n", " * What command shows the project number? (hint: we used it already)\n", " * What is the name of this variable? (hint: it is case sensitive)\n", " * What command shows *just* the Project Number (hint: use `--format`)\n", @@ -567,7 +567,7 @@ "id": "bda44ee1-337a-447d-8377-e8f7ae563316", "metadata": {}, "source": [ - "### Advanced Call-Out: Formating Output.\n", + "## Advanced Call-Out: Formating Output.\n", "\n", "The following example shows how to output the configuration as `JSON`. The command `jq` can also be used to parse the JSON data." ] diff --git a/content/GCP/05_intro_to_cloud_storage.ipynb b/content/GCP/05_intro_to_cloud_storage.ipynb index 0d826dd..f53c14f 100644 --- a/content/GCP/05_intro_to_cloud_storage.ipynb +++ b/content/GCP/05_intro_to_cloud_storage.ipynb @@ -5,7 +5,7 @@ "id": "fb98ae89-8494-4fd7-aa97-44380424d68d", "metadata": {}, "source": [ - "## Introduction to Google Cloud Storage\n", + "# Introduction to Google Cloud Storage\n", "\n", "Learner Questions\n", " * How do I store data in the cloud?\n", @@ -37,7 +37,7 @@ "tags": [] }, "source": [ - "### Google Cloud Storage\n", + "## Google Cloud Storage\n", "\n", "To learn more about the Google Cloud Storage service we will use the web console to create, explore, and destroy a Bucket. The ability to quickly create, explore, and then discard resources in the web-console is a powerful tool to explore a service. After an initial exploration with the web-console, programmatic access using scripts and programming languages should be used.\n", "\n", @@ -53,7 +53,7 @@ "tags": [] }, "source": [ - "### Security\n", + "## Security\n", "\n", "Everything in the cloud requires permission (authorization). Let's first verify that we have the permissions to create a bucket. A Bucket (a resource) is created within a project and inheres permissions from it.\n", "\n", @@ -73,7 +73,7 @@ "tags": [] }, "source": [ - "### Allocate Google Cloud Storage\n", + "## Allocate Google Cloud Storage\n", "\n", "Now that we have verified the permissions we can now create a bucket. Buckets are where objects are stored and have a globally unique name.\n", "\n", @@ -108,7 +108,7 @@ "tags": [] }, "source": [ - "### Follow the Storage Allocation\n", + "## Follow the Storage Allocation\n", "\n", "Just as with security, we will audit (follow) the bucket creation by examining at the project *activity*.\n", "\n", @@ -129,7 +129,7 @@ "tags": [] }, "source": [ - "### Enumerate the Buckets\n", + "## Enumerate the Buckets\n", "\n", "Now lets find and examine the bucket. To view a bucket we do the following:\n", "\n", @@ -147,7 +147,7 @@ "tags": [] }, "source": [ - "### Review what is Important\n", + "## Review what is Important\n", "\n", "It is always important to review what is important to you. It may be cost, or keeping the data secure. Later on we will show how to monitor overall costs.\n", "\n", @@ -162,11 +162,10 @@ "cell_type": "markdown", "id": "0362e76c-3f03-4299-8217-24a2f39ee503", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "### Delete the Storage - Exercise \n", + "## Delete the Storage - Exercise \n", "\n", "We will now complete the resource life-cycle by deleting the bucket we just created. Think about the process we did for creating the bucket and follow a similar one. \n", "\n", @@ -180,7 +179,7 @@ "id": "3a28e28d-1d70-44fa-a952-4f3506ea85ec", "metadata": {}, "source": [ - "### Discussion\n", + "## Discussion\n", "\n", "* What does the words \"Secure\", \"Allocate\", \"Follow\", and \"Enumerate\" Spell?\n", "* Why do you think we use this framework?" @@ -191,7 +190,7 @@ "id": "97d7ebc5-4a81-4f1a-aaf3-517adf70640a", "metadata": {}, "source": [ - "### Resources in Google Cloud Platform - Optional\n", + "## Resources in Google Cloud Platform - Optional\n", "\n", "Even though we only covered the Google Cloud Storage service in this episode, this process can be used for other *resources* allocated in the cloud. The term *resource* is used for the \"things\" that live in a Project, such as compute, storage, and networking and other services. Resources have the following characteristics:\n", "\n", @@ -209,8 +208,7 @@ "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", "metadata": {}, "source": [ - "\n", - "### References\n", + "## References\n", " * API reference https://cloud.google.com/storage/docs/reference/libraries\n", " * Permissions (IAM): https://cloud.google.com/storage/docs/access-control/iam-permissions" ] diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index 074a9f6..c563cb7 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -7,52 +7,34 @@ "source": [ "# Putting It All Together\n", "\n", - "\n", "```{admonition} Overview\n", ":class: tip\n", "\n", + "**Teaching:**\n", "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", + "**Exercises:**\n", + "\n", + "**Questions:**\n", + " * \n", + "\n", + "**Objectives:**\n", + " * \n", "\n", - "print(\"A python cell\")\n", "```" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/07_monitoring_costs.ipynb b/content/GCP/07_monitoring_costs.ipynb index 2cdbf95..e971611 100644 --- a/content/GCP/07_monitoring_costs.ipynb +++ b/content/GCP/07_monitoring_costs.ipynb @@ -7,30 +7,26 @@ "source": [ "# Monitoring Costs\n", "\n", - "\n", "```{admonition} Overview\n", ":class: tip\n", "\n", + "**Teaching:**\n", "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", + "**Exercises:**\n", + "\n", + "**Questions:**\n", + " * \n", + "\n", + "**Objectives:**\n", + " * \n", "\n", - "print(\"A python cell\")\n", "```" ] }, { "cell_type": "code", "execution_count": null, - "id": "c543528f", + "id": "1c5f78a5-ec20-46a8-9fc4-eb8fcc6c4482", "metadata": {}, "outputs": [], "source": [] @@ -38,21 +34,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/08_cleaning_up_resources.ipynb b/content/GCP/08_cleaning_up_resources.ipynb index 0e0c1c6..9f476a5 100644 --- a/content/GCP/08_cleaning_up_resources.ipynb +++ b/content/GCP/08_cleaning_up_resources.ipynb @@ -7,52 +7,34 @@ "source": [ "# Cleaning Up Resources\n", "\n", - "\n", "```{admonition} Overview\n", ":class: tip\n", "\n", + "**Teaching:**\n", "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", + "**Exercises:**\n", + "\n", + "**Questions:**\n", + " * \n", + "\n", + "**Objectives:**\n", + " * \n", "\n", - "print(\"A python cell\")\n", "```" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/glossary.ipynb b/content/GCP/glossary.ipynb new file mode 100644 index 0000000..e690bf9 --- /dev/null +++ b/content/GCP/glossary.ipynb @@ -0,0 +1,34 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1f0027c2-df7b-426e-b4f4-3e1db9916d14", + "metadata": {}, + "source": [ + "# Glossary\n", + "\n", + "* **Account** - The Google account (email) associated with an individual. This is the identity that you use to access console and takes the form of an email address.\n", + "* **Project** - Resources and services in Google Cloud are created within a single Project. \n", + "* **Project Name** - The long human readable name of your project. You can change this.\n", + "* **Project ID** - an alpha-numeric string identifier for the project. This is not changeable.\n", + "* **Project Number** - a long number that also identifies the project but is rarely used (it is an internal identifier). \n", + "* **Billing Account** - an account used to pay for Google Cloud services. A Billing Account is associated with a Project and enables resource allocation. A project has only one Billing Account associated with it. A Billing account can be used for multiple Projects." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Bash", + "language": "bash", + "name": "bash" + }, + "language_info": { + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/content/GCP/intro_to_GCP_Essentials.ipynb b/content/GCP/intro_to_GCP_Essentials.ipynb index cdfbf38..4e9c5ab 100644 --- a/content/GCP/intro_to_GCP_Essentials.ipynb +++ b/content/GCP/intro_to_GCP_Essentials.ipynb @@ -10,7 +10,7 @@ "The modules in this lesson are: \n", "\n", "1. [Introduction to the GCP Cloud Console](./01_intro_to_cloud_console)\n", - "2. [Introduction to Cloud Compute](./02_intro_to_compute_part1)\n", + "2. [Introduction to Cloud Compute](./02_intro_to_compute)\n", "3. [Introduction to the Cloud CLI](./04_intro_to_cli)\n", "4. [Introduction to Cloud Storage](./05_intro_to_cloud_storage)\n", "5. [Running Analysis on the Cloud](./06_running_analysis)\n", diff --git a/content/_toc.yml b/content/_toc.yml index fc2c14e..e771863 100644 --- a/content/_toc.yml +++ b/content/_toc.yml @@ -36,13 +36,13 @@ parts: - file: GCP/intro_to_GCP_Essentials sections: - file: GCP/01_intro_to_cloud_console - - file: GCP/02_intro_to_cloud_compute_part1 - - file: GCP/03_intro_to_cloud_compute_part2 + - file: GCP/02_intro_to_compute - file: GCP/04_intro_to_cli - file: GCP/05_intro_to_cloud_storage - file: GCP/06_running_analysis - file: GCP/07_monitoring_costs - file: GCP/08_cleaning_up_resources + - file: GCP/glossary - caption: Extra Learning Materials chapters: From 2cbf61c81d9b1f6911fd500d601c943005554b73 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Mon, 8 Nov 2021 22:36:00 +0000 Subject: [PATCH 2/6] Switch to bash kernel --- content/GCP/01_intro_to_cloud_console.ipynb | 20 +++++++------------- content/GCP/02_intro_to_compute.ipynb | 20 +++++++------------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/content/GCP/01_intro_to_cloud_console.ipynb b/content/GCP/01_intro_to_cloud_console.ipynb index b357627..ff4e087 100644 --- a/content/GCP/01_intro_to_cloud_console.ipynb +++ b/content/GCP/01_intro_to_cloud_console.ipynb @@ -168,21 +168,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/02_intro_to_compute.ipynb b/content/GCP/02_intro_to_compute.ipynb index c8af305..785c3e4 100644 --- a/content/GCP/02_intro_to_compute.ipynb +++ b/content/GCP/02_intro_to_compute.ipynb @@ -260,21 +260,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.3" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, From 951f0326311f78e2f41ae813db162a474cc00329 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Tue, 9 Nov 2021 20:03:04 +0000 Subject: [PATCH 3/6] Fix whitespace in jupyter.sh --- scripts/jupyter-lab.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/jupyter-lab.sh b/scripts/jupyter-lab.sh index f67c182..36ea3a0 100755 --- a/scripts/jupyter-lab.sh +++ b/scripts/jupyter-lab.sh @@ -16,6 +16,6 @@ python3 -m pip install --upgrade jupyterlab python3 -m pip install --upgrade bash_kernel python3 -m bash_kernel.install -python3 -m pip install --upgrade jupyterlab-spellchecker +python3 -m pip install --upgrade jupyterlab-spellchecker jupyter-lab --port=8081 From 774dd530f8c608858db23242109399f94b10df54 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Tue, 9 Nov 2021 20:43:32 +0000 Subject: [PATCH 4/6] Basic VM/code/Bucket usage --- content/GCP/06_running_analysis.ipynb | 259 ++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index c563cb7..05428b8 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -22,6 +22,265 @@ "\n", "```" ] + }, + { + "cell_type": "markdown", + "id": "c87e200c-1d18-44f5-b4e6-28b697b04b3c", + "metadata": {}, + "source": [ + "## Connect to the VM\n", + "\n", + "First login to the instance from the Cloud Shell\n", + "```\n", + "gcloud compute ssh instance-1\n", + "```\n", + "If prompted for a zone select `n` to find it automatically. You can see an example session below." + ] + }, + { + "cell_type": "markdown", + "id": "44f32772-0abb-4fa7-a013-202fc646df5e", + "metadata": {}, + "source": [ + "```\n", + "student31@cloudshell:~ (just-armor-301114)$ gcloud compute ssh instance-1\n", + "Did you mean zone [us-central1-b] for instance: [instance-1] (Y/n)? n\n", + "\n", + "No zone specified. Using zone [us-west2-c] for instance: [instance-1].\n", + "Linux instance-1 4.19.0-18-cloud-amd64 #1 SMP Debian 4.19.208-1 (2021-09-29) x86_64\n", + "\n", + "The programs included with the Debian GNU/Linux system are free software;\n", + "the exact distribution terms for each program are described in the\n", + "individual files in /usr/share/doc/*/copyright.\n", + "\n", + "Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent\n", + "permitted by applicable law.\n", + "Last login: Tue Nov 9 20:12:49 2021 from 34.133.99.196\n", + "student31@instance-1:~$\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "id": "b7c4db9e-f098-41bc-80d2-b524444eec7f", + "metadata": {}, + "source": [ + "## Get Example Code\n", + "\n", + "We will now install `git` and use it to download the example code and change into the directory." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "05fadd30-4f35-4a93-b713-cc132131b948", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading package lists... Done\n", + "Building dependency tree \n", + "Reading state information... Done\n", + "git is already the newest version (1:2.20.1-2+deb10u3).\n", + "0 upgraded, 0 newly installed, 0 to remove and 7 not upgraded.\n" + ] + } + ], + "source": [ + "sudo apt install --yes git" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'CLASS-Examples'...\n", + "remote: Enumerating objects: 16, done.\u001b[K\n", + "remote: Counting objects: 100% (16/16), done.\u001b[K\n", + "remote: Compressing objects: 100% (13/13), done.\u001b[K\n", + "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25\u001b[K\n", + "Unpacking objects: 100% (41/41), done.\n" + ] + } + ], + "source": [ + "git clone https://github.internet2.edu/CLASS/CLASS-Examples.git" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab", + "metadata": {}, + "outputs": [], + "source": [ + "cd CLASS-Examples/landsat/" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 24\n", + "-rw-r--r-- 1 student31 student31 72 Nov 9 20:18 clean.sh\n", + "-rw-r--r-- 1 student31 student31 256 Nov 9 20:18 download.sh\n", + "-rw-r--r-- 1 student31 student31 314 Nov 9 20:18 get-index.sh\n", + "-rw-r--r-- 1 student31 student31 841 Nov 9 20:18 ReadMe.md\n", + "-rw-r--r-- 1 student31 student31 110 Nov 9 20:18 search.json\n", + "-rw-r--r-- 1 student31 student31 1447 Nov 9 20:18 search.py\n" + ] + } + ], + "source": [ + "ls -l" + ] + }, + { + "cell_type": "markdown", + "id": "2b8b3144-7453-4350-a1cf-7fa74af2bcbf", + "metadata": {}, + "source": [ + "## Access the bucket\n", + "\n", + "First test bucket access by listing the bucket" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "gs://gcp-public-data-landsat/index.csv.gz\n", + "gs://gcp-public-data-landsat/LC08/\n", + "gs://gcp-public-data-landsat/LE07/\n", + "gs://gcp-public-data-landsat/LM01/\n", + "gs://gcp-public-data-landsat/LM02/\n", + "gs://gcp-public-data-landsat/LM03/\n", + "gs://gcp-public-data-landsat/LM04/\n", + "gs://gcp-public-data-landsat/LM05/\n", + "gs://gcp-public-data-landsat/LO08/\n", + "gs://gcp-public-data-landsat/LT04/\n", + "gs://gcp-public-data-landsat/LT05/\n", + "gs://gcp-public-data-landsat/LT08/\n" + ] + } + ], + "source": [ + "gsutil ls gs://gcp-public-data-landsat/" + ] + }, + { + "cell_type": "markdown", + "id": "89acffba-cbce-436c-98dd-05467b6675a6", + "metadata": {}, + "source": [ + "Get the index and uncompress the file placing it in the `data/` directory (this is ignored by git). This should take around 2 min with a `e2-medium` instance in the `us-west2` region" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "mkdir: created directory 'data'\n" + ] + } + ], + "source": [ + "mkdir -v data" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copying gs://gcp-public-data-landsat/index.csv.gz...\n", + "==> NOTE: You are downloading one or more large file(s), which would \n", + "run significantly faster if you enabled sliced object downloads. This\n", + "feature is enabled by default but requires that compiled crcmod be\n", + "installed (see \"gsutil help crcmod\").\n", + "\n", + "| [1 files][757.2 MiB/757.2 MiB] 737.1 KiB/s \n", + "Operation completed over 1 objects/757.2 MiB. \n" + ] + } + ], + "source": [ + "gsutil cp gs://gcp-public-data-landsat/index.csv.gz data/" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "gzip: data/index.csv: No space left on device\n" + ] + }, + { + "ename": "", + "evalue": "1", + "output_type": "error", + "traceback": [] + } + ], + "source": [ + "gzip -d data/index.csv.gz" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b005876c-f9af-43d6-80c6-f04295413b9b", + "metadata": {}, + "outputs": [], + "source": [ + "ls -lh data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbb26cfb-4b0d-4458-bc24-13a069556c3a", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 7ab26105a544c36351875b0c10ef2b31cf974f48 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Tue, 9 Nov 2021 22:37:37 +0000 Subject: [PATCH 5/6] Run GCP examples in a VM --- content/GCP/.gitignore | 1 + content/GCP/06_running_analysis.ipynb | 96 +++++++++++++-------------- content/GCP/clean.sh | 7 ++ scripts/gcp-dev.sh | 41 ++++++++++++ scripts/gcp-stop.sh | 7 ++ 5 files changed, 104 insertions(+), 48 deletions(-) create mode 100644 content/GCP/.gitignore create mode 100755 content/GCP/clean.sh create mode 100755 scripts/gcp-dev.sh create mode 100755 scripts/gcp-stop.sh diff --git a/content/GCP/.gitignore b/content/GCP/.gitignore new file mode 100644 index 0000000..f71307d --- /dev/null +++ b/content/GCP/.gitignore @@ -0,0 +1 @@ +CLASS-Examples/ diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index 05428b8..29291a2 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "05fadd30-4f35-4a93-b713-cc132131b948", "metadata": {}, "outputs": [ @@ -83,18 +83,32 @@ "Reading package lists... Done\n", "Building dependency tree \n", "Reading state information... Done\n", - "git is already the newest version (1:2.20.1-2+deb10u3).\n", - "0 upgraded, 0 newly installed, 0 to remove and 7 not upgraded.\n" + "Suggested packages:\n", + " git-daemon-run | git-daemon-sysvinit git-doc git-el git-email git-gui gitk\n", + " gitweb git-cvs git-mediawiki git-svn\n", + "The following NEW packages will be installed:\n", + " git\n", + "0 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\n", + "Need to get 0 B/5633 kB of archives.\n", + "After this operation, 36.2 MB of additional disk space will be used.\n", + "debconf: unable to initialize frontend: Dialog\n", + "debconf: (Dialog frontend will not work on a dumb terminal, an emacs shell buffer, or without a controlling terminal.)\n", + "debconf: falling back to frontend: Readline\n", + "Selecting previously unselected package git.\n", + "(Reading database ... 55447 files and directories currently installed.)\n", + "Preparing to unpack .../git_1%3a2.20.1-2+deb10u3_amd64.deb ...\n", + "Unpacking git (1:2.20.1-2+deb10u3) ...\n", + "Setting up git (1:2.20.1-2+deb10u3) ...\n" ] } ], "source": [ - "sudo apt install --yes git" + "sudo apt-get install --yes git" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7", "metadata": {}, "outputs": [ @@ -103,10 +117,10 @@ "output_type": "stream", "text": [ "Cloning into 'CLASS-Examples'...\n", - "remote: Enumerating objects: 16, done.\u001b[K\n", - "remote: Counting objects: 100% (16/16), done.\u001b[K\n", - "remote: Compressing objects: 100% (13/13), done.\u001b[K\n", - "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25\u001b[K\n", + "remote: Enumerating objects: 16, done. \n", + "remote: Counting objects: 100% (16/16), done. \n", + "remote: Compressing objects: 100% (13/13), done. \n", + "remote: Total 41 (delta 4), reused 15 (delta 3), pack-reused 25 \n", "Unpacking objects: 100% (41/41), done.\n" ] } @@ -117,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab", "metadata": {}, "outputs": [], @@ -127,7 +141,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2", "metadata": {}, "outputs": [ @@ -136,12 +150,12 @@ "output_type": "stream", "text": [ "total 24\n", - "-rw-r--r-- 1 student31 student31 72 Nov 9 20:18 clean.sh\n", - "-rw-r--r-- 1 student31 student31 256 Nov 9 20:18 download.sh\n", - "-rw-r--r-- 1 student31 student31 314 Nov 9 20:18 get-index.sh\n", - "-rw-r--r-- 1 student31 student31 841 Nov 9 20:18 ReadMe.md\n", - "-rw-r--r-- 1 student31 student31 110 Nov 9 20:18 search.json\n", - "-rw-r--r-- 1 student31 student31 1447 Nov 9 20:18 search.py\n" + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 841 Nov 9 22:31 ReadMe.md\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 72 Nov 9 22:31 clean.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 256 Nov 9 22:31 download.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 314 Nov 9 22:31 get-index.sh\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 110 Nov 9 22:31 search.json\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 1447 Nov 9 22:31 search.py\n" ] } ], @@ -161,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd", "metadata": {}, "outputs": [ @@ -198,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e", "metadata": {}, "outputs": [ @@ -216,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 7, "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38", "metadata": {}, "outputs": [ @@ -230,7 +244,7 @@ "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "| [1 files][757.2 MiB/757.2 MiB] 737.1 KiB/s \n", + "/ [1 files][757.2 MiB/757.2 MiB] \n", "Operation completed over 1 objects/757.2 MiB. \n" ] } @@ -241,46 +255,32 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 8, "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "gzip: data/index.csv: No space left on device\n" - ] - }, - { - "ename": "", - "evalue": "1", - "output_type": "error", - "traceback": [] - } - ], + "outputs": [], "source": [ "gzip -d data/index.csv.gz" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "b005876c-f9af-43d6-80c6-f04295413b9b", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 2.6G\n", + "-rw-r--r-- 1 tmiddelkoop tmiddelkoop 2.6G Nov 9 22:31 index.csv\n" + ] + } + ], "source": [ "ls -lh data" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbb26cfb-4b0d-4458-bc24-13a069556c3a", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/content/GCP/clean.sh b/content/GCP/clean.sh new file mode 100755 index 0000000..454a326 --- /dev/null +++ b/content/GCP/clean.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "=== clean.sh" + +## Cleanup for 06_running_analysis +sudo apt remove git +rm -rf ./CLASS-Examples diff --git a/scripts/gcp-dev.sh b/scripts/gcp-dev.sh new file mode 100755 index 0000000..f64a96c --- /dev/null +++ b/scripts/gcp-dev.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Options +BRANCH="${1:-gcp-dev}" # checkout branch $1 + +# Static Config - update gcp-*.sh files +VM=essentials +ZONE=us-west2-c +PROJECT=CLASS-Essentials +REPO="git@github.internet2.edu:CLASS/${PROJECT}.git" + +echo "=== gcp-dev.sh $PROJECT $BRANCH" + +echo "+++ creating VM" +gcloud compute instances create --zone=$ZONE $VM --machine-type=e2-highcpu-4 --preemptible + + +echo "+++ wait for boot" +while ! gcloud compute ssh --zone=$ZONE $VM --command='hostname' ; do + sleep 1 +done + +echo "+++ configuring VM" + +gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-A' < .ssh/known_hosts +git config --global color.ui auto +git config --global push.default simple +git config --global pull.ff only +git clone --branch $BRANCH $REPO +cd $PROJECT +git config user.name "$(git config user.name)" +git config user.email "$(git config user.name)" +EOF + +echo "+++ configure local ssh" +gcloud compute config-ssh + +echo "+++ starting Jypter" +gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-L 8080:localhost:8080 -L 8081:localhost:8081' --command="cd $PROJECT ; ./scripts/jupyter-lab.sh" diff --git a/scripts/gcp-stop.sh b/scripts/gcp-stop.sh new file mode 100755 index 0000000..7237119 --- /dev/null +++ b/scripts/gcp-stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +## User settings in gcp-dev.sh +VM=essentials +ZONE=us-west2-c + +gcloud --quiet compute instances delete --zone=$ZONE $VM From b7a4a3598d46d4d145749a050701c31ef0078640 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Wed, 10 Nov 2021 08:08:39 -0600 Subject: [PATCH 6/6] Add pty for clean jupyter shutdown and rename --- scripts/{gcp-dev.sh => gcp-start.sh} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename scripts/{gcp-dev.sh => gcp-start.sh} (87%) diff --git a/scripts/gcp-dev.sh b/scripts/gcp-start.sh similarity index 87% rename from scripts/gcp-dev.sh rename to scripts/gcp-start.sh index f64a96c..2a30cfd 100755 --- a/scripts/gcp-dev.sh +++ b/scripts/gcp-start.sh @@ -38,4 +38,4 @@ echo "+++ configure local ssh" gcloud compute config-ssh echo "+++ starting Jypter" -gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-L 8080:localhost:8080 -L 8081:localhost:8081' --command="cd $PROJECT ; ./scripts/jupyter-lab.sh" +gcloud compute ssh --zone=$ZONE $VM --ssh-flag='-t -L 8080:localhost:8080 -L 8081:localhost:8081' --command="cd $PROJECT ; ./scripts/jupyter-lab.sh"