diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bcf3611 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.DS_Store/ +.ipynb_checkpoints/ +/.vscode/ +.venv/ +log/ +data/ +_build/ +local* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4ae7d8a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "files.watcherExclude": { + "**/.venv/**": true + }, + "python.pythonPath": ".venv/bin/python3" +} diff --git a/Build.md b/Build.md new file mode 100644 index 0000000..d9b5a55 --- /dev/null +++ b/Build.md @@ -0,0 +1,21 @@ +# Book Build Instructions + +To build the book for local development run +``` +./build-book.sh +``` + +To copy results to the `gh-pages` branch and push run +``` +./push-book.sh +``` + +## GCP + +Using Jupyter launched from a remote Visual Code Studio session inside GCP Cloud-Shell requires some local settings stored in `local.sh`, use the following as a template replacing the project and email with the proper vaules: +``` +gcloud config set account student31@class.internet2.edu +export GOOGLE_CLOUD_PROJECT=just-armor-301114 +export DEVSHELL_PROJECT_ID=$GOOGLE_CLOUD_PROJECT +gcloud config set project $GOOGLE_CLOUD_PROJECT +``` diff --git a/content/.DS_Store b/content/.DS_Store deleted file mode 100644 index 5c68101..0000000 Binary files a/content/.DS_Store and /dev/null differ diff --git a/content/AWS/.ipynb_checkpoints/intro_to_AWS_Essentials-checkpoint.ipynb b/content/AWS/.ipynb_checkpoints/intro_to_AWS_Essentials-checkpoint.ipynb deleted file mode 100644 index f6480ee..0000000 --- a/content/AWS/.ipynb_checkpoints/intro_to_AWS_Essentials-checkpoint.ipynb +++ /dev/null @@ -1,43 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21ff6943", - "metadata": {}, - "source": [ - "# Amazon Web Services (AWS)\n", - "\n", - "The modules in this lesson are: \n", - "\n", - "1. [Introduction to the AWS Cloud Console](./01_intro_to_cloud_console)\n", - "2. [Introduction to Cloud Compute](./02_intro_to_compute_part1)\n", - "3. [Introduction to the AWS CLI](./04_intro_to_cli)\n", - "4. [Introduction to Cloud Storage](./05_intro_to_cloud_storage)\n", - "5. [Running Analysis on the Cloud](./06_running_analysis)\n", - "6. [Monitoring Costs](./07_monitoring_costs)\n", - "7. [Cleaning up Resources and Best Practices](./08_cleaning_up_resources)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/AWS/images/.DS_Store b/content/AWS/images/.DS_Store deleted file mode 100644 index a4eb98c..0000000 Binary files a/content/AWS/images/.DS_Store and /dev/null differ diff --git a/content/Azure/.ipynb_checkpoints/intro_to_Azure_Essentials-checkpoint.ipynb b/content/Azure/.ipynb_checkpoints/intro_to_Azure_Essentials-checkpoint.ipynb deleted file mode 100644 index 45acd85..0000000 --- a/content/Azure/.ipynb_checkpoints/intro_to_Azure_Essentials-checkpoint.ipynb +++ /dev/null @@ -1,43 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21ff6943", - "metadata": {}, - "source": [ - "# Microsoft Azure\n", - "\n", - "The modules in this lesson are: \n", - "\n", - "1. [Introduction to the Azure Cloud Portal](./01_intro_to_cloud_console)\n", - "2. [Introduction to Cloud Compute (Azure Virtual Machines)](./02_intro_to_compute_part1)\n", - "3. [Introduction to the Azure CLI](./04_intro_to_cli)\n", - "4. [Introduction to Cloud Storage (Blob Storage)](./05_intro_to_cloud_storage)\n", - "5. [Running Analysis on the Cloud](./06_running_analysis)\n", - "6. [Monitoring Costs](./07_monitoring_costs)\n", - "7. [Cleaning up Resources and Best Practices](./08_cleaning_up_resources)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/01_intro_to_cloud_console-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/01_intro_to_cloud_console-checkpoint.ipynb deleted file mode 100644 index 43bdad1..0000000 --- a/content/GCP/.ipynb_checkpoints/01_intro_to_cloud_console-checkpoint.ipynb +++ /dev/null @@ -1,191 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "24b51f8d-dcc2-4d19-8bd9-88a71fa8abfe", - "metadata": { - "tags": [] - }, - "source": [ - "## Introduction to the GCP Cloud Console\n", - "\n", - "Learner Questions\n", - " * How do I find things in the web console?\n", - "\n", - "Learning Objectives\n", - " * Login to the Google Cloud web console\n", - " * Find and verify important information about your cloud environment.\n" - ] - }, - { - "cell_type": "markdown", - "id": "3316c7f9-b0c8-4565-bdc6-7582c4f16127", - "metadata": {}, - "source": [ - "### The Who, What, and Where of the Cloud\n", - "\n", - "Before we get started we must first define, and find, a few key pieces of information about your environment that will be used throughout this lesson. This information is also important to ensure that this information is what you expect, otherwise you may use the wrong account or run or store things in the wrong place. It is important to be clear about these terms as they are often different in other cloud providers. We provide a [glossary of GCP terms](./glossary.ipynb) for your reference." - ] - }, - { - "cell_type": "markdown", - "id": "afaf408a-8639-4f8d-9108-eddcc6b13b6f", - "metadata": { - "tags": [] - }, - "source": [ - "### Accounts (Authentication and Identity)\n", - "\n", - "The (Google) **Account** is the identity used to access the Google Cloud Platform web console and is in the form of an email address and is associated with an individual (the *who* is *you*!). This is typically the account you use to access Google services associated with your institution and not your personal account. This is the \"Who\" of the cloud. We will now login to the Google Cloud Platform web console and verify our account.\n", - "\n", - " * Login to the web console with your (Google) Account by going to [https://console.cloud.google.com/](https://console.cloud.google.com/)\n", - " * Login with your credentials if you have not done so already.\n", - " * Find your user/account under the \"Avatar\" in the top right (see below).\n", - " ![avatar-account](img/avatar-account.png) and click on it if it is not the top one.\n", - " * If you do not see your account click on \"Add Account\" and repeat the previous step.\n", - "\n", - "The account list found under the avatar shows all the accounts that are logged in and the top one is the current active or selected account." - ] - }, - { - "cell_type": "markdown", - "id": "11f0cbb4-f68f-4525-a30b-02a3a4d251a1", - "metadata": { - "tags": [] - }, - "source": [ - "#### Exercise\n", - "\n", - "Take a few moments to ask yourself the following questions:\n", - " * What account are you currently using in the Google Cloud web console?\n", - " * Are you logged into any other accounts?\n", - " * How do you know which account is being used?\n", - " \n", - " If you are not currently using the correct account, switch to it." - ] - }, - { - "cell_type": "markdown", - "id": "5cfe3d13-c050-49ee-873b-be823fda5b10", - "metadata": {}, - "source": [ - "### Projects\n", - "\n", - "Almost everything you will do within Google Cloud Platform must be associated with a **Project**. This is the \"Where\" of the cloud. You must have at least one Project and you can manage multiple Projects within a single (Google) Account. Always make sure you are working in the correct project!\n", - "\n", - "To find more information and change project settings\n", - " * Click on the \"Settings and Utilities button (kabob on the top right - see below) and then click on \"Project Settings\"\n", - " ![kabob-project](img/kabob-project.png)\n", - " * The Project Name is the human friendly description and can be changed.\n", - " * The Project ID is set on project creation and cannot be changed. The Project ID is almost always used when specifying a project.\n", - " * The Project Number is used internally by Google Services and is usually used indirectly. It is a good idea to be able to recognize this number.\n", - " \n", - "For a Project to do anything useful it must also have an enabled Billing Account associated with it. An enabled Billing Account is a prerequisite for this lesson.\n", - "\n", - "A Project, just like the word, should be associated with a real world project (for example a research effort or grant, lab, or your Ph.D. Project. etc). For Drew, this will be the Acceleration Lab project. In this way it is easier to track and allocate costs and to manage permissions and access for resources within the project.\n", - "\n", - "The active project is also shown next to the project icon (three hexagons) and clicking it brings up the project selection dialog. The current project has a check mark and and the active project can be changed by clicking on the project name or double clicking the row.\n", - " ![select-project](img/select-project.png) \n", - "\n", - "#### Exercise\n", - "\n", - "Take a few moments to ask yourself the following questions:\n", - " * What is the *name* of the Project you are currently using in the Google Cloud Platform web console?\n", - " * What is the *Project ID* of that project?\n", - " * What is the *Project Number* of that project?\n", - " * Do you have multiple projects?\n", - " * How do you know which project you are currently using?\n", - "\n", - "If you are currently not using the correct project for this lesson, switch to the correct one." - ] - }, - { - "cell_type": "markdown", - "id": "294a3fa0-7a55-4547-af19-4ef542b8dae7", - "metadata": { - "tags": [] - }, - "source": [ - "### Services\n", - "\n", - "The web console is used to control and observe the cloud from the browser. It should only be used for simple and one-time tasks, exploring new services, accessing documentation, or for monitoring and debugging resources in the cloud. Programmatic control through the console, programming languages (for example Python), and other automation tools should be used for day to day activities to make the most out of the cloud and to help with the reproducibility of research and teaching.\n", - "\n", - "The Navigation Menu (often called the hamburger) is used to navigate to the various products, which are also sometimes called services.\n", - "\n", - "![hamburger-navigation](img/hamburger-navigation.png)\n", - "\n", - " * Click on a product to navigate the product page\n", - " * You can pin frequently used items on the top of this page by clicking on the pin icon. \n", - " * Click the hamburger again to hide the Left Sidebar.\n", - "\n", - "All the different products and services are the \"What\" of the cloud. We will start with Cloud Storage in the next Episode. \n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "e955eb3a-4fdc-448b-aef8-ed1dbe2a73d9", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Everything Else\n", - "\n", - "If you did not already notice, every page on the GCP web console has a blue navigation bar at the top of the page. This can be used to quickly navigate to the page you need.\n", - "\n", - "![blue-bar](img/blue-bar.png)\n", - "\n", - "From left to right of the blue top navigation bar we have the following items (hover over the item with your mouse to see the name):\n", - " 1. Navigation Menu (often called the hamburger) allows you to go to each project and services page (shown below).\n", - " 2. Dashboard (\"Google Cloud Platform\") provides an overview of your project.\n", - " * Click it and note resources, billing, and platform Status\n", - " * This page is customizable and may look different.\n", - " 3. Project selection button showing the current project name with a drop-down arrow.\n", - " 4. Search bar - type in service or keywords\n", - " * Type in \"Cloud Storage\" and note results. Click to navigate to the Cloud Storage page.\n", - " 5. Activate Cloud Shell (terminal icon) - start the Cloud Shell\n", - " * Click it and note the shell that appears on the bottom of the page.\n", - " 6. Help (shown as a question) - quick link to documentation\n", - " 7. Notifications (shown as a bell) - a dot indicates unread notifications\n", - " 8. Settings and Utilities (often called the kabobb) - shortcut to project settings\n", - " 9. Google Account (avatar at the far right) - verify and switch accounts." - ] - }, - { - "cell_type": "markdown", - "id": "118254c6-a5d9-4bd0-8a0f-64d4f72b3cf0", - "metadata": { - "tags": [] - }, - "source": [ - "### Exercise\n", - "\n", - "Before we move on, do the following:\n", - "* Double check that you are in the correct account and project that you will be using for this lesson.\n", - "* Take a moment to reflect on the \"Who, What, and Where\" of the cloud. Why is this important?\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/02_intro_to_compute_part1-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/02_intro_to_compute_part1-checkpoint.ipynb deleted file mode 100644 index 1a1a1ac..0000000 --- a/content/GCP/.ipynb_checkpoints/02_intro_to_compute_part1-checkpoint.ipynb +++ /dev/null @@ -1,283 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "fb98ae89-8494-4fd7-aa97-44380424d68d", - "metadata": { - "tags": [] - }, - "source": [ - "## Introduction to Cloud Compute (Google Compute Engine)\n", - "\n", - "Learner Questions\n", - " * How do I compute in the cloud?\n", - "\n", - "Learning Objectives:\n", - " * Navigate the Google Compute Engine service and terminology.\n", - " * Allocate a virtual machine in Google Compute Engine.\n", - " * Choose a machine type based on the application.\n", - " * Understand how storage is managed for virtual machines.\n", - " * Use the cost estimator for Google Compute Engine.\n", - " * Understand what a service account is and how they are used.\n" - ] - }, - { - "cell_type": "markdown", - "id": "9897048a-6aa8-4d85-a557-d85b802f3f1d", - "metadata": {}, - "source": [ - "Drew, after working with a Research Computing and Data (RCD) facilitator, has decided to start their cloud journey with compute, specifically a Virtual Machine, to process data in the cloud with a large and powerful machine to do the analysis. " - ] - }, - { - "cell_type": "markdown", - "id": "739746ba-1c95-4d1e-94c7-1e5330d17f52", - "metadata": { - "tags": [] - }, - "source": [ - "### Google Compute Engine\n", - "\n", - "To learn more about the Google Compute Engine service we will use the web console to create, explore, and destroy a Virtual Machine. The ability to quickly create, explore, and then discard resources in the web-console is a powerful tool to explore a service. After an initial exploration with the web-console, programmatic access using scripts or configuration management systems should be used.\n", - "\n", - "Unfortunately, the terminology used for cloud computing differs from one commercial cloud provider to another. Google Compute Engine uses the term **VM instance** to refer to a running virtual machine, **image** for the boot disk containing the operating system, and **disk** for the writable storage used for a running *VM Instance*.\n", - "\n", - "We now take Drew through the process of creating a Google Compute Engine VM instance." - ] - }, - { - "cell_type": "markdown", - "id": "c5430b40-1a5f-40df-9e13-529ef3ece4ce", - "metadata": { - "tags": [] - }, - "source": [ - "### Allocate a VM Instance\n", - "\n", - "We will first create an *VM instance* with defaults to explore the Google Compute Engine service.\n", - "\n", - "To create a bucket we do the following:\n", - " * Click **Navigation Menu** -> **Compute Engine** (under Compute) -> **VM Instances** -> **+Create Bucket** (just under the blue bar) to open the *Create an instance* page.\n", - " * In the **New VM instance** tab on the left (selected by default) configure the *VM instance* as follows:\n", - " * For **Name**, enter a unique name for the instances (example: \"essentials-instance-1\")\n", - " * For **Region** leave the default or select your \"home\" region. The region is the physical location where your data will reside. Your \"home\" region should be close to your work and the region you use most often.\n", - " * For **Zone** leave the default (note how the name is constructed and that it is a separate data center).\n", - " * In the **Machine configuration** section:\n", - " * For **Series** select *E2* (default). The other options are for types of machines.\n", - " * For **Machine type** select *e2-medium* (default)\n", - " * Leave the remaining options as default.\n", - " * In the **Identity and API access** section note that for the **Service Accounts** -> **Service account** option that the **Compute Engine default service account** is selected. This allows the *VM Instance* to ***access and edit any resources in the project***.\n", - " * Note the **Monthly estimate** in the right side panel. This is only for running the *VM instance* and does not include other storage, networking, compute, and other charges that may occur by running software. Click **Details** show a breakdown of the *VM instance* only costs.\n", - " * Finally, click **Create** to create the *VM Instance* (allocate the resource).\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "41c63432-a614-4a1e-9967-f49b68f9069e", - "metadata": {}, - "source": [ - "### Security\n", - "\n", - "Everything in the cloud requires permission (authorization). Ordinary we would configure and check security first but in the case of exploring services it is often easier to do things out of order. We noted that the *VM instance* was created with the *Compute Engine default service account*, which by *default* has access to all the resources that your Google Account does.\n", - "\n", - "Specifically, we are interested in verifying what permissions that the *Compute Engine default service account* has for your *VM instances* . To do this navigate to the IAM page (**Navigation Menu** -> **IAM & Admin** -> **IAM** -> **Permissions** -> View By: **Principals**)\n", - "\n", - "*Note: There is a powerful filter box to limit the permissions shown.*\n", - "\n", - "In the **Name** column you should see a row with **Compute Engine default service account**. Here you should see the **Editor** role in the Role column. A *role* is a collection of permissions managed by Google or someone else. The **Editor** role for a project will *allow* a **VM instance** to modify resources *in* the **project**. The principal for the *Compute Engine default service account* should start with the *Project ID* of your project. It is possible to create a custom service account with the appropriate roles for the *VM Instance*.\n", - "\n", - "There are three important pieces of information that work together to form the **IAM policy**. The permission (role), the identity (principal or member), and the resource (project)." - ] - }, - { - "cell_type": "markdown", - "id": "da301540-9b02-4b27-ae8e-f301b3dd36f2", - "metadata": { - "tags": [] - }, - "source": [ - "### Follow the Storage Allocation\n", - "\n", - "Just as with security, we will audit (follow) the *VM instance* creation by examining at the project *activity*.\n", - "\n", - "To view the project activity we do the following:\n", - "\n", - " * Navigate to the project **Dashboard** by clicking on \"Google Cloud Platform\" on the left side of the top blue bar.\n", - " * Select **Activity** tab just below the top blue bar.\n", - " * Look for the row with **Create VM** on the top of the table. This shows that the *VM Instance* was created.\n", - " * Expand the detail by clicking the **down arrow** on the right side of the row. You can see what identity created the bucket and the name. The rest of the information details the initial settings that you configured when creating the bucket. The details are not really important at this time, just know they exist.\n", - "\n", - "Note that the activity is on the project, not the account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "3d376f83-966a-4c7d-9ecf-4973e7f00ff7", - "metadata": { - "tags": [] - }, - "source": [ - "### Enumerate the VM Instances\n", - "\n", - "To see an overview of resources in the project navigate to the **dashboard** and you will now see at least **1 instance** in the *resources* card under *Compute Engine*.\n", - "\n", - "Now lets find and connect to the *VM Instance*.\n", - " * Navigate to the Google Compute Engine page by clicking **Navigation Menu** -> **Compute Engine** (under Compute) -> **Instances**.\n", - " * **Find** the *VM instance* you just created. You can use the filter to find a bucket if there are a lot of them.\n", - " * Click on the *VM instance* name to open the **VM instance details** (it will display as a hyperlink when you hover over the *VM instance* name)." - ] - }, - { - "cell_type": "markdown", - "id": "7f4bc715-f7b5-4ab8-b13a-720dae419326", - "metadata": { - "tags": [] - }, - "source": [ - "### Connect to the VM Instance\n", - "\n", - "To connect to the *VM Instance* we will use the cloud shell in the web console. Open up a shell by clicking on the **Activate Cloud Shell** icon in the top blue bar if it is not already open.\n", - "\n", - "We first verify that the cloud shell is using the correct identity and *project id* by looking at the welcome message and the prompt.\n", - "\n", - "![cloud-shell](img/cloud-shell.png)\n", - "\n", - "To connect to the *VM instance* we enter the following command in the cloud shell:\n", - "```\n", - "gcloud compute ssh essentials-instance-1\n", - "```\n", - "\n", - "If you have not used the cloud shell to connect to a *VM Instance* before you will probably be asked to create a new *ssh key*. The Compute Engine will use this key to allow you to access the *VM instance* in a secure manner. If this is the case you will see a message similar to the following:\n", - "```\n", - "WARNING: The private SSH key file for gcloud does not exist.\n", - "WARNING: The public SSH key file for gcloud does not exist.\n", - "WARNING: You do not have an SSH key for gcloud.\n", - "WARNING: SSH keygen will be executed to generate a key.\n", - "This tool needs to create the directory [/home/student231/.ssh] before\n", - " being able to generate SSH keys.\n", - "\n", - "Do you want to continue (Y/n)?\n", - "```\n", - "\n", - "Press enter to continue to create the key. You will see another message asking you for a passphrase, similar to the following:\n", - "```\n", - "Generating public/private rsa key pair.\n", - "Enter passphrase (empty for no passphrase):\n", - "```\n", - "Here you should enter a passphrase (a series of words) to protect the key. You will be asked to re-enter the passphrase as shown below:\n", - "```\n", - "Enter same passphrase again:\n", - "```\n", - "\n", - "After successfully retyping the passphrase the key generation process will continue showing similar message as follows (the fingerprint and art will look different):\n", - "```\n", - "Your identification has been saved in /home/student231/.ssh/google_compute_engine.\n", - "Your public key has been saved in /home/student231/.ssh/google_compute_engine.pub.\n", - "The key fingerprint is:\n", - "SHA256:D9yj1nZEUigahirvjkIFoCP7RfjDu67Ku88M7nHQftI student231@cs-748001990186-default-boost-r9phq\n", - "The key's randomart image is:\n", - "+---[RSA 2048]----+\n", - "|o . .. |\n", - "|.. .. o . .. |\n", - "|+ o... o .. . |\n", - "|ooo= .. . o |\n", - "|.+..= S o . |\n", - "| o+..o = o |\n", - "|.+.+.E o + . |\n", - "|+.B o. . . . |\n", - "|=O**o |\n", - "+----[SHA256]-----+\n", - "```\n", - "\n", - "The ssh key generation is now complete. This will only happen once for your account and you will start here on future connections.\n", - "\n", - "At this point the command will attempt to connect to the *VM Instance* and will ask the following question:\n", - "```\n", - "Did you mean zone [us-central1-a] for instance: [instance-1] (Y/n)? n\n", - "```\n", - "Answer \"n\".\n", - "\n", - "The command will now configure the instance to allow your ssh key and connect to it.\n", - "\n", - "```\n", - "No zone specified. Using zone [us-central1-a] for instance: [instance-1].\n", - "Updating project ssh metadata...working..Updated [https://www.googleapis.com/compute/v1/projects/class-essentials-gcp-demo-1].\n", - "Updating project ssh metadata...done.\n", - "Waiting for SSH key to propagate.\n", - "Warning: Permanently added 'compute.74517428106645607' (ECDSA) to the list of known hosts.\n", - "```\n", - "\n", - "Once connected you will see the machine login banner and prompt similar to the following:\n", - "```\n", - "Linux instance-1 4.19.0-17-cloud-amd64 #1 SMP Debian 4.19.194-3 (2021-07-18) x86_64\n", - "\n", - "The programs included with the Debian GNU/Linux system are free software;\n", - "the exact distribution terms for each program are described in the\n", - "individual files in /usr/share/doc/*/copyright.\n", - "\n", - "Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent\n", - "permitted by applicable law.\n", - "student231@essentials-instance-1:~$\n", - "```\n", - "\n", - "Congratulations you have now created a *VM Instance* and connected to it.\n", - "\n", - "Feel free to interact with the machine for a while and type\n", - "```\n", - "exit\n", - "```\n", - "when you are done.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "0362e76c-3f03-4299-8217-24a2f39ee503", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Delete the Storage - Exercise \n", - "\n", - "We will now complete the resource life-cycle by deleting the *VM Instance* we just created. Think about the process we did for creating the bucket and follow a similar one. \n", - "\n", - "Did you \"Follow\" the bucket by looking at the **activity** page?\n", - "\n", - "Since we care about paying for resources we are not using we **review** by visiting the **dashboard** and reviewing that we no longer have any buckets, if this was our only bucket. " - ] - }, - { - "cell_type": "markdown", - "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", - "metadata": {}, - "source": [ - "### References\n", - " * Compute Engine Permissions (IAM): https://cloud.google.com/compute/docs/access/iam" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb deleted file mode 100644 index d8c3b55..0000000 --- a/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb +++ /dev/null @@ -1,661 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5439e525-b985-495f-85a6-e4c8d7452956", - "metadata": {}, - "source": [ - "## Introduction to the gcloud CLI\n", - "\n", - "Learner Questions\n", - " * How do I use the Cloud CLI?\n", - " * How do I use `gcloud`?\n", - "\n", - "Learning Objectives\n", - " * Find and use the \"Cloud Shell\" in the web console.\n", - " * Use basic cloud CLI commands (`gcloud` and `gsutil`).\n", - " * Verify basic settings.\n", - " * Use environment variables for configuration.\n", - " * Understand the importance of using variables for configuration.\n", - " * Recognize the value of reproducibility and automation.\n", - " \n", - "Resources\n", - " * `gcloud` documentation: https://cloud.google.com/sdk/gcloud\n", - " * `gsutil` documentation: https://cloud.google.com/storage/docs/gsutil\n", - " * Instructions to install `gcloud` and `gsutil` on your local machine: https://cloud.google.com/sdk/docs/install" - ] - }, - { - "cell_type": "markdown", - "id": "c4e1a4c5-7d28-483d-b27c-ef2afa4fb7e5", - "metadata": {}, - "source": [ - "### Cloud Automation\n", - "\n", - "Now that Drew can create a bucket in the web console they wish to learn how to use tools to automate the process. Drew noticed that the web console could be used to upload and share objects but wishes to learn the command line so that they can automate the and document the process (it is much easier to document commands then describing and showing mouse clicks).\n", - "\n", - "The cloud can be controlled using a Command Line Interface (CLI) or a programming language such as Python. Collectively these tools interact with the cloud over a Application Programming Interface (API) and this capability forms the basis of the cloud, the ability to control infrastructure programmatically.\n", - "\n", - "Just as with navigating the web console it is important to know the **who**, **what**, and **where** of CLI access to reduce the possibility of access mistakes. We will first verify the tools are installed and configured correctly. Next we get the Account being used (who) and the Project ID of the active project (where) using the `gcloud` command. We will then use the `gcloud` and `gsutil` commands to list some public Buckets (what).\n", - "\n", - "The `gcloud` command is used to control most aspects of GCP and the `gsutil` command is used to control Google Cloud Storage Buckets. To access the manual pages for a command just add `--help` to the end of the command or run `gcloud help` for more information.\n", - "\n", - "When running CLI commands you may get a dialog box to authorize the command. You should \"Authorize\" the request to allow the Cloud Shell to access the Project using your (Google) Account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "ce598878-1b1c-4e6b-9122-e679e84c7dbb", - "metadata": {}, - "source": [ - "### Open a Cloud Shell\n", - "\n", - "The Cloud Shell is an easy way to access Comand Line Interface (CLI) commands in a web browser. Under the hood Cloud Shell is Debian running in a container with 5GB of no-charge persistent storage with most of the Google uti and a lot of other useful tools (git, docker, jq, Docker, etc.) installed by default.\n", - "\n", - "To use the Cloud Shell:\n", - " * Open up a Cloud Shell by clicking on the **Activate Cloud Shell** icon in the right side of the top blue bar. If a \"Authorize Cloud Shell\" dialog appears click \"Authorize\" to allow the Cloud Shell to access the project.\n", - " * Click the **Maximize** icon on the right side of the Cloud Shell bar. You can find the correct icon by hovering over the icons to see their names.\n", - " * If a shell is associated with a project it will show it in the tab and in **yellow text** in round parenthesis in the shell prompt. **Verify the Project ID**.\n", - " * To open up a shell with a specific project click the \"down arrow\" icon next to the plus(+) to open a new tab with a specific project. Selecting the plus (+) will usually open up a shell with the default project.\n", - " * Note: You can open the Cloud Shell in a new tab by clicking on icon **Open in a New Window** on the far right of the Cloud Shell bar.\n", - "\n", - "\n", - "#### Opening Cloud Shell in a Tab - Optional\n", - "\n", - " * Click the **Open in a New Window** icon on the far left of the \"Cloud Shell\" tab on the bottom to open up in a separate tab. Hover the mouse over the icons to see their name/function. **Note:** you may need to allow popups or enable popups for this website depending on the browser you use. You can also pull up the tab to increase the shell screen size.\n", - " * Click the **\"Close editor\"** button (the pencil icon in the top right of the window).\n", - "\n", - "#### Advanced call-out\n", - "You can access the cloud shell from your laptop via `gcloud cloud-shell ssh` if you have the [Google Cloud SDK installed](https://cloud.google.com/sdk/docs/install). Installing the Google Cloud SDK will also allow you to run `gcloud` and `gsutil` commands directly from your laptop, workstation, or cluster. *Pro-tip, you cannot use a service account to access the cloud shell.*" - ] - }, - { - "cell_type": "markdown", - "id": "e4fa29a6-7d8e-4591-af3f-8539b94b3bef", - "metadata": {}, - "source": [ - "### Verify the Configuration (Who, What, Where)" - ] - }, - { - "cell_type": "markdown", - "id": "f786f92c-7127-4f26-b0c0-fc27a364aca4", - "metadata": {}, - "source": [ - "First, let's verify that the Account being used for access (who) is what we expect." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "3ce9e6fb-9f5f-42e1-8512-35c0564f8e85", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "student31@class.internet2.edu\n" - ] - } - ], - "source": [ - "gcloud config get-value account" - ] - }, - { - "cell_type": "markdown", - "id": "68b97adf-7378-4e9c-b2b8-2075f043afab", - "metadata": {}, - "source": [ - "Next, show verify the active project (where)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cfb10587-e007-45ae-839f-7bcfe088c711", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config get-value project" - ] - }, - { - "cell_type": "markdown", - "id": "c2972d7b-f393-42b5-8330-cf8292d28afb", - "metadata": {}, - "source": [ - "Now we will use `gcloud` to list a well known public bucket (what). " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "617325c9-d853-4291-a1db-938ab9439fee", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "gs://gcp-public-data-landsat/index.csv.gz\n", - "gs://gcp-public-data-landsat/LC08/\n", - "gs://gcp-public-data-landsat/LE07/\n", - "gs://gcp-public-data-landsat/LM01/\n", - "gs://gcp-public-data-landsat/LM02/\n", - "gs://gcp-public-data-landsat/LM03/\n", - "gs://gcp-public-data-landsat/LM04/\n", - "gs://gcp-public-data-landsat/LM05/\n", - "gs://gcp-public-data-landsat/LO08/\n", - "gs://gcp-public-data-landsat/LT04/\n", - "gs://gcp-public-data-landsat/LT05/\n", - "gs://gcp-public-data-landsat/LT08/\n" - ] - } - ], - "source": [ - "gcloud alpha storage ls gs://gcp-public-data-landsat" - ] - }, - { - "cell_type": "markdown", - "id": "42e6ad5b-186d-4cd1-ba03-e7c85ad40e38", - "metadata": {}, - "source": [ - "*Advanced Callout: The `alpha` (and `beta`) command allows us to access commands that have not been released for production and care should be taken when using these in a production environment. At this time this is not the recommended way to access storage buckets, but it does help verify that everything is working correctly.*" - ] - }, - { - "cell_type": "markdown", - "id": "1389ca4f-7234-4ea3-9ad8-85914d88ede5", - "metadata": {}, - "source": [ - "Finally, we will verify that the separate and preferred `gsutil` command is installed and working by listing the same well known public bucket. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1a13a344-65df-40bc-997f-27d9084f69cf", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "gs://gcp-public-data-landsat/index.csv.gz\n", - "gs://gcp-public-data-landsat/LC08/\n", - "gs://gcp-public-data-landsat/LE07/\n", - "gs://gcp-public-data-landsat/LM01/\n", - "gs://gcp-public-data-landsat/LM02/\n", - "gs://gcp-public-data-landsat/LM03/\n", - "gs://gcp-public-data-landsat/LM04/\n", - "gs://gcp-public-data-landsat/LM05/\n", - "gs://gcp-public-data-landsat/LO08/\n", - "gs://gcp-public-data-landsat/LT04/\n", - "gs://gcp-public-data-landsat/LT05/\n", - "gs://gcp-public-data-landsat/LT08/\n" - ] - } - ], - "source": [ - "gsutil ls gs://gcp-public-data-landsat" - ] - }, - { - "cell_type": "markdown", - "id": "598b5d12-9bea-49de-a772-4113727e0859", - "metadata": {}, - "source": [ - "### Environment Variables\n", - "\n", - "When using the CLI you quickly notice that you have to enter many of the details, such as the Project ID, many times (Drew is also Dyslexic, which makes it even harder for them). Not only is this tedious, it also makes moving to another project or using another bucket difficult making reusability and collaboration painful. We will demonstrate the use environment variables to address this issue. Most cloud environments and tools rely heavily on this technique.\n", - "\n", - "*To make the point even clearer, we use environment variables in the Cloud Essentials notebooks instead of hard coding them, which makes it easy for anyone to run, use, and edit and thus collaborate and contribute to them!*\n", - "\n", - "We now demonstrate the basic use environment variables. As a reminder, environment variables are used to store configuration information that can be easily passed between programs and programming languages." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b8f74fe9-0943-4bd9-a91b-da09c001f8a1", - "metadata": {}, - "outputs": [], - "source": [ - "TEST=1" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b937f6c7-3697-4bc0-8e32-da0b0faec593", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n" - ] - } - ], - "source": [ - "echo $TEST" - ] - }, - { - "cell_type": "markdown", - "id": "78fe7013-a5ae-498f-aeab-ba1f3c710de1", - "metadata": {}, - "source": [ - "Although we do not always need to, we should always enclose shell variables in double quotes and enclose the variable name in curly braces (funny things may happen otherwise). In most cases it is easier to do this every time than try to figure out when it is possible to use the short form. We only use the short form (`$TEST`) when it is used frequently and known to always work, in our case it is `$PROJECT` and `$BUCKET`. When in doubt, use the explicit form.\n", - "\n", - "There are a number of environment variables already set that contain useful information, for example:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "bae0587c-0379-40e3-9c31-16ba5505487a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "student31\n" - ] - } - ], - "source": [ - "echo \"${USER}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "52b4add5-0d18-4faf-a304-b7e0ee8ba82e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cs-730348008320-default-default-w724w\n" - ] - } - ], - "source": [ - "echo \"${HOSTNAME}\"" - ] - }, - { - "cell_type": "markdown", - "id": "ef895f38-ad16-4f2d-aa18-ec913732dd69", - "metadata": {}, - "source": [ - "We can also use the output of commands to set values." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "776806e4-209f-45fa-b14e-f771f5d5089d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue 26 Oct 2021 07:44:54 PM UTC\n" - ] - } - ], - "source": [ - "date" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "560fde63-0eb8-4c52-9787-4debfe35ec23", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Date: Tue 26 Oct 2021 07:44:55 PM UTC\n" - ] - } - ], - "source": [ - "echo \"Date: $(date)\"" - ] - }, - { - "cell_type": "markdown", - "id": "6323f7bf-b2fe-4c94-a87a-2d35e75a1a49", - "metadata": { - "tags": [] - }, - "source": [ - "Now lets combine all this to set the `PROJECT` environment variable to the default project so we do not need to use the actual value in future commands." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "18f40620-8537-42b5-948f-dd581e604a36", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config get-value project" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "a959f857-8a20-4356-bc30-dbf8c75be96d", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n" - ] - } - ], - "source": [ - "PROJECT=\"$(gcloud config get-value project)\"" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "c1d27e89-cc69-4979-82c4-79c3fb36bccb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "echo $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "9d3a8504-a75c-4945-ab54-85f1d9a7e2b2", - "metadata": {}, - "source": [ - "Note the \"Your active configuration is: ...\" line is a \"warning\" and not actually part of the output of the program." - ] - }, - { - "cell_type": "markdown", - "id": "c25e6431-4f92-454a-bd10-e77cb28deec6", - "metadata": {}, - "source": [ - "Now use the `PROJECT` envrionment variable to get more information about the active project." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "31b01f78-94b1-44f0-a2af-d657058941b2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "createTime: '2021-01-08T14:17:35.831Z'\n", - "lifecycleState: ACTIVE\n", - "name: CLASS Project\n", - "projectId: just-armor-301114\n", - "projectNumber: '1002111293252'\n" - ] - } - ], - "source": [ - "gcloud projects describe $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "ed2ab9ce-6b0a-4939-88c7-8103e29413b7", - "metadata": {}, - "source": [ - "Note the extraneous message about the active configuration produced by the `gcloud config get-value project` command. A slightly different way to access the data can be used to remove the extraneous message by using the following command:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "845b2aaf-0032-4043-a40a-d2482693d23e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[accessibility]\n", - "screen_reader = True\n", - "[component_manager]\n", - "disable_update_check = True\n", - "[compute]\n", - "gce_metadata_read_timeout_sec = 30\n", - "[core]\n", - "account = student31@class.internet2.edu\n", - "disable_usage_reporting = True\n", - "project = just-armor-301114\n", - "[metrics]\n", - "environment = devshell\n", - "\n", - "Your active configuration is: [cloudshell-27180]\n" - ] - } - ], - "source": [ - "gcloud config list" - ] - }, - { - "cell_type": "markdown", - "id": "d919b26d-2f64-4a43-9c76-34f7798fe853", - "metadata": {}, - "source": [ - "We can now use the `--format='value()'` option to get specific values. The following command returns the project number without the extraneous message." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "9714ebe2-2921-498b-9e0e-69ec6a85896a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config list --format='value(core.project)'" - ] - }, - { - "cell_type": "markdown", - "id": "ba660d2b-5ddf-449b-8049-56a40f2e366b", - "metadata": {}, - "source": [ - "Note where the `core` part comes in the output of the `gcloud config list` command." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "7d803ddd-acca-4ad1-933a-4d6e6044def6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "PROJECT=\"$(gcloud config list --format='value(core.project)')\"\n", - "echo $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "d40b2c7b-7331-412d-80e1-f1a77204075a", - "metadata": {}, - "source": [ - "We will use this form from now on. Since we know the Project ID only contains numbers, letters, and dashes with no spaces there is no need to use quotes." - ] - }, - { - "cell_type": "markdown", - "id": "270e0abb-d5e9-476a-af91-060cc0ea0349", - "metadata": {}, - "source": [ - "### Exercise\n", - " * What command shows the project number? (hint: we used it already)\n", - " * What is the name of this variable? (hint: it is case sensitive)\n", - " * What command shows *just* the Project Number (hint: use `--format`)\n", - " * Set the `PROJECT_NUMBER` environment variable with the correct value. (hint: be careful with punctuation (single and double quotes) and matching brackets and punctuation)" - ] - }, - { - "cell_type": "markdown", - "id": "bda44ee1-337a-447d-8377-e8f7ae563316", - "metadata": {}, - "source": [ - "### Advanced Call-Out: Formating Output.\n", - "\n", - "The following example shows how to output the configuration as `JSON`. The command `jq` can also be used to parse the JSON data." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "678a1735-8cad-403c-b302-639de1fb0f4b", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"accessibility\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"screen_reader\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"component_manager\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"disable_update_check\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"compute\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"gce_metadata_read_timeout_sec\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"30\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"core\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"account\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"student31@class.internet2.edu\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"disable_usage_reporting\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"project\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"just-armor-301114\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"metrics\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"environment\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"devshell\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - "\u001b[1;39m}\u001b[0m\n" - ] - } - ], - "source": [ - "gcloud config list --format=json |jq" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "b0dec5ed-dc7f-4003-b228-f6605119ce47", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config list --format=json |jq -r .core.project" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a22e397-9f21-4c7b-914c-88708592eeb1", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb deleted file mode 100644 index 20d2329..0000000 --- a/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "fb98ae89-8494-4fd7-aa97-44380424d68d", - "metadata": {}, - "source": [ - "## Introduction to Google Cloud Storage\n", - "\n", - "Learner Questions\n", - " * How do I store data in the cloud?\n", - "\n", - "Learning Objectives:\n", - " * Navigate the Google Cloud Storage service and terminology\n", - " * Understand the roles and permissions needed to use Google Cloud Storage in projects\n", - " * Allocate storage in Google Cloud Storage\n", - " * Find the cost estimator for Google Cloud Storage\n", - " * Recognize that resources have a \"location\"\n", - " * Recognize danger of public access\n", - " * De-allocate Google Cloud Storage storage" - ] - }, - { - "cell_type": "markdown", - "id": "9897048a-6aa8-4d85-a557-d85b802f3f1d", - "metadata": {}, - "source": [ - "Drew, after working with a Research Computing and Data (RCD) facilitator, has decided to start their cloud journey with storage. Data is usually the center of a research project and it makes sense to start here.\n", - "\n", - "One of the most common and economic ways to store data in the cloud is to use object storage. In GCP object storage is called *Google Cloud Storage*, which is similar to the Simple Storage Service, also known as S3, on Amazon Web Services (AWS). For object storage, information is stored as a collection of key-value pairs." - ] - }, - { - "cell_type": "markdown", - "id": "739746ba-1c95-4d1e-94c7-1e5330d17f52", - "metadata": { - "tags": [] - }, - "source": [ - "### Google Cloud Storage\n", - "\n", - "To learn more about the Google Cloud Storage service we will use the web console to create, explore, and destroy a Bucket. The ability to quickly create, explore, and then discard resources in the web-console is a powerful tool to explore a service. After an initial exploration with the web-console, programmatic access using scripts and programming languages should be used.\n", - "\n", - "Unfortunately, the terminology used for object storage differs from one commercial cloud provider to another. Google Cloud Storage uses the terms Bucket and Object. Data is stored in Objects and Objects are stored in Buckets. Objects (data) are stored as a key-value pair, which is similar to a Python dictionary but persistent. Values are stored and retrieved using a unique key and the value can contain any information (more on this later).\n", - "\n", - "We now take Drew through the process of creating a Google Cloud Storage bucket." - ] - }, - { - "cell_type": "markdown", - "id": "4b1d2914-cac0-465e-b61a-11bbc243cc8c", - "metadata": { - "tags": [] - }, - "source": [ - "### Security\n", - "\n", - "Everything in the cloud requires permission (authorization). Let's first verify that we have the permissions to create a bucket. A Bucket (a resource) is created within a project and inheres permissions from it.\n", - "\n", - "We are interested in what permissions that *your* account has for *your* project. To do this navigate to the IAM page (**Navigation Menu -> IAM & Admin -> IAM -> Permissions -> View By: Principals**). This shows the permissions for the project.\n", - "\n", - "*Note: There is a powerful filter box to limit the permissions shown.*\n", - "\n", - "You should see a row with your account shown in the Principal column. Here you should see the \"Owner\" Role in the Role column. A *role* is a collection of permissions managed by Google or someone else. The **Owner** or the **Storage Admin** role for a project will *allow* *you* to create, access, and delete Buckets *in* the project.\n", - "\n", - "There are three important pieces of information that work together to form the **IAM policy**. The permission (role), the identity (principal or member), and the resource (project)." - ] - }, - { - "cell_type": "markdown", - "id": "c5430b40-1a5f-40df-9e13-529ef3ece4ce", - "metadata": { - "tags": [] - }, - "source": [ - "### Allocate Google Cloud Storage\n", - "\n", - "Now that we have verified the permissions we can now create a bucket. Buckets are where objects are stored and have a globally unique name.\n", - "\n", - "To create a bucket we do the following:\n", - " * Click **Navigation Menu** -> **Cloud Storage** (under Storage) -> **Browser** -> **+Create Bucket** (just under the blue bar) to open the *Create a bucket* page.\n", - " * In *Name your bucket*:\n", - " * For **Name**, enter a globally unique name for the bucket (example \"essentials-test-myname-2021-01-01\")\n", - " * Click **Continue**\n", - " * In *Choose where to store your data*:\n", - " * For *Location Type* select **Region** (cheapest and fastest)\n", - " * For *Location* leave the default or select your \"home\" region. The region is the physical location where your data will reside. Your \"home\" region should be close to your work and the region you use most often.\n", - " * Click **Continue**\n", - " * In *Choose a default storage class for your data*:\n", - " * Select **Standard** (default). The other options are for long-term storage.\n", - " * Click **Continue**\n", - " * In *Choose how to control access to objects*:\n", - " * Under *Prevent Public Access* check \"**Enforce public access prevention on this bucket**\". This will try to prevent others from accessing your data. \n", - " * Under *Access Control* Select **Uniform** (default). *Uniform* permissions are on the entire Bucket. *Fine-grained* permissions are on the objects.\n", - " * Select **Continue**\n", - " * Note the **monthly cost estimate** in the right side panel. This is only for reading, writing, and storing data and does not include networking, compute, and other charges that may occur by using the data.\n", - " * *Storage size* is the total amount of data stored.\n", - " * *Data Retrieval* is only used for for long-term, non *standard*, storage.\n", - " * *Operations* is the number of \"operations\" performed on an object, for example reading and writing an object. Storing and accessing many objects require many operations.\n", - " * Finally, click **Create** to create the bucket (allocate the resource).\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "da301540-9b02-4b27-ae8e-f301b3dd36f2", - "metadata": { - "tags": [] - }, - "source": [ - "### Follow the Storage Allocation\n", - "\n", - "Just as with security, we will audit (follow) the bucket creation by examining at the project *activity*.\n", - "\n", - "To view the project activity we do the following:\n", - "\n", - " * Navigate to the project **Dashboard** by clicking on \"Google Cloud Platform\" on the left side of the top blue bar.\n", - " * Select **Activity** tab just below the top blue bar.\n", - " * Look for the row with **Create Bucket** on the top of the table. This shows that the bucket was created.\n", - " * Expand the detail by clicking the **down arrow** on the right side of the row. You can see what identity created the bucket and the name. The rest of the information details the initial settings that you configured when creating the bucket. The details are not really important at this time, just know they exist.\n", - "\n", - "Note that the activity is on the project, not the account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "c37dbf70-db45-44b3-92b6-2c89d772ec24", - "metadata": { - "tags": [] - }, - "source": [ - "### Enumerate the Buckets\n", - "\n", - "Now lets find and examine the bucket. To view a bucket we do the following:\n", - "\n", - " * Navigate to the Google Storage page by clicking **Navigation Menu** -> **Cloud Storage** (under Storage) -> **Browser**. \n", - " * **Find** the bucket you just created. You can use the filter to find a bucket if there are a lot of them.\n", - " * Click on the bucket name to open the **bucket details** (it will display as a hyperlink when you hover over the bucket name).\n", - "\n", - "Navigate to the **dashboard** and you will now see at least **1 bucket** in the *resources* card under storage." - ] - }, - { - "cell_type": "markdown", - "id": "258044b3-550e-4a4f-a650-27cd0d6dd057", - "metadata": { - "tags": [] - }, - "source": [ - "### Review what is Important\n", - "\n", - "It is always important to review what is important to you. It may be cost, or keeping the data secure. Later on we will show how to monitor overall costs.\n", - "\n", - "For Drew, we will review that the bucket **public access** is *not public* by doing the following:\n", - " * Go to **Navigation Menu -> Cloud Storage -> Browser**\n", - " * Select the bucket of interest by **checking the box** next to the Bucket name.\n", - " * In the Right Side Bar (open if necessary) in the **Permissions** tab in the **Public Access** card you should see **Not Public**. This means that public access prevention is turned on.\n", - " * You can also see the **permissions** for the bucket in the bottom of the bar." - ] - }, - { - "cell_type": "markdown", - "id": "0362e76c-3f03-4299-8217-24a2f39ee503", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Delete the Storage - Exercise \n", - "\n", - "We will now complete the resource life-cycle by deleting the bucket we just created. Think about the process we did for creating the bucket and follow a similar one. \n", - "\n", - "Did you \"Follow\" the bucket by looking at the **activity** page?\n", - "\n", - "Since we care about paying for resources we are not using we **review** by visiting the **dashboard** and reviewing that we no longer have any buckets, if this was our only bucket. " - ] - }, - { - "cell_type": "markdown", - "id": "3a28e28d-1d70-44fa-a952-4f3506ea85ec", - "metadata": {}, - "source": [ - "### Discussion\n", - "\n", - "* What does the words \"Secure\", \"Allocate\", \"Follow\", and \"Enumerate\" Spell?\n", - "* Why do you think we use this framework?" - ] - }, - { - "cell_type": "markdown", - "id": "97d7ebc5-4a81-4f1a-aaf3-517adf70640a", - "metadata": {}, - "source": [ - "### Resources in Google Cloud Platform - Optional\n", - "\n", - "Even though we only covered the Google Cloud Storage service in this episode, this process can be used for other *resources* allocated in the cloud. The term *resource* is used for the \"things\" that live in a Project, such as compute, storage, and networking and other services. Resources have the following characteristics:\n", - "\n", - " * Resources are like \"objects\" and usually have their own product/service page in the navigation menu. \n", - " * Resources have many properties (how it is configured), costs, permissions, and almost always belong to a Project.\n", - " * Resource creation permission is at the parent (Project).\n", - " * Resources live in a machine somewhere, and usually specify a location (Region and Zone).\n", - " * Resource creation has limits (Quota) that can be changed (up or down).\n", - " * Resources usually have multiple charges associated with it (even if they are `$0.00`).\n", - " * Resources have a beginning, middle, and an end. Don't forget the end. Have a plan!" - ] - }, - { - "cell_type": "markdown", - "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", - "metadata": {}, - "source": [ - "\n", - "### References\n", - " * API reference https://cloud.google.com/storage/docs/reference/libraries\n", - " * Permissions (IAM): https://cloud.google.com/storage/docs/access-control/iam-permissions" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb deleted file mode 100644 index 32ded70..0000000 --- a/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb +++ /dev/null @@ -1,43 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21ff6943", - "metadata": {}, - "source": [ - "# Google Cloud Platform (GCP)\n", - "\n", - "The modules in this lesson are: \n", - "\n", - "1. [Introduction to the GCP Cloud Console](./01_intro_to_cloud_console)\n", - "2. [Introduction to Cloud Compute](./02_intro_to_compute_part1)\n", - "3. [Introduction to the Cloud CLI](./04_intro_to_cli)\n", - "4. [Introduction to Cloud Storage](./05_intro_to_cloud_storage)\n", - "5. [Running Analysis on the Cloud](./06_running_analysis)\n", - "6. [Monitoring Costs](./07_monitoring_costs)\n", - "7. [Cleaning up Resources and Best Practices](./08_cleaning_up_resources)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/01_intro_to_cloud_console.ipynb b/content/GCP/01_intro_to_cloud_console.ipynb index 43bdad1..2a50a63 100644 --- a/content/GCP/01_intro_to_cloud_console.ipynb +++ b/content/GCP/01_intro_to_cloud_console.ipynb @@ -169,7 +169,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -183,7 +183,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/02_intro_to_compute_part1.ipynb b/content/GCP/02_intro_to_compute_part1.ipynb index 1a1a1ac..8ef2504 100644 --- a/content/GCP/02_intro_to_compute_part1.ipynb +++ b/content/GCP/02_intro_to_compute_part1.ipynb @@ -261,7 +261,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -275,7 +275,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/04_intro_to_cli.ipynb b/content/GCP/04_intro_to_cli.ipynb index d8c3b55..f14e01e 100644 --- a/content/GCP/04_intro_to_cli.ipynb +++ b/content/GCP/04_intro_to_cli.ipynb @@ -95,7 +95,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "student31@class.internet2.edu\n" ] } @@ -122,7 +122,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "just-armor-301114\n" ] } @@ -229,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "id": "b8f74fe9-0943-4bd9-a91b-da09c001f8a1", "metadata": {}, "outputs": [], @@ -239,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "id": "b937f6c7-3697-4bc0-8e32-da0b0faec593", "metadata": {}, "outputs": [ @@ -293,7 +293,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "cs-730348008320-default-default-w724w\n" + "cs-730348008320-default-default-wwhtf\n" ] } ], @@ -319,7 +319,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tue 26 Oct 2021 07:44:54 PM UTC\n" + "Tue 02 Nov 2021 07:15:33 PM UTC\n" ] } ], @@ -337,7 +337,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Date: Tue 26 Oct 2021 07:44:55 PM UTC\n" + "Date: Tue 02 Nov 2021 07:15:34 PM UTC\n" ] } ], @@ -365,7 +365,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "just-armor-301114\n" ] } @@ -386,7 +386,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n" + "Your active configuration is: [cloudshell-17640]\n" ] } ], @@ -481,7 +481,7 @@ "[metrics]\n", "environment = devshell\n", "\n", - "Your active configuration is: [cloudshell-27180]\n" + "Your active configuration is: [cloudshell-17640]\n" ] } ], @@ -574,7 +574,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "id": "678a1735-8cad-403c-b302-639de1fb0f4b", "metadata": { "tags": [] @@ -612,7 +612,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 19, "id": "b0dec5ed-dc7f-4003-b228-f6605119ce47", "metadata": {}, "outputs": [ @@ -627,33 +627,19 @@ "source": [ "gcloud config list --format=json |jq -r .core.project" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a22e397-9f21-4c7b-914c-88708592eeb1", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/05_intro_to_cloud_storage.ipynb b/content/GCP/05_intro_to_cloud_storage.ipynb index 20d2329..0d826dd 100644 --- a/content/GCP/05_intro_to_cloud_storage.ipynb +++ b/content/GCP/05_intro_to_cloud_storage.ipynb @@ -218,21 +218,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index a6ca11e..074a9f6 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/07_monitoring_costs.ipynb b/content/GCP/07_monitoring_costs.ipynb index c8b0712..2cdbf95 100644 --- a/content/GCP/07_monitoring_costs.ipynb +++ b/content/GCP/07_monitoring_costs.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/08_cleaning_up_resources.ipynb b/content/GCP/08_cleaning_up_resources.ipynb index d1604da..0e0c1c6 100644 --- a/content/GCP/08_cleaning_up_resources.ipynb +++ b/content/GCP/08_cleaning_up_resources.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/intro_to_GCP_Essentials.ipynb b/content/GCP/intro_to_GCP_Essentials.ipynb index 32ded70..4be4197 100644 --- a/content/GCP/intro_to_GCP_Essentials.ipynb +++ b/content/GCP/intro_to_GCP_Essentials.ipynb @@ -21,7 +21,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -35,7 +35,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/jupyter.sh b/jupyter.sh new file mode 100755 index 0000000..901c38e --- /dev/null +++ b/jupyter.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ -r ./local.sh ] ; then + . ./local.sh +fi + +echo "=== Install and run JupyterLab locally" + +echo "+++ installing jupyter" +python3 -m venv .venv +. .venv/bin/activate + +python3 -m pip install pip --upgrade +python3 -m pip install --upgrade jupyterlab + +python3 -m pip install --upgrade bash_kernel +python3 -m bash_kernel.install + +python3 -m pip install --upgrade jupyterlab-spellchecker + +jupyter-lab --port=8080