From ddb1356070ea8c7910ad5249fafec1ec4f636a59 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Tue, 2 Nov 2021 19:52:14 +0000 Subject: [PATCH] Add build tools for Jupyter and .gitignore --- .gitignore | 8 + .vscode/settings.json | 6 + Build.md | 21 + content/.DS_Store | Bin 6148 -> 0 bytes .../intro_to_AWS_Essentials-checkpoint.ipynb | 43 -- content/AWS/images/.DS_Store | Bin 6148 -> 0 bytes ...intro_to_Azure_Essentials-checkpoint.ipynb | 43 -- ...01_intro_to_cloud_console-checkpoint.ipynb | 191 ----- ...02_intro_to_compute_part1-checkpoint.ipynb | 283 -------- .../04_intro_to_cli-checkpoint.ipynb | 661 ------------------ ...05_intro_to_cloud_storage-checkpoint.ipynb | 240 ------- .../intro_to_GCP_Essentials-checkpoint.ipynb | 43 -- content/GCP/01_intro_to_cloud_console.ipynb | 4 +- content/GCP/02_intro_to_compute_part1.ipynb | 4 +- content/GCP/04_intro_to_cli.ipynb | 52 +- content/GCP/05_intro_to_cloud_storage.ipynb | 20 +- content/GCP/06_running_analysis.ipynb | 4 +- content/GCP/07_monitoring_costs.ipynb | 4 +- content/GCP/08_cleaning_up_resources.ipynb | 4 +- content/GCP/intro_to_GCP_Essentials.ipynb | 4 +- jupyter.sh | 21 + 21 files changed, 94 insertions(+), 1562 deletions(-) create mode 100644 .gitignore create mode 100644 .vscode/settings.json create mode 100644 Build.md delete mode 100644 content/.DS_Store delete mode 100644 content/AWS/.ipynb_checkpoints/intro_to_AWS_Essentials-checkpoint.ipynb delete mode 100644 content/AWS/images/.DS_Store delete mode 100644 content/Azure/.ipynb_checkpoints/intro_to_Azure_Essentials-checkpoint.ipynb delete mode 100644 content/GCP/.ipynb_checkpoints/01_intro_to_cloud_console-checkpoint.ipynb delete mode 100644 content/GCP/.ipynb_checkpoints/02_intro_to_compute_part1-checkpoint.ipynb delete mode 100644 content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb delete mode 100644 content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb delete mode 100644 content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb create mode 100755 jupyter.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bcf3611 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.DS_Store/ +.ipynb_checkpoints/ +/.vscode/ +.venv/ +log/ +data/ +_build/ +local* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..4ae7d8a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "files.watcherExclude": { + "**/.venv/**": true + }, + "python.pythonPath": ".venv/bin/python3" +} diff --git a/Build.md b/Build.md new file mode 100644 index 0000000..d9b5a55 --- /dev/null +++ b/Build.md @@ -0,0 +1,21 @@ +# Book Build Instructions + +To build the book for local development run +``` +./build-book.sh +``` + +To copy results to the `gh-pages` branch and push run +``` +./push-book.sh +``` + +## GCP + +Using Jupyter launched from a remote Visual Code Studio session inside GCP Cloud-Shell requires some local settings stored in `local.sh`, use the following as a template replacing the project and email with the proper vaules: +``` +gcloud config set account student31@class.internet2.edu +export GOOGLE_CLOUD_PROJECT=just-armor-301114 +export DEVSHELL_PROJECT_ID=$GOOGLE_CLOUD_PROJECT +gcloud config set project $GOOGLE_CLOUD_PROJECT +``` diff --git a/content/.DS_Store b/content/.DS_Store deleted file mode 100644 index 5c68101d5aa61f10f8e70681f955f0b39c409a83..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK%}xR_5N-jnV2m70@Ystt4*VD+2a^rr!JEk%J*a`*KwuNtCG6rx5wnk>Z{!>J zI?l9(WP={O7$Y;u^qWp++VtB^r)7-s)~r`!ti%`-pop0QG+zjgqpnHDco4aoqt8M& z4kzv?aMz;A@fR7uZg zTh8Bj3*uHV^LwG1$JPTv z6hxs@D$nQjW<@qmYKw}TH|y1kJZc;-76nm0teswT$B$3*=f%sbCYHlLsASLJ65hZ_ z?eW2$`lG-ffPa~@j3p!nhyh|?6BsaOpV--i&C(u;0b<}s4B+`7K@ly3iAHsFK!evu z9M2)5fQ@enL}}16m}rC%5Ux`Jbt*Sc46f6`FHM|fFwv;f8CNUAJZ5EXUMO6x4t}Y^ z8Mic2OAHVLX$G>oTf+K({{8zuokTrifEf5!4DejrZnt1drnata4r{Fh?Si6UT%vK6 k0){HZ5R0Yw08|P1B^rR1!9*i?K2E32%}(%NvbXQJN}G z&&*0$mDTy`nSZ4RejeogtQGW5sCN+LSD~{HwNBpcAUuk@gHCO6SEqRpr`?_@j-xK7 zoF2z%qz5hCPovC?Vfc6K4O6WN(ESjwY8~S~uc$ts{ZMsVkDuf7IBXmU **Compute Engine** (under Compute) -> **VM Instances** -> **+Create Bucket** (just under the blue bar) to open the *Create an instance* page.\n", - " * In the **New VM instance** tab on the left (selected by default) configure the *VM instance* as follows:\n", - " * For **Name**, enter a unique name for the instances (example: \"essentials-instance-1\")\n", - " * For **Region** leave the default or select your \"home\" region. The region is the physical location where your data will reside. Your \"home\" region should be close to your work and the region you use most often.\n", - " * For **Zone** leave the default (note how the name is constructed and that it is a separate data center).\n", - " * In the **Machine configuration** section:\n", - " * For **Series** select *E2* (default). The other options are for types of machines.\n", - " * For **Machine type** select *e2-medium* (default)\n", - " * Leave the remaining options as default.\n", - " * In the **Identity and API access** section note that for the **Service Accounts** -> **Service account** option that the **Compute Engine default service account** is selected. This allows the *VM Instance* to ***access and edit any resources in the project***.\n", - " * Note the **Monthly estimate** in the right side panel. This is only for running the *VM instance* and does not include other storage, networking, compute, and other charges that may occur by running software. Click **Details** show a breakdown of the *VM instance* only costs.\n", - " * Finally, click **Create** to create the *VM Instance* (allocate the resource).\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "41c63432-a614-4a1e-9967-f49b68f9069e", - "metadata": {}, - "source": [ - "### Security\n", - "\n", - "Everything in the cloud requires permission (authorization). Ordinary we would configure and check security first but in the case of exploring services it is often easier to do things out of order. We noted that the *VM instance* was created with the *Compute Engine default service account*, which by *default* has access to all the resources that your Google Account does.\n", - "\n", - "Specifically, we are interested in verifying what permissions that the *Compute Engine default service account* has for your *VM instances* . To do this navigate to the IAM page (**Navigation Menu** -> **IAM & Admin** -> **IAM** -> **Permissions** -> View By: **Principals**)\n", - "\n", - "*Note: There is a powerful filter box to limit the permissions shown.*\n", - "\n", - "In the **Name** column you should see a row with **Compute Engine default service account**. Here you should see the **Editor** role in the Role column. A *role* is a collection of permissions managed by Google or someone else. The **Editor** role for a project will *allow* a **VM instance** to modify resources *in* the **project**. The principal for the *Compute Engine default service account* should start with the *Project ID* of your project. It is possible to create a custom service account with the appropriate roles for the *VM Instance*.\n", - "\n", - "There are three important pieces of information that work together to form the **IAM policy**. The permission (role), the identity (principal or member), and the resource (project)." - ] - }, - { - "cell_type": "markdown", - "id": "da301540-9b02-4b27-ae8e-f301b3dd36f2", - "metadata": { - "tags": [] - }, - "source": [ - "### Follow the Storage Allocation\n", - "\n", - "Just as with security, we will audit (follow) the *VM instance* creation by examining at the project *activity*.\n", - "\n", - "To view the project activity we do the following:\n", - "\n", - " * Navigate to the project **Dashboard** by clicking on \"Google Cloud Platform\" on the left side of the top blue bar.\n", - " * Select **Activity** tab just below the top blue bar.\n", - " * Look for the row with **Create VM** on the top of the table. This shows that the *VM Instance* was created.\n", - " * Expand the detail by clicking the **down arrow** on the right side of the row. You can see what identity created the bucket and the name. The rest of the information details the initial settings that you configured when creating the bucket. The details are not really important at this time, just know they exist.\n", - "\n", - "Note that the activity is on the project, not the account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "3d376f83-966a-4c7d-9ecf-4973e7f00ff7", - "metadata": { - "tags": [] - }, - "source": [ - "### Enumerate the VM Instances\n", - "\n", - "To see an overview of resources in the project navigate to the **dashboard** and you will now see at least **1 instance** in the *resources* card under *Compute Engine*.\n", - "\n", - "Now lets find and connect to the *VM Instance*.\n", - " * Navigate to the Google Compute Engine page by clicking **Navigation Menu** -> **Compute Engine** (under Compute) -> **Instances**.\n", - " * **Find** the *VM instance* you just created. You can use the filter to find a bucket if there are a lot of them.\n", - " * Click on the *VM instance* name to open the **VM instance details** (it will display as a hyperlink when you hover over the *VM instance* name)." - ] - }, - { - "cell_type": "markdown", - "id": "7f4bc715-f7b5-4ab8-b13a-720dae419326", - "metadata": { - "tags": [] - }, - "source": [ - "### Connect to the VM Instance\n", - "\n", - "To connect to the *VM Instance* we will use the cloud shell in the web console. Open up a shell by clicking on the **Activate Cloud Shell** icon in the top blue bar if it is not already open.\n", - "\n", - "We first verify that the cloud shell is using the correct identity and *project id* by looking at the welcome message and the prompt.\n", - "\n", - "![cloud-shell](img/cloud-shell.png)\n", - "\n", - "To connect to the *VM instance* we enter the following command in the cloud shell:\n", - "```\n", - "gcloud compute ssh essentials-instance-1\n", - "```\n", - "\n", - "If you have not used the cloud shell to connect to a *VM Instance* before you will probably be asked to create a new *ssh key*. The Compute Engine will use this key to allow you to access the *VM instance* in a secure manner. If this is the case you will see a message similar to the following:\n", - "```\n", - "WARNING: The private SSH key file for gcloud does not exist.\n", - "WARNING: The public SSH key file for gcloud does not exist.\n", - "WARNING: You do not have an SSH key for gcloud.\n", - "WARNING: SSH keygen will be executed to generate a key.\n", - "This tool needs to create the directory [/home/student231/.ssh] before\n", - " being able to generate SSH keys.\n", - "\n", - "Do you want to continue (Y/n)?\n", - "```\n", - "\n", - "Press enter to continue to create the key. You will see another message asking you for a passphrase, similar to the following:\n", - "```\n", - "Generating public/private rsa key pair.\n", - "Enter passphrase (empty for no passphrase):\n", - "```\n", - "Here you should enter a passphrase (a series of words) to protect the key. You will be asked to re-enter the passphrase as shown below:\n", - "```\n", - "Enter same passphrase again:\n", - "```\n", - "\n", - "After successfully retyping the passphrase the key generation process will continue showing similar message as follows (the fingerprint and art will look different):\n", - "```\n", - "Your identification has been saved in /home/student231/.ssh/google_compute_engine.\n", - "Your public key has been saved in /home/student231/.ssh/google_compute_engine.pub.\n", - "The key fingerprint is:\n", - "SHA256:D9yj1nZEUigahirvjkIFoCP7RfjDu67Ku88M7nHQftI student231@cs-748001990186-default-boost-r9phq\n", - "The key's randomart image is:\n", - "+---[RSA 2048]----+\n", - "|o . .. |\n", - "|.. .. o . .. |\n", - "|+ o... o .. . |\n", - "|ooo= .. . o |\n", - "|.+..= S o . |\n", - "| o+..o = o |\n", - "|.+.+.E o + . |\n", - "|+.B o. . . . |\n", - "|=O**o |\n", - "+----[SHA256]-----+\n", - "```\n", - "\n", - "The ssh key generation is now complete. This will only happen once for your account and you will start here on future connections.\n", - "\n", - "At this point the command will attempt to connect to the *VM Instance* and will ask the following question:\n", - "```\n", - "Did you mean zone [us-central1-a] for instance: [instance-1] (Y/n)? n\n", - "```\n", - "Answer \"n\".\n", - "\n", - "The command will now configure the instance to allow your ssh key and connect to it.\n", - "\n", - "```\n", - "No zone specified. Using zone [us-central1-a] for instance: [instance-1].\n", - "Updating project ssh metadata...working..Updated [https://www.googleapis.com/compute/v1/projects/class-essentials-gcp-demo-1].\n", - "Updating project ssh metadata...done.\n", - "Waiting for SSH key to propagate.\n", - "Warning: Permanently added 'compute.74517428106645607' (ECDSA) to the list of known hosts.\n", - "```\n", - "\n", - "Once connected you will see the machine login banner and prompt similar to the following:\n", - "```\n", - "Linux instance-1 4.19.0-17-cloud-amd64 #1 SMP Debian 4.19.194-3 (2021-07-18) x86_64\n", - "\n", - "The programs included with the Debian GNU/Linux system are free software;\n", - "the exact distribution terms for each program are described in the\n", - "individual files in /usr/share/doc/*/copyright.\n", - "\n", - "Debian GNU/Linux comes with ABSOLUTELY NO WARRANTY, to the extent\n", - "permitted by applicable law.\n", - "student231@essentials-instance-1:~$\n", - "```\n", - "\n", - "Congratulations you have now created a *VM Instance* and connected to it.\n", - "\n", - "Feel free to interact with the machine for a while and type\n", - "```\n", - "exit\n", - "```\n", - "when you are done.\n", - "\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "0362e76c-3f03-4299-8217-24a2f39ee503", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Delete the Storage - Exercise \n", - "\n", - "We will now complete the resource life-cycle by deleting the *VM Instance* we just created. Think about the process we did for creating the bucket and follow a similar one. \n", - "\n", - "Did you \"Follow\" the bucket by looking at the **activity** page?\n", - "\n", - "Since we care about paying for resources we are not using we **review** by visiting the **dashboard** and reviewing that we no longer have any buckets, if this was our only bucket. " - ] - }, - { - "cell_type": "markdown", - "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", - "metadata": {}, - "source": [ - "### References\n", - " * Compute Engine Permissions (IAM): https://cloud.google.com/compute/docs/access/iam" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb deleted file mode 100644 index d8c3b55..0000000 --- a/content/GCP/.ipynb_checkpoints/04_intro_to_cli-checkpoint.ipynb +++ /dev/null @@ -1,661 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "5439e525-b985-495f-85a6-e4c8d7452956", - "metadata": {}, - "source": [ - "## Introduction to the gcloud CLI\n", - "\n", - "Learner Questions\n", - " * How do I use the Cloud CLI?\n", - " * How do I use `gcloud`?\n", - "\n", - "Learning Objectives\n", - " * Find and use the \"Cloud Shell\" in the web console.\n", - " * Use basic cloud CLI commands (`gcloud` and `gsutil`).\n", - " * Verify basic settings.\n", - " * Use environment variables for configuration.\n", - " * Understand the importance of using variables for configuration.\n", - " * Recognize the value of reproducibility and automation.\n", - " \n", - "Resources\n", - " * `gcloud` documentation: https://cloud.google.com/sdk/gcloud\n", - " * `gsutil` documentation: https://cloud.google.com/storage/docs/gsutil\n", - " * Instructions to install `gcloud` and `gsutil` on your local machine: https://cloud.google.com/sdk/docs/install" - ] - }, - { - "cell_type": "markdown", - "id": "c4e1a4c5-7d28-483d-b27c-ef2afa4fb7e5", - "metadata": {}, - "source": [ - "### Cloud Automation\n", - "\n", - "Now that Drew can create a bucket in the web console they wish to learn how to use tools to automate the process. Drew noticed that the web console could be used to upload and share objects but wishes to learn the command line so that they can automate the and document the process (it is much easier to document commands then describing and showing mouse clicks).\n", - "\n", - "The cloud can be controlled using a Command Line Interface (CLI) or a programming language such as Python. Collectively these tools interact with the cloud over a Application Programming Interface (API) and this capability forms the basis of the cloud, the ability to control infrastructure programmatically.\n", - "\n", - "Just as with navigating the web console it is important to know the **who**, **what**, and **where** of CLI access to reduce the possibility of access mistakes. We will first verify the tools are installed and configured correctly. Next we get the Account being used (who) and the Project ID of the active project (where) using the `gcloud` command. We will then use the `gcloud` and `gsutil` commands to list some public Buckets (what).\n", - "\n", - "The `gcloud` command is used to control most aspects of GCP and the `gsutil` command is used to control Google Cloud Storage Buckets. To access the manual pages for a command just add `--help` to the end of the command or run `gcloud help` for more information.\n", - "\n", - "When running CLI commands you may get a dialog box to authorize the command. You should \"Authorize\" the request to allow the Cloud Shell to access the Project using your (Google) Account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "ce598878-1b1c-4e6b-9122-e679e84c7dbb", - "metadata": {}, - "source": [ - "### Open a Cloud Shell\n", - "\n", - "The Cloud Shell is an easy way to access Comand Line Interface (CLI) commands in a web browser. Under the hood Cloud Shell is Debian running in a container with 5GB of no-charge persistent storage with most of the Google uti and a lot of other useful tools (git, docker, jq, Docker, etc.) installed by default.\n", - "\n", - "To use the Cloud Shell:\n", - " * Open up a Cloud Shell by clicking on the **Activate Cloud Shell** icon in the right side of the top blue bar. If a \"Authorize Cloud Shell\" dialog appears click \"Authorize\" to allow the Cloud Shell to access the project.\n", - " * Click the **Maximize** icon on the right side of the Cloud Shell bar. You can find the correct icon by hovering over the icons to see their names.\n", - " * If a shell is associated with a project it will show it in the tab and in **yellow text** in round parenthesis in the shell prompt. **Verify the Project ID**.\n", - " * To open up a shell with a specific project click the \"down arrow\" icon next to the plus(+) to open a new tab with a specific project. Selecting the plus (+) will usually open up a shell with the default project.\n", - " * Note: You can open the Cloud Shell in a new tab by clicking on icon **Open in a New Window** on the far right of the Cloud Shell bar.\n", - "\n", - "\n", - "#### Opening Cloud Shell in a Tab - Optional\n", - "\n", - " * Click the **Open in a New Window** icon on the far left of the \"Cloud Shell\" tab on the bottom to open up in a separate tab. Hover the mouse over the icons to see their name/function. **Note:** you may need to allow popups or enable popups for this website depending on the browser you use. You can also pull up the tab to increase the shell screen size.\n", - " * Click the **\"Close editor\"** button (the pencil icon in the top right of the window).\n", - "\n", - "#### Advanced call-out\n", - "You can access the cloud shell from your laptop via `gcloud cloud-shell ssh` if you have the [Google Cloud SDK installed](https://cloud.google.com/sdk/docs/install). Installing the Google Cloud SDK will also allow you to run `gcloud` and `gsutil` commands directly from your laptop, workstation, or cluster. *Pro-tip, you cannot use a service account to access the cloud shell.*" - ] - }, - { - "cell_type": "markdown", - "id": "e4fa29a6-7d8e-4591-af3f-8539b94b3bef", - "metadata": {}, - "source": [ - "### Verify the Configuration (Who, What, Where)" - ] - }, - { - "cell_type": "markdown", - "id": "f786f92c-7127-4f26-b0c0-fc27a364aca4", - "metadata": {}, - "source": [ - "First, let's verify that the Account being used for access (who) is what we expect." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "3ce9e6fb-9f5f-42e1-8512-35c0564f8e85", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "student31@class.internet2.edu\n" - ] - } - ], - "source": [ - "gcloud config get-value account" - ] - }, - { - "cell_type": "markdown", - "id": "68b97adf-7378-4e9c-b2b8-2075f043afab", - "metadata": {}, - "source": [ - "Next, show verify the active project (where)." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "cfb10587-e007-45ae-839f-7bcfe088c711", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config get-value project" - ] - }, - { - "cell_type": "markdown", - "id": "c2972d7b-f393-42b5-8330-cf8292d28afb", - "metadata": {}, - "source": [ - "Now we will use `gcloud` to list a well known public bucket (what). " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "617325c9-d853-4291-a1db-938ab9439fee", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "gs://gcp-public-data-landsat/index.csv.gz\n", - "gs://gcp-public-data-landsat/LC08/\n", - "gs://gcp-public-data-landsat/LE07/\n", - "gs://gcp-public-data-landsat/LM01/\n", - "gs://gcp-public-data-landsat/LM02/\n", - "gs://gcp-public-data-landsat/LM03/\n", - "gs://gcp-public-data-landsat/LM04/\n", - "gs://gcp-public-data-landsat/LM05/\n", - "gs://gcp-public-data-landsat/LO08/\n", - "gs://gcp-public-data-landsat/LT04/\n", - "gs://gcp-public-data-landsat/LT05/\n", - "gs://gcp-public-data-landsat/LT08/\n" - ] - } - ], - "source": [ - "gcloud alpha storage ls gs://gcp-public-data-landsat" - ] - }, - { - "cell_type": "markdown", - "id": "42e6ad5b-186d-4cd1-ba03-e7c85ad40e38", - "metadata": {}, - "source": [ - "*Advanced Callout: The `alpha` (and `beta`) command allows us to access commands that have not been released for production and care should be taken when using these in a production environment. At this time this is not the recommended way to access storage buckets, but it does help verify that everything is working correctly.*" - ] - }, - { - "cell_type": "markdown", - "id": "1389ca4f-7234-4ea3-9ad8-85914d88ede5", - "metadata": {}, - "source": [ - "Finally, we will verify that the separate and preferred `gsutil` command is installed and working by listing the same well known public bucket. " - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "1a13a344-65df-40bc-997f-27d9084f69cf", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "gs://gcp-public-data-landsat/index.csv.gz\n", - "gs://gcp-public-data-landsat/LC08/\n", - "gs://gcp-public-data-landsat/LE07/\n", - "gs://gcp-public-data-landsat/LM01/\n", - "gs://gcp-public-data-landsat/LM02/\n", - "gs://gcp-public-data-landsat/LM03/\n", - "gs://gcp-public-data-landsat/LM04/\n", - "gs://gcp-public-data-landsat/LM05/\n", - "gs://gcp-public-data-landsat/LO08/\n", - "gs://gcp-public-data-landsat/LT04/\n", - "gs://gcp-public-data-landsat/LT05/\n", - "gs://gcp-public-data-landsat/LT08/\n" - ] - } - ], - "source": [ - "gsutil ls gs://gcp-public-data-landsat" - ] - }, - { - "cell_type": "markdown", - "id": "598b5d12-9bea-49de-a772-4113727e0859", - "metadata": {}, - "source": [ - "### Environment Variables\n", - "\n", - "When using the CLI you quickly notice that you have to enter many of the details, such as the Project ID, many times (Drew is also Dyslexic, which makes it even harder for them). Not only is this tedious, it also makes moving to another project or using another bucket difficult making reusability and collaboration painful. We will demonstrate the use environment variables to address this issue. Most cloud environments and tools rely heavily on this technique.\n", - "\n", - "*To make the point even clearer, we use environment variables in the Cloud Essentials notebooks instead of hard coding them, which makes it easy for anyone to run, use, and edit and thus collaborate and contribute to them!*\n", - "\n", - "We now demonstrate the basic use environment variables. As a reminder, environment variables are used to store configuration information that can be easily passed between programs and programming languages." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b8f74fe9-0943-4bd9-a91b-da09c001f8a1", - "metadata": {}, - "outputs": [], - "source": [ - "TEST=1" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "b937f6c7-3697-4bc0-8e32-da0b0faec593", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n" - ] - } - ], - "source": [ - "echo $TEST" - ] - }, - { - "cell_type": "markdown", - "id": "78fe7013-a5ae-498f-aeab-ba1f3c710de1", - "metadata": {}, - "source": [ - "Although we do not always need to, we should always enclose shell variables in double quotes and enclose the variable name in curly braces (funny things may happen otherwise). In most cases it is easier to do this every time than try to figure out when it is possible to use the short form. We only use the short form (`$TEST`) when it is used frequently and known to always work, in our case it is `$PROJECT` and `$BUCKET`. When in doubt, use the explicit form.\n", - "\n", - "There are a number of environment variables already set that contain useful information, for example:" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "bae0587c-0379-40e3-9c31-16ba5505487a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "student31\n" - ] - } - ], - "source": [ - "echo \"${USER}\"" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "52b4add5-0d18-4faf-a304-b7e0ee8ba82e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "cs-730348008320-default-default-w724w\n" - ] - } - ], - "source": [ - "echo \"${HOSTNAME}\"" - ] - }, - { - "cell_type": "markdown", - "id": "ef895f38-ad16-4f2d-aa18-ec913732dd69", - "metadata": {}, - "source": [ - "We can also use the output of commands to set values." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "776806e4-209f-45fa-b14e-f771f5d5089d", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Tue 26 Oct 2021 07:44:54 PM UTC\n" - ] - } - ], - "source": [ - "date" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "560fde63-0eb8-4c52-9787-4debfe35ec23", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Date: Tue 26 Oct 2021 07:44:55 PM UTC\n" - ] - } - ], - "source": [ - "echo \"Date: $(date)\"" - ] - }, - { - "cell_type": "markdown", - "id": "6323f7bf-b2fe-4c94-a87a-2d35e75a1a49", - "metadata": { - "tags": [] - }, - "source": [ - "Now lets combine all this to set the `PROJECT` environment variable to the default project so we do not need to use the actual value in future commands." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "18f40620-8537-42b5-948f-dd581e604a36", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n", - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config get-value project" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "a959f857-8a20-4356-bc30-dbf8c75be96d", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Your active configuration is: [cloudshell-27180]\n" - ] - } - ], - "source": [ - "PROJECT=\"$(gcloud config get-value project)\"" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "c1d27e89-cc69-4979-82c4-79c3fb36bccb", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "echo $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "9d3a8504-a75c-4945-ab54-85f1d9a7e2b2", - "metadata": {}, - "source": [ - "Note the \"Your active configuration is: ...\" line is a \"warning\" and not actually part of the output of the program." - ] - }, - { - "cell_type": "markdown", - "id": "c25e6431-4f92-454a-bd10-e77cb28deec6", - "metadata": {}, - "source": [ - "Now use the `PROJECT` envrionment variable to get more information about the active project." - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "31b01f78-94b1-44f0-a2af-d657058941b2", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "createTime: '2021-01-08T14:17:35.831Z'\n", - "lifecycleState: ACTIVE\n", - "name: CLASS Project\n", - "projectId: just-armor-301114\n", - "projectNumber: '1002111293252'\n" - ] - } - ], - "source": [ - "gcloud projects describe $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "ed2ab9ce-6b0a-4939-88c7-8103e29413b7", - "metadata": {}, - "source": [ - "Note the extraneous message about the active configuration produced by the `gcloud config get-value project` command. A slightly different way to access the data can be used to remove the extraneous message by using the following command:" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "845b2aaf-0032-4043-a40a-d2482693d23e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[accessibility]\n", - "screen_reader = True\n", - "[component_manager]\n", - "disable_update_check = True\n", - "[compute]\n", - "gce_metadata_read_timeout_sec = 30\n", - "[core]\n", - "account = student31@class.internet2.edu\n", - "disable_usage_reporting = True\n", - "project = just-armor-301114\n", - "[metrics]\n", - "environment = devshell\n", - "\n", - "Your active configuration is: [cloudshell-27180]\n" - ] - } - ], - "source": [ - "gcloud config list" - ] - }, - { - "cell_type": "markdown", - "id": "d919b26d-2f64-4a43-9c76-34f7798fe853", - "metadata": {}, - "source": [ - "We can now use the `--format='value()'` option to get specific values. The following command returns the project number without the extraneous message." - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "9714ebe2-2921-498b-9e0e-69ec6a85896a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config list --format='value(core.project)'" - ] - }, - { - "cell_type": "markdown", - "id": "ba660d2b-5ddf-449b-8049-56a40f2e366b", - "metadata": {}, - "source": [ - "Note where the `core` part comes in the output of the `gcloud config list` command." - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "7d803ddd-acca-4ad1-933a-4d6e6044def6", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "PROJECT=\"$(gcloud config list --format='value(core.project)')\"\n", - "echo $PROJECT" - ] - }, - { - "cell_type": "markdown", - "id": "d40b2c7b-7331-412d-80e1-f1a77204075a", - "metadata": {}, - "source": [ - "We will use this form from now on. Since we know the Project ID only contains numbers, letters, and dashes with no spaces there is no need to use quotes." - ] - }, - { - "cell_type": "markdown", - "id": "270e0abb-d5e9-476a-af91-060cc0ea0349", - "metadata": {}, - "source": [ - "### Exercise\n", - " * What command shows the project number? (hint: we used it already)\n", - " * What is the name of this variable? (hint: it is case sensitive)\n", - " * What command shows *just* the Project Number (hint: use `--format`)\n", - " * Set the `PROJECT_NUMBER` environment variable with the correct value. (hint: be careful with punctuation (single and double quotes) and matching brackets and punctuation)" - ] - }, - { - "cell_type": "markdown", - "id": "bda44ee1-337a-447d-8377-e8f7ae563316", - "metadata": {}, - "source": [ - "### Advanced Call-Out: Formating Output.\n", - "\n", - "The following example shows how to output the configuration as `JSON`. The command `jq` can also be used to parse the JSON data." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "678a1735-8cad-403c-b302-639de1fb0f4b", - "metadata": { - "tags": [] - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"accessibility\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"screen_reader\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"component_manager\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"disable_update_check\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"compute\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"gce_metadata_read_timeout_sec\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"30\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"core\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"account\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"student31@class.internet2.edu\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"disable_usage_reporting\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"True\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"project\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"just-armor-301114\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", - " \u001b[0m\u001b[34;1m\"metrics\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", - " \u001b[0m\u001b[34;1m\"environment\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"devshell\"\u001b[0m\u001b[1;39m\n", - " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", - "\u001b[1;39m}\u001b[0m\n" - ] - } - ], - "source": [ - "gcloud config list --format=json |jq" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "b0dec5ed-dc7f-4003-b228-f6605119ce47", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "just-armor-301114\n" - ] - } - ], - "source": [ - "gcloud config list --format=json |jq -r .core.project" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a22e397-9f21-4c7b-914c-88708592eeb1", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb deleted file mode 100644 index 20d2329..0000000 --- a/content/GCP/.ipynb_checkpoints/05_intro_to_cloud_storage-checkpoint.ipynb +++ /dev/null @@ -1,240 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "fb98ae89-8494-4fd7-aa97-44380424d68d", - "metadata": {}, - "source": [ - "## Introduction to Google Cloud Storage\n", - "\n", - "Learner Questions\n", - " * How do I store data in the cloud?\n", - "\n", - "Learning Objectives:\n", - " * Navigate the Google Cloud Storage service and terminology\n", - " * Understand the roles and permissions needed to use Google Cloud Storage in projects\n", - " * Allocate storage in Google Cloud Storage\n", - " * Find the cost estimator for Google Cloud Storage\n", - " * Recognize that resources have a \"location\"\n", - " * Recognize danger of public access\n", - " * De-allocate Google Cloud Storage storage" - ] - }, - { - "cell_type": "markdown", - "id": "9897048a-6aa8-4d85-a557-d85b802f3f1d", - "metadata": {}, - "source": [ - "Drew, after working with a Research Computing and Data (RCD) facilitator, has decided to start their cloud journey with storage. Data is usually the center of a research project and it makes sense to start here.\n", - "\n", - "One of the most common and economic ways to store data in the cloud is to use object storage. In GCP object storage is called *Google Cloud Storage*, which is similar to the Simple Storage Service, also known as S3, on Amazon Web Services (AWS). For object storage, information is stored as a collection of key-value pairs." - ] - }, - { - "cell_type": "markdown", - "id": "739746ba-1c95-4d1e-94c7-1e5330d17f52", - "metadata": { - "tags": [] - }, - "source": [ - "### Google Cloud Storage\n", - "\n", - "To learn more about the Google Cloud Storage service we will use the web console to create, explore, and destroy a Bucket. The ability to quickly create, explore, and then discard resources in the web-console is a powerful tool to explore a service. After an initial exploration with the web-console, programmatic access using scripts and programming languages should be used.\n", - "\n", - "Unfortunately, the terminology used for object storage differs from one commercial cloud provider to another. Google Cloud Storage uses the terms Bucket and Object. Data is stored in Objects and Objects are stored in Buckets. Objects (data) are stored as a key-value pair, which is similar to a Python dictionary but persistent. Values are stored and retrieved using a unique key and the value can contain any information (more on this later).\n", - "\n", - "We now take Drew through the process of creating a Google Cloud Storage bucket." - ] - }, - { - "cell_type": "markdown", - "id": "4b1d2914-cac0-465e-b61a-11bbc243cc8c", - "metadata": { - "tags": [] - }, - "source": [ - "### Security\n", - "\n", - "Everything in the cloud requires permission (authorization). Let's first verify that we have the permissions to create a bucket. A Bucket (a resource) is created within a project and inheres permissions from it.\n", - "\n", - "We are interested in what permissions that *your* account has for *your* project. To do this navigate to the IAM page (**Navigation Menu -> IAM & Admin -> IAM -> Permissions -> View By: Principals**). This shows the permissions for the project.\n", - "\n", - "*Note: There is a powerful filter box to limit the permissions shown.*\n", - "\n", - "You should see a row with your account shown in the Principal column. Here you should see the \"Owner\" Role in the Role column. A *role* is a collection of permissions managed by Google or someone else. The **Owner** or the **Storage Admin** role for a project will *allow* *you* to create, access, and delete Buckets *in* the project.\n", - "\n", - "There are three important pieces of information that work together to form the **IAM policy**. The permission (role), the identity (principal or member), and the resource (project)." - ] - }, - { - "cell_type": "markdown", - "id": "c5430b40-1a5f-40df-9e13-529ef3ece4ce", - "metadata": { - "tags": [] - }, - "source": [ - "### Allocate Google Cloud Storage\n", - "\n", - "Now that we have verified the permissions we can now create a bucket. Buckets are where objects are stored and have a globally unique name.\n", - "\n", - "To create a bucket we do the following:\n", - " * Click **Navigation Menu** -> **Cloud Storage** (under Storage) -> **Browser** -> **+Create Bucket** (just under the blue bar) to open the *Create a bucket* page.\n", - " * In *Name your bucket*:\n", - " * For **Name**, enter a globally unique name for the bucket (example \"essentials-test-myname-2021-01-01\")\n", - " * Click **Continue**\n", - " * In *Choose where to store your data*:\n", - " * For *Location Type* select **Region** (cheapest and fastest)\n", - " * For *Location* leave the default or select your \"home\" region. The region is the physical location where your data will reside. Your \"home\" region should be close to your work and the region you use most often.\n", - " * Click **Continue**\n", - " * In *Choose a default storage class for your data*:\n", - " * Select **Standard** (default). The other options are for long-term storage.\n", - " * Click **Continue**\n", - " * In *Choose how to control access to objects*:\n", - " * Under *Prevent Public Access* check \"**Enforce public access prevention on this bucket**\". This will try to prevent others from accessing your data. \n", - " * Under *Access Control* Select **Uniform** (default). *Uniform* permissions are on the entire Bucket. *Fine-grained* permissions are on the objects.\n", - " * Select **Continue**\n", - " * Note the **monthly cost estimate** in the right side panel. This is only for reading, writing, and storing data and does not include networking, compute, and other charges that may occur by using the data.\n", - " * *Storage size* is the total amount of data stored.\n", - " * *Data Retrieval* is only used for for long-term, non *standard*, storage.\n", - " * *Operations* is the number of \"operations\" performed on an object, for example reading and writing an object. Storing and accessing many objects require many operations.\n", - " * Finally, click **Create** to create the bucket (allocate the resource).\n", - " " - ] - }, - { - "cell_type": "markdown", - "id": "da301540-9b02-4b27-ae8e-f301b3dd36f2", - "metadata": { - "tags": [] - }, - "source": [ - "### Follow the Storage Allocation\n", - "\n", - "Just as with security, we will audit (follow) the bucket creation by examining at the project *activity*.\n", - "\n", - "To view the project activity we do the following:\n", - "\n", - " * Navigate to the project **Dashboard** by clicking on \"Google Cloud Platform\" on the left side of the top blue bar.\n", - " * Select **Activity** tab just below the top blue bar.\n", - " * Look for the row with **Create Bucket** on the top of the table. This shows that the bucket was created.\n", - " * Expand the detail by clicking the **down arrow** on the right side of the row. You can see what identity created the bucket and the name. The rest of the information details the initial settings that you configured when creating the bucket. The details are not really important at this time, just know they exist.\n", - "\n", - "Note that the activity is on the project, not the account.\n" - ] - }, - { - "cell_type": "markdown", - "id": "c37dbf70-db45-44b3-92b6-2c89d772ec24", - "metadata": { - "tags": [] - }, - "source": [ - "### Enumerate the Buckets\n", - "\n", - "Now lets find and examine the bucket. To view a bucket we do the following:\n", - "\n", - " * Navigate to the Google Storage page by clicking **Navigation Menu** -> **Cloud Storage** (under Storage) -> **Browser**. \n", - " * **Find** the bucket you just created. You can use the filter to find a bucket if there are a lot of them.\n", - " * Click on the bucket name to open the **bucket details** (it will display as a hyperlink when you hover over the bucket name).\n", - "\n", - "Navigate to the **dashboard** and you will now see at least **1 bucket** in the *resources* card under storage." - ] - }, - { - "cell_type": "markdown", - "id": "258044b3-550e-4a4f-a650-27cd0d6dd057", - "metadata": { - "tags": [] - }, - "source": [ - "### Review what is Important\n", - "\n", - "It is always important to review what is important to you. It may be cost, or keeping the data secure. Later on we will show how to monitor overall costs.\n", - "\n", - "For Drew, we will review that the bucket **public access** is *not public* by doing the following:\n", - " * Go to **Navigation Menu -> Cloud Storage -> Browser**\n", - " * Select the bucket of interest by **checking the box** next to the Bucket name.\n", - " * In the Right Side Bar (open if necessary) in the **Permissions** tab in the **Public Access** card you should see **Not Public**. This means that public access prevention is turned on.\n", - " * You can also see the **permissions** for the bucket in the bottom of the bar." - ] - }, - { - "cell_type": "markdown", - "id": "0362e76c-3f03-4299-8217-24a2f39ee503", - "metadata": { - "jp-MarkdownHeadingCollapsed": true, - "tags": [] - }, - "source": [ - "### Delete the Storage - Exercise \n", - "\n", - "We will now complete the resource life-cycle by deleting the bucket we just created. Think about the process we did for creating the bucket and follow a similar one. \n", - "\n", - "Did you \"Follow\" the bucket by looking at the **activity** page?\n", - "\n", - "Since we care about paying for resources we are not using we **review** by visiting the **dashboard** and reviewing that we no longer have any buckets, if this was our only bucket. " - ] - }, - { - "cell_type": "markdown", - "id": "3a28e28d-1d70-44fa-a952-4f3506ea85ec", - "metadata": {}, - "source": [ - "### Discussion\n", - "\n", - "* What does the words \"Secure\", \"Allocate\", \"Follow\", and \"Enumerate\" Spell?\n", - "* Why do you think we use this framework?" - ] - }, - { - "cell_type": "markdown", - "id": "97d7ebc5-4a81-4f1a-aaf3-517adf70640a", - "metadata": {}, - "source": [ - "### Resources in Google Cloud Platform - Optional\n", - "\n", - "Even though we only covered the Google Cloud Storage service in this episode, this process can be used for other *resources* allocated in the cloud. The term *resource* is used for the \"things\" that live in a Project, such as compute, storage, and networking and other services. Resources have the following characteristics:\n", - "\n", - " * Resources are like \"objects\" and usually have their own product/service page in the navigation menu. \n", - " * Resources have many properties (how it is configured), costs, permissions, and almost always belong to a Project.\n", - " * Resource creation permission is at the parent (Project).\n", - " * Resources live in a machine somewhere, and usually specify a location (Region and Zone).\n", - " * Resource creation has limits (Quota) that can be changed (up or down).\n", - " * Resources usually have multiple charges associated with it (even if they are `$0.00`).\n", - " * Resources have a beginning, middle, and an end. Don't forget the end. Have a plan!" - ] - }, - { - "cell_type": "markdown", - "id": "85c697a1-25bc-4b57-9ab6-7c38b6dbbc89", - "metadata": {}, - "source": [ - "\n", - "### References\n", - " * API reference https://cloud.google.com/storage/docs/reference/libraries\n", - " * Permissions (IAM): https://cloud.google.com/storage/docs/access-control/iam-permissions" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb b/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb deleted file mode 100644 index 32ded70..0000000 --- a/content/GCP/.ipynb_checkpoints/intro_to_GCP_Essentials-checkpoint.ipynb +++ /dev/null @@ -1,43 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21ff6943", - "metadata": {}, - "source": [ - "# Google Cloud Platform (GCP)\n", - "\n", - "The modules in this lesson are: \n", - "\n", - "1. [Introduction to the GCP Cloud Console](./01_intro_to_cloud_console)\n", - "2. [Introduction to Cloud Compute](./02_intro_to_compute_part1)\n", - "3. [Introduction to the Cloud CLI](./04_intro_to_cli)\n", - "4. [Introduction to Cloud Storage](./05_intro_to_cloud_storage)\n", - "5. [Running Analysis on the Cloud](./06_running_analysis)\n", - "6. [Monitoring Costs](./07_monitoring_costs)\n", - "7. [Cleaning up Resources and Best Practices](./08_cleaning_up_resources)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/GCP/01_intro_to_cloud_console.ipynb b/content/GCP/01_intro_to_cloud_console.ipynb index 43bdad1..2a50a63 100644 --- a/content/GCP/01_intro_to_cloud_console.ipynb +++ b/content/GCP/01_intro_to_cloud_console.ipynb @@ -169,7 +169,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -183,7 +183,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/02_intro_to_compute_part1.ipynb b/content/GCP/02_intro_to_compute_part1.ipynb index 1a1a1ac..8ef2504 100644 --- a/content/GCP/02_intro_to_compute_part1.ipynb +++ b/content/GCP/02_intro_to_compute_part1.ipynb @@ -261,7 +261,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -275,7 +275,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/04_intro_to_cli.ipynb b/content/GCP/04_intro_to_cli.ipynb index d8c3b55..f14e01e 100644 --- a/content/GCP/04_intro_to_cli.ipynb +++ b/content/GCP/04_intro_to_cli.ipynb @@ -95,7 +95,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "student31@class.internet2.edu\n" ] } @@ -122,7 +122,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "just-armor-301114\n" ] } @@ -229,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 5, "id": "b8f74fe9-0943-4bd9-a91b-da09c001f8a1", "metadata": {}, "outputs": [], @@ -239,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "id": "b937f6c7-3697-4bc0-8e32-da0b0faec593", "metadata": {}, "outputs": [ @@ -293,7 +293,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "cs-730348008320-default-default-w724w\n" + "cs-730348008320-default-default-wwhtf\n" ] } ], @@ -319,7 +319,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tue 26 Oct 2021 07:44:54 PM UTC\n" + "Tue 02 Nov 2021 07:15:33 PM UTC\n" ] } ], @@ -337,7 +337,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Date: Tue 26 Oct 2021 07:44:55 PM UTC\n" + "Date: Tue 02 Nov 2021 07:15:34 PM UTC\n" ] } ], @@ -365,7 +365,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n", + "Your active configuration is: [cloudshell-17640]\n", "just-armor-301114\n" ] } @@ -386,7 +386,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Your active configuration is: [cloudshell-27180]\n" + "Your active configuration is: [cloudshell-17640]\n" ] } ], @@ -481,7 +481,7 @@ "[metrics]\n", "environment = devshell\n", "\n", - "Your active configuration is: [cloudshell-27180]\n" + "Your active configuration is: [cloudshell-17640]\n" ] } ], @@ -574,7 +574,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 18, "id": "678a1735-8cad-403c-b302-639de1fb0f4b", "metadata": { "tags": [] @@ -612,7 +612,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 19, "id": "b0dec5ed-dc7f-4003-b228-f6605119ce47", "metadata": {}, "outputs": [ @@ -627,33 +627,19 @@ "source": [ "gcloud config list --format=json |jq -r .core.project" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8a22e397-9f21-4c7b-914c-88708592eeb1", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/05_intro_to_cloud_storage.ipynb b/content/GCP/05_intro_to_cloud_storage.ipynb index 20d2329..0d826dd 100644 --- a/content/GCP/05_intro_to_cloud_storage.ipynb +++ b/content/GCP/05_intro_to_cloud_storage.ipynb @@ -218,21 +218,15 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" + "display_name": "Bash", + "language": "bash", + "name": "bash" }, "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" } }, "nbformat": 4, diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index a6ca11e..074a9f6 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/07_monitoring_costs.ipynb b/content/GCP/07_monitoring_costs.ipynb index c8b0712..2cdbf95 100644 --- a/content/GCP/07_monitoring_costs.ipynb +++ b/content/GCP/07_monitoring_costs.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/08_cleaning_up_resources.ipynb b/content/GCP/08_cleaning_up_resources.ipynb index d1604da..0e0c1c6 100644 --- a/content/GCP/08_cleaning_up_resources.ipynb +++ b/content/GCP/08_cleaning_up_resources.ipynb @@ -38,7 +38,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +52,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/content/GCP/intro_to_GCP_Essentials.ipynb b/content/GCP/intro_to_GCP_Essentials.ipynb index 32ded70..4be4197 100644 --- a/content/GCP/intro_to_GCP_Essentials.ipynb +++ b/content/GCP/intro_to_GCP_Essentials.ipynb @@ -21,7 +21,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -35,7 +35,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.5" + "version": "3.7.3" } }, "nbformat": 4, diff --git a/jupyter.sh b/jupyter.sh new file mode 100755 index 0000000..901c38e --- /dev/null +++ b/jupyter.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +if [ -r ./local.sh ] ; then + . ./local.sh +fi + +echo "=== Install and run JupyterLab locally" + +echo "+++ installing jupyter" +python3 -m venv .venv +. .venv/bin/activate + +python3 -m pip install pip --upgrade +python3 -m pip install --upgrade jupyterlab + +python3 -m pip install --upgrade bash_kernel +python3 -m bash_kernel.install + +python3 -m pip install --upgrade jupyterlab-spellchecker + +jupyter-lab --port=8080