From 508ece27b55e5b29b5549bdcd6a0c22edf4e9d04 Mon Sep 17 00:00:00 2001 From: Timothy Middelkoop Date: Thu, 2 Dec 2021 19:40:41 +0000 Subject: [PATCH] Remove duplicate and temp files from git --- .../analysis-checkpoint.ipynb | 60 ------ .../best_practices-checkpoint.ipynb | 41 ---- .../class_program-checkpoint.ipynb | 35 ---- .../clean_up-checkpoint.ipynb | 60 ------ .../cost_analysis-checkpoint.ipynb | 41 ---- .../data_management_plan-checkpoint.ipynb | 6 - .../example_workflows-checkpoint.ipynb | 41 ---- .../intro_to_cli-checkpoint.ipynb | 60 ------ .../intro_to_cloud-checkpoint.ipynb | 92 --------- .../intro_to_cloud_console-checkpoint.ipynb | 139 ------------- .../intro_to_ec2-Copy1-checkpoint.ipynb | 70 ------- .../intro_to_ec2-checkpoint.ipynb | 182 ------------------ .../intro_to_ec2_part2-checkpoint.ipynb | 151 --------------- .../intro_to_s3-checkpoint.ipynb | 60 ------ .../is_the_cloud_right-checkpoint.ipynb | 72 ------- .../monitor_costs-checkpoint.ipynb | 60 ------ .../openscience-checkpoint.ipynb | 41 ---- .../where_to_find_funds-checkpoint.ipynb | 6 - .../which_cloud-checkpoint.ipynb | 45 ----- .../working with s3-checkpoint.ipynb | 60 ------ content/AWS/images/acceleratescience.svg | 1 - content/AWS/images/is_the_cloud_right.ipynb | 72 ------- content/Azure/images/acceleratescience.svg | 1 - .../01_bash_shell-checkpoint.ipynb | 33 ---- 24 files changed, 1429 deletions(-) delete mode 100644 content/.ipynb_checkpoints/analysis-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/best_practices-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/class_program-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/clean_up-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/cost_analysis-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/data_management_plan-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/example_workflows-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_cli-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_cloud-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_cloud_console-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_ec2-Copy1-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_ec2-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_ec2_part2-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/intro_to_s3-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/is_the_cloud_right-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/monitor_costs-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/openscience-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/where_to_find_funds-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/which_cloud-checkpoint.ipynb delete mode 100644 content/.ipynb_checkpoints/working with s3-checkpoint.ipynb delete mode 100644 content/AWS/images/acceleratescience.svg delete mode 100644 content/AWS/images/is_the_cloud_right.ipynb delete mode 100644 content/Azure/images/acceleratescience.svg delete mode 100644 content/ELM/.ipynb_checkpoints/01_bash_shell-checkpoint.ipynb diff --git a/content/.ipynb_checkpoints/analysis-checkpoint.ipynb b/content/.ipynb_checkpoints/analysis-checkpoint.ipynb deleted file mode 100644 index a6ca11e..0000000 --- a/content/.ipynb_checkpoints/analysis-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Putting It All Together\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/best_practices-checkpoint.ipynb b/content/.ipynb_checkpoints/best_practices-checkpoint.ipynb deleted file mode 100644 index 18c53f1..0000000 --- a/content/.ipynb_checkpoints/best_practices-checkpoint.ipynb +++ /dev/null @@ -1,41 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "90e130db", - "metadata": {}, - "source": [ - "## Cloud Best Practices" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "881c3977", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/class_program-checkpoint.ipynb b/content/.ipynb_checkpoints/class_program-checkpoint.ipynb deleted file mode 100644 index 3840b8a..0000000 --- a/content/.ipynb_checkpoints/class_program-checkpoint.ipynb +++ /dev/null @@ -1,35 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "6bf4ef2d", - "metadata": {}, - "source": [ - "# CLASS Program\n", - "\n", - "
" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/clean_up-checkpoint.ipynb b/content/.ipynb_checkpoints/clean_up-checkpoint.ipynb deleted file mode 100644 index b914bd5..0000000 --- a/content/.ipynb_checkpoints/clean_up-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to Compute on the Cloud\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/cost_analysis-checkpoint.ipynb b/content/.ipynb_checkpoints/cost_analysis-checkpoint.ipynb deleted file mode 100644 index fa46f20..0000000 --- a/content/.ipynb_checkpoints/cost_analysis-checkpoint.ipynb +++ /dev/null @@ -1,41 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "90e130db", - "metadata": {}, - "source": [ - "## How do I estimate costs?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2234c14b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/data_management_plan-checkpoint.ipynb b/content/.ipynb_checkpoints/data_management_plan-checkpoint.ipynb deleted file mode 100644 index 363fcab..0000000 --- a/content/.ipynb_checkpoints/data_management_plan-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/example_workflows-checkpoint.ipynb b/content/.ipynb_checkpoints/example_workflows-checkpoint.ipynb deleted file mode 100644 index c5cc162..0000000 --- a/content/.ipynb_checkpoints/example_workflows-checkpoint.ipynb +++ /dev/null @@ -1,41 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "da5b9cad", - "metadata": {}, - "source": [ - "# Cloud Journey Workflows" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ac2d50e", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_cli-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_cli-checkpoint.ipynb deleted file mode 100644 index 049e850..0000000 --- a/content/.ipynb_checkpoints/intro_to_cli-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to the AWS CLI\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_cloud-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_cloud-checkpoint.ipynb deleted file mode 100644 index 5579a6e..0000000 --- a/content/.ipynb_checkpoints/intro_to_cloud-checkpoint.ipynb +++ /dev/null @@ -1,92 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "21ff6943", - "metadata": {}, - "source": [ - "# Cloud Computing for Research\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "**Teaching:** 5 mins\n", - "\n", - "**Exercises:** 0 mins\n", - "\n", - "**Questions:**\n", - "* What is cloud computing for research?\n", - "\n", - "**Objectives:**\n", - "* Understand the basics of what the cloud is.\n", - "* Understand the benefits of utilizing the cloud for research.\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "993c138d", - "metadata": {}, - "source": [ - "## Background\n", - "\n", - "Cloud computing is an on-demand computing resource that is scalable and follows a pay-as-you-go model. Instead of a singular data center or super-computing center, large cloud providers have data centers spanning multiple locations. The largest cloud computing providers are [Microsoft (Azure)](https://azure.microsoft.com/), [Amazon (Amazon Web Services, AWS)](https://aws.amazon.com/) and [Google (Google Cloud Platform, GCP)](https://cloud.google.com/). Together, they are often referred to as “public” or “commercial” cloud providers.\n", - "\n", - "In contrast to buying your own desktop or laptop computer, a cluster of machines, or with buying external storage devices (such as a RAID, redundant array of independent disks), cloud computing allows you to provision computing and storage on machines that only available to you through an intermediated interface (such as a web-browser or through ssh). Simply put, cloud computing is a delivery of computing services over the Internet. \n", - "\n", - "## Benefits of the Cloud for Research\n", - "\n", - "Many researchers move to the commercial cloud simply because their local compute resources (local HPC clusters, or departmental clusters) are insufficient to deal with the volume of data and type of computation. With the cloud, there is no wait time to obtain the computing resources you need. With sufficient funds, you may even be able to obtain a near infinite number of CPUs, RAM and GPUs and compute can start as soon as you want it!\n", - "\n", - "With cloud computing, you do not need to purchase or maintain and update hardware, operating systems and a slew of dependencies. For the most part, providers maintain their hardware. Further, cloud providers just keep making new services to keep up with demands the rapidly expanding community building cloud-native workflows. Cloud providers are constantly evolving their tools and resources with a focus on storage, reliability, and security.\n", - "\n", - "## A Change in Paradigm\n", - "Working on the cloud involves a paradigm shift: researchers are no longer bringing their data to the compute (i.e. downloading data) but are instead bring their compute to the data. Cloud computing constitutes a learning curve including knowing cloud vocabulary and understanding the best practices to accelerate your research workflow, optimize costs and ensure security of your cloud architecture.\n", - "\n", - "## Drew's Pipeline\n", - "Drew Anders is an ecologist who works on understanding how much [boreal Arctic lakes are greening](https://www.pnas.org/content/118/15/e2021219118) under current climate conditions. To assess this, Drew needs to process 158.6TB (150 scenes) of satellite imagery from a cloud-hosted storage bucket and extract [Normalized Difference Vegetation Index (NDVI)](https://en.wikipedia.org/wiki/Normalized_difference_vegetation_index) values. Drew is currently using the departmental computing server to download and process the data using a Python script, `process_sat.py` and is uploading the processed data to an FTP server to share with collaborators. \n", - "\n", - "Unfortunately, the departmental server is running out of storage space and the processing units have insufficient memory to process the data. Drew has calculated that with the departmental server, the wall clock time to download, process and analyze the data would take 48 days. Drew has to publish a paper by the end of the month for a special issue of \"Ecology Outsphere Today\". Further, Drew needs to make processed data available to reviewers of the publication and to collaborators. \n", - "\n", - "After speaking with the deparmental IT administrator, Drew has decided to explore cloud computing as a means for scalability (increasing computational power), data storage, and to reduce the time to publication. Drew's PI has a approved a small amount of money to be spent on a prototype, with potential for turning a successful cloud-based workflow into a grant proposal. \n", - "\n", - "Over the next few lessons in the CLASS Essentials course, Drew will learn how to:\n", - "1. Utilize cloud compute to increase processing speed and memory and reduce wall clock time\n", - "2. Utilize cloud storage buckets to store and retrieve data\n", - "3. Run `process_sat.py` on cloud compute and retrieve data directly from cloud storage\n", - "4. Monitor costs and understand best practices for working on the cloud\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f406b779", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_cloud_console-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_cloud_console-checkpoint.ipynb deleted file mode 100644 index 144522e..0000000 --- a/content/.ipynb_checkpoints/intro_to_cloud_console-checkpoint.ipynb +++ /dev/null @@ -1,139 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to the AWS Console\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "**Teaching:** 15 mins\n", - "\n", - "**Exercises:** 5 mins\n", - "\n", - "**Questions:**\n", - "* How do I navigate the web console?\n", - "\n", - "**Objectives:**\n", - "* Log in to the AWS web console\n", - "* Navigate the AWS web console\n", - "* Recognize important information about your cloud environment\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "id": "acf07f2b", - "metadata": {}, - "source": [ - "### Setup\n", - "\n", - "The console is the point of entry where you will start your AWS journey. If you are participating in the CLASS Essentials Workshop, you will have received a csv file via email. This csv file contains your **username**, **temporary password** and **console login link**. \n", - "\n", - "```{admonition} Be Aware\n", - ":class: danger\n", - "If you are doing self-paced learning, your login information may be different!\n", - "```\n", - "\n", - "### Logging in to the console\n", - "Paste your console login link into your web browser. You will see the AWS site requesting you to sign in as an IAM user. Fill in your username and password that was assigned to you in the csv file. You will be asked to change your password the first time you sign in to the console. Fill in your old password and then choose a new password that you can remember! \n", - "\n", - "You are now logged on. Welcome to the first step of your cloud journey! \n", - "\n", - "Figure 1 is what your screen should look like when you log in to the AWS console for the first time. If you encounter any pop up boxes, you can exit them. Next, let's take a closer look at some key concepts and components of the AWS console. \n", - "\n", - "\n", - "\n", - "
Figure 1: First time logging in to the AWS console

" - ] - }, - { - "cell_type": "markdown", - "id": "930220c5", - "metadata": {}, - "source": [ - "### Key concepts and components of the AWS console\n", - "\n", - "\n", - "
Figure 2: Basic but important components of the AWS Console

\n" - ] - }, - { - "cell_type": "markdown", - "id": "22bf4b20", - "metadata": {}, - "source": [ - "Figure 2 lists the basic components you will see when you first log in to the AWS console. \n", - "\n", - "**1. Services**\n", - ": If you click on the dropdown menu labeled \"Services\" you will see the entire list of services that is offered by AWS. Currently, AWS has over 200 services ranging from compute to machine learning to networking to data storage. The list grows frequently!\n", - "\n", - "**2. Search bar**\n", - ": You can also use the search bar to find a particular service or offering that you are interested in. Cloud vendors have proprietary vocabulary or jargon, so it is often useful to be aware of those as you start exploring more in your cloud adoption journey. [Internet2's CLASS Core documentation](https://github.internet2.edu/pages/CLASS/CLASS-Essentials-AWS/vendor_vocabulary.html) provides a basic list of vendor specific cloud terminology. For example, if you wanted to provision or utilize a \"virtual server\", you can try using the search bar to search for \"EC2\" which stands for \"Elastic Cloud Compute\" and is the AWS vocabulary for virtual computers.\n", - "\n", - "**3. IAM Username and Account**\n", - ": The menu bar on the console also lists your IAM username and account number. IAM is short for Identity and Access Management. Identity is a mechanism to authenticate users (e.g. password) and Access is a mechanism for authorizing what kinds of services users have access to. For example, in this scenario, student1 is an IAM user on the account \"4414-3982-1395\". In the creation of the IAM user \"student1\", the administrator (overlord) of the account assigned a particular permission scope (in AWS this is known as roles) to ensure that student1 only has access to a particular set of services and also assigned a password to authenticate user login. IAM is a key component in managing security on the cloud.\n", - "\n", - "**4. Region**\n", - ": AWS (and most cloud providers) uses the term region to denote the physical location of the data center(s) you are building your cloud service(s) in. Here we see that the region is \"Ohio\". This literally means that as you develop some of your AWS components like compute and storage, the physical location of your server and data is in the state of Ohio! Regions are important to know and understand for two major reasons: costs and latency. Transferring data between regions in AWS incurs what is known as a inter-region data fee (roughly $0.16/GB as of 2021). If you work with larger datasets, you may find that data transfer speeds (e.g. downloading, processing data, etc.) in different regions can occur more slowly due to something called latency. For the sake of optimizing costs and minimizing latency, you are encouraged to choose a region that is closest to you and your cloud data. For the CLASS Essentials Workshop, we will use \"Ohio\" as the region. \n", - "\n", - "**5. Quick links** \n", - ": As you work more frequently on the console, your frequently used services will pop up on your console, and you can also access the entire suite of AWS services via the All Services drop down. \n" - ] - }, - { - "cell_type": "markdown", - "id": "b2240a49", - "metadata": {}, - "source": [ - "```{admonition} Exercise\n", - ":class: attention\n", - "\n", - "* What is your IAM username? \n", - "* Why is IAM important?\n", - "* What are the three ways you can find an AWS service on the console?\n", - "````" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b8fab5e4", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_ec2-Copy1-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_ec2-Copy1-checkpoint.ipynb deleted file mode 100644 index 0f04b4a..0000000 --- a/content/.ipynb_checkpoints/intro_to_ec2-Copy1-checkpoint.ipynb +++ /dev/null @@ -1,70 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to Compute on the Cloud\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "**Teaching:** 5 mins\n", - "\n", - "**Exercises:** 0 mins\n", - "\n", - "**Questions:**\n", - "* What is cloud computing for research?\n", - "\n", - "**Objectives:**\n", - "* Understand the basics of what the cloud is.\n", - "* Understand the benefits of utilizing the cloud for research.\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_ec2-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_ec2-checkpoint.ipynb deleted file mode 100644 index 62d6f2d..0000000 --- a/content/.ipynb_checkpoints/intro_to_ec2-checkpoint.ipynb +++ /dev/null @@ -1,182 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to Elastic Cloud Compute (EC2)\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "**Teaching:** 45 mins\n", - "\n", - "**Exercises:** 10 mins\n", - "\n", - "**Questions:**\n", - "* What is an EC2 instance?\n", - "* When would I use an EC2 instance?\n", - "* How do I launch an EC2 instance?\n", - "\n", - "**Objectives:**\n", - "* Understand the concept of virtual servers.\n", - "* Understand what an Elastic Cloud Compute (EC2) instance is.\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "Recall that the two fundamental components of cloud computing is compute and storage. On AWS, a \"virtual server\" or \"virtual computer\" is known as an **Elastic Cloud Compute (EC2) instance**; sometimes it's called \"EC2\", sometimes it's called an \"instance\" to denote that the ability to build and terminate this server instantaneously, but they all mean the same thing. An EC2 instance is no different from a server that sits under your desk, or your local departmental cluster, or even your local HPC cluster. You even boot up an EC2 instance through the web console, install software and then shut down your instance just like you would a real computer, except that Amazon takes care of the physical machinery while you are in charge of process of creating the computer. In some sense, you can think of utilizing an EC2 instance as renting a server or computer from Amazon! \n", - "\n", - "In cloud jargon, the term **elasticity** denotes the ability to quickly expand or decrease computer processing, memory, and storage resources to meet changing demands. In that way, you can expand the size of your CPU, RAM and disk size on your EC2 instance almost instantenously. Since EC2 forms the backbone of most of AWS's core infrastructure, it is an important part of your cloud journey. \n", - " \n", - "\n", - "Let's walk through some of the steps on getting an EC2 instance up and running. \n" - ] - }, - { - "cell_type": "markdown", - "id": "bc5d082d", - "metadata": {}, - "source": [ - "We begin with the AWS console again. Under the \"Build a Solution\" panel, select `Launch a Virtual Machine`\n", - "\n", - "\n", - "\n", - "
Figure 1: Start page for the AWS console

\n", - "\n", - "This will then lead you through a series of steps to get a **Free Tier** EC2 instance up and running. \n", - "\n", - "```{admonition} Note\n", - ":class: note\n", - "\n", - "AWS Free Tier refers to several of the services that AWS offers to help users gain more hands on experience on the AWS platform without being charged. [Click here](https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRank&all-free-tier.sort-order=asc&awsf.Free%20Tier%20Types=*all&awsf.Free%20Tier%20Categories=*all) for more info about the AWS Free Tier [external link] . \n", - "```\n", - "\n", - "There are 7 steps to walk through to create a new EC2 instance; we will go through each in detail: \n", - "1. Select an AMI\n", - "2. Choose Instance Type\n", - "3. Configure Instance \n", - "4. Add Storage\n", - "5. Add Tags\n", - "6. Configure Security Group\n", - "7. Review/Launch" - ] - }, - { - "cell_type": "markdown", - "id": "b9809503", - "metadata": {}, - "source": [ - "## 1. Select an AMI\n", - "\n", - "An Amazon Machine Image (AMI) is a template that Amazon uses to describe the operating system, disk type and all the software configuration that is needed to make sure a computer runs smoothly. Imagine that you are purchasing a new laptop; fresh out of the box, the laptop is pre-configured with an operating system (e.g. Windows, Mac OS, Ubuntu etc.), configuration files that tells the laptop what peripherals are attached, and pre-installed software like Adobe PDF reader. An AMI contains all this information so that your EC2 instance runs exactly like it would a new laptop out of the box! There is much more to learn about AMIs and how they can used for collaboration and data sharing but that is not within the scope of CLASS Essentials. \n", - "\n", - "As you scroll through the AMI list (Figure 2) you will notice that the list contains offerings from various vendors (e.g. Amazon, RedHat, Windows, etc.). We will be choosing the Ubuntu operating system for flexibility and versatility (can be used outside of the AWS ecosystem). \n", - "\n", - "To list all the Free Tier AMIs, check the box on the right that says ```Free tier only```.\n", - "\n", - "\n", - "\n", - "
Figure 2: Step 1 - Select an AMI - Free Tier Only

\n", - "\n", - "Scroll to ```Ubuntu Server 20.04 LTS(HVM), SSD Volume Type``` (Figure 3). Select ```64-bit(x86)```. \n", - "\n", - "\n", - "\n", - "
Figure 3: Step 1 - Select an AMI - Operating System Selection

" - ] - }, - { - "cell_type": "markdown", - "id": "17597535", - "metadata": {}, - "source": [ - "## Step 2: Choose an Instance Type\n", - "\n", - "Choosing an instance type is choosing the hardware for your computing system: you get to pick the number of CPUs and memory size for your instance. \n", - "\n", - "Instance types are group by [**families**](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) and denotes whether, for example, an instance is optimized for batch processing (compute-optimized, C-family), optimized for databases (memory-optimized, R-family) or has accelerated hardware (GPUs) for AI or Machine Learning pipelines. \n", - "\n", - "When you choose an Instance Type (Figure 3), the screen show additional information about the selected instance type including the number of CPUs, the memory size, the type of storage and information about networking. \n", - "\n", - "In the Instance Storage (GB) column, you will notice a term called **EBS**. EBS is the acronym for **Elastic Block Storage** and is analogous to the hard disk or boot drive on your personal computer or laptop. More details about EBS and different kinds of disk storage on EC2 instances are beyond the scope of CLASS Essentials. \n", - "\n", - "```{admonition} Note\n", - ":class: note\n", - "The four most common types of storage you will encounter on AWS are: Elastic Block Storage (EBS), Elastic File Storage (EFS), Simple Storage Service (s3) and s3 Glacier. In the simplest terms, EBS is analogous to a computer hard drive and EFS is analogous to a network file system (NFS) or shared file system. s3 is AWS's object storage which is discussed [here](./intro_to_s3). s3 Glacier is a cost-effective way of storing s3 files that you do not need to access frequently. \n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "id": "4ee655a3", - "metadata": {}, - "source": [ - "\n", - "\n", - "
Figure 3: Step 2 - Choose an Instance Type

\n", - "\n", - "Select ```Next: Configure Instance Details```.\n", - "\n", - "## Step 3: Configure Instance Details\n", - "Step 3 in creating an EC2 instance involves a rudimentary understanding of several key AWS and cloud jargon (Figure 4). While delving deeper into some of the terminology is outside of the scope of CLASS Essentials, we go will through these terms in brief as we learn how to configure your EC2 instance. \n", - "\n", - "\n", - "\n", - "
Figure 4: Step 3 - Configure Instance Details

\n", - "\n", - "**Number of instances** : This indicates how many instances you want to create at the same time. Here, we will leave the value as '1' but in actuality, you can can have up to 20 instances per region. \n", - "\n", - "```{admonition} Note\n", - ":class: note\n", - "Recall that we learned about regions in the [previous chapter](./intro_to_cloud_console). \n", - "```\n", - "\n", - "**Purchasing Options** : Throughout your AWS journey, you will hear the term **Spot Instances**. Spot instances make use of the servers that go unused in AWS data centers to minimize costs. Spot Instances provide Amazon with a flexible way to sell extra capacity. The instances are acquired through a bidding process in which the customer specifies a price per hour he is willing to pay.\n", - "\n", - "When an EC2 instance becomes available at that price, the customer's instance will run. The instance will be cut off when the Spot price increases and exceeds the customer's bid. " - ] - }, - { - "cell_type": "markdown", - "id": "97d4256b", - "metadata": {}, - "source": [ - "```{admonition} Exercise\n", - ":class: attention\n", - "\n", - "* What kind of information is contained in an AMI? \n", - "* How do Spot Instances help you optimize costs?\n", - "````" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_ec2_part2-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_ec2_part2-checkpoint.ipynb deleted file mode 100644 index 15ec6d4..0000000 --- a/content/.ipynb_checkpoints/intro_to_ec2_part2-checkpoint.ipynb +++ /dev/null @@ -1,151 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# EC2, AMI and Instances\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "**Teaching:** 45 mins\n", - "\n", - "**Exercises:** 10 mins\n", - "\n", - "**Questions:**\n", - "* What is an EC2 instance?\n", - "* When would I use an EC2 instance?\n", - "* How do I launch an EC2 instance?\n", - "\n", - "**Objectives:**\n", - "* Understand the concept of virtual servers.\n", - "* Understand what an Elastic Cloud Compute (EC2) instance is.\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "Recall that the two fundamental components of cloud computing is compute and storage. On AWS, a \"virtual server\" or \"virtual computer\" is known as an **Elastic Cloud Compute (EC2) instance**; sometimes it's called \"EC2\", sometimes it's called an \"instance\" to denote that the ability to build and terminate this server instantaneously, but they all mean the same thing. An EC2 instance is no different from a server that sits under your desk, or your local departmental cluster, or even your local HPC cluster. You even boot up an EC2 instance through the web console, install software and then shut down your instance just like you would a real computer, except that Amazon takes care of the physical machinery while you are in charge of process of creating the computer. In some sense, you can think of utilizing an EC2 instance as renting a server or computer from Amazon! \n", - "\n", - "In cloud jargon, the term **elasticity** denotes the ability to quickly expand or decrease computer processing, memory, and storage resources to meet changing demands. In that way, you can expand the size of your CPU, RAM and disk size on your EC2 instance almost instantenously. Since EC2 forms the backbone of most of AWS's core infrastructure, it is an important part of your cloud journey. \n", - " \n", - "\n", - "Let's walk through some of the steps on getting an EC2 instance up and running. \n" - ] - }, - { - "cell_type": "markdown", - "id": "bc5d082d", - "metadata": {}, - "source": [ - "We begin with the AWS console again. Under the \"Build a Solution\" panel, select `Launch a Virtual Machine`\n", - "\n", - "\n", - "\n", - "
Figure 1: Start page for the AWS console

\n", - "\n", - "This will then lead you through a series of steps to get a **Free Tier** EC2 instance up and running. \n", - "\n", - "```{admonition} Note\n", - ":class: note\n", - "\n", - "AWS Free Tier refers to several of the services that AWS offers to help users gain more hands on experience on the AWS platform without being charged. [Click here](https://aws.amazon.com/free/?all-free-tier.sort-by=item.additionalFields.SortRank&all-free-tier.sort-order=asc&awsf.Free%20Tier%20Types=*all&awsf.Free%20Tier%20Categories=*all) for more info about the AWS Free Tier [external link] . \n", - "```\n", - "\n", - "There are 7 steps to walk through to create a new EC2 instance; we will go through each in detail: \n", - "1. Select an AMI\n", - "2. Choose Instance Type\n", - "3. Configure Instance \n", - "4. Add Storage\n", - "5. Add Tags\n", - "6. Configure Security Group\n", - "7. Review/Launch" - ] - }, - { - "cell_type": "markdown", - "id": "b9809503", - "metadata": {}, - "source": [ - "## 1. Select an AMI\n", - "\n", - "An Amazon Machine Image (AMI) is a template that Amazon uses to describe the operating system, disk type and all the software configuration that is needed to make sure a computer runs smoothly. Imagine that you are purchasing a new laptop; fresh out of the box, the laptop is pre-configured with an operating system (e.g. Windows, Mac OS, Ubuntu etc.), configuration files that tells the laptop what peripherals are attached, and pre-installed software like Adobe PDF reader. An AMI contains all this information so that your EC2 instance runs exactly like it would a new laptop out of the box! There is much more to learn about AMIs and how they can used for collaboration and data sharing but that is not within the scope of CLASS Essentials. \n", - "\n", - "As you scroll through the AMI list (Figure 2) you will notice that the list contains offerings from various vendors (e.g. Amazon, RedHat, Windows, etc.). We will be choosing the Ubuntu operating system for flexibility and versatility (can be used outside of the AWS ecosystem). \n", - "\n", - "To list all the Free Tier AMIs, check the box on the right that says ```Free tier only```.\n", - "\n", - "\n", - "\n", - "
Figure 2: Step 1 - Select an AMI - Free Tier Only

\n", - "\n", - "Scroll to ```Ubuntu Server 20.04 LTS(HVM), SSD Volume Type``` (Figure 3). Select ```64-bit(x86)```. \n", - "\n", - "\n", - "\n", - "
Figure 3: Step 1 - Select an AMI - Operating System Selection

" - ] - }, - { - "cell_type": "markdown", - "id": "17597535", - "metadata": {}, - "source": [ - "## Step 2: Choose an Instance Type\n", - "\n", - "Choosing an instance type is choosing the hardware for your computing system: you get to pick the number of CPUs and memory size for your instance. \n", - "\n", - "Instance types are group by [**families**](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html) and denotes whether, for example, an instance is optimized for batch processing (compute-optimized, C-family), optimized for databases (memory-optimized, R-family) or has accelerated hardware (GPUs) for AI or Machine Learning pipelines. \n", - "\n", - "When you choose an Instance Type (Figure 3), the screen show additional information about the selected instance type including the number of CPUs, the memory size, the type of storage and information about networking. \n", - "\n", - "In the Instance Storage (GB) column, you will notice a term called **EBS**. EBS is the acronym for **Elastic Block Storage** and is analogous to the hard disk or boot drive on your personal computer or laptop. More details about EBS and different kinds of disk storage on EC2 instances are beyond the scope of CLASS Essentials. \n", - "\n", - "```{admonition} Note\n", - ":class: note\n", - "The four most common types of storage you will encounter on AWS are: Elastic Block Storage (EBS), Elastic File Storage (EFS), Simple Storage Service (s3) and s3 Glacier. In the simplest terms, EBS is analogous to a computer hard drive and EFS is analogous to a network file system (NFS) or shared file system. s3 is AWS's object storage which is discussed [here](./intro_to_s3). s3 Glacier is a cost-effective way of storing s3 files that you do not need to access frequently. \n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "id": "4ee655a3", - "metadata": {}, - "source": [ - "\n", - "\n", - "
Figure 3: Step 2 - Choose an Instance Type

\n", - "\n", - "Select ```Next: Configure Instance Details``` and we'll move on to the next chapter. " - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/intro_to_s3-checkpoint.ipynb b/content/.ipynb_checkpoints/intro_to_s3-checkpoint.ipynb deleted file mode 100644 index f3f90ef..0000000 --- a/content/.ipynb_checkpoints/intro_to_s3-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Introduction to Cloud Storage\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/is_the_cloud_right-checkpoint.ipynb b/content/.ipynb_checkpoints/is_the_cloud_right-checkpoint.ipynb deleted file mode 100644 index 60aefce..0000000 --- a/content/.ipynb_checkpoints/is_the_cloud_right-checkpoint.ipynb +++ /dev/null @@ -1,72 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1b6fcbc0", - "metadata": {}, - "source": [ - "# Is the Cloud Right for Me?\n", - "\n", - "Now that you have an idea of what cloud computing is, the question that is most pertinent to researchers is whether or not you ***should*** move your work to the cloud. The following chart outlines some fundamental questions to ask in your assessment: \n", - "\n", - "
\n", - "\n", - "Many researchers move to the commercial cloud simply because their local compute resources (local HPC clusters, or departmental clusters) are insufficient to deal with the volume of data and type of computation. With the cloud, there is no wait time to obtain the computing resources you need. With sufficient funds, you may even be able to obtain a near infinite number of CPUs, RAM and GPUs and compute can start as soon as you want it!\n", - "\n", - "```{note}\n", - "Scalable computing is the ability to obtain more computers(horizontal scaling) and/or to obtain more powerful computers(vertical scaling)\n", - "```\n", - "\n", - "With cloud computing, you do not need to purchase or maintain and update hardware, operating systems and a slew of dependencies. For the most part, providers maintain their hardware. Further, cloud providers just keep making new services to keep up with demands the rapidly expanding community building cloud-native workflows. Cloud providers are constantly evolving their tools and resources with a focus on storage, reliability, and security. \n", - "\n", - "\n", - "Other factors that will play in your decision to move to the cloud include knowing where common large datasets are hosted. The core idea in working on the cloud involves a paradigm shift: researchers should no longer bring their data to the compute (i.e. downloading data) but should instead bring their compute to the data! If you think about it, why spend hours on end to download data and find a place to store it when you can work directly with the data?\n", - "\n", - "\n", - "If your collaborators are already working in the cloud i.e. hosting data on the cloud or building cloud-based workflows, it would also make sense for you to bring your work to the cloud. Go here for more on Open Science and Collaboration.\n", - "\n", - "\n", - "## Why *not* to migrate to the cloud?\n", - "\n", - "If you have already identified an adequate-to-your-needs computing environment like XSEDE or you already have the access to the resources you need, it just doesn't make sense to migrate to the cloud!\n", - "\n", - "Using the public cloud requires a learning curve. Sometimes you may just simply not have the time or resources to do this important step. The CLASS program can help you overcome some of these hurdles, but you may prefer to spend your time learning other things or exploring other avenues and it just doesn't seem worth it to invest your time in learning about the cloud. \n", - "\n", - "If you operate your computer(s) at a very high duty cycle i.e. you computer is constantly computing something and you have massive datasets that you work with frequently that is stored on-premise, the cloud may not be a very cost-effective option. \n", - "\n", - "Finally, if there is too much of an administrative drag preventing you from using the cloud (e.g. regulations and compliance such as HIPAA or FERPA), you may want to reconsider staying with your local infrastructure. \n", - "\n", - "In the next chapter, we will discuss the million dollar question: \"But which cloud provider should I choose?!\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ddad00f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/monitor_costs-checkpoint.ipynb b/content/.ipynb_checkpoints/monitor_costs-checkpoint.ipynb deleted file mode 100644 index c8b0712..0000000 --- a/content/.ipynb_checkpoints/monitor_costs-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dc57021c", - "metadata": {}, - "source": [ - "# Monitoring Costs\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/openscience-checkpoint.ipynb b/content/.ipynb_checkpoints/openscience-checkpoint.ipynb deleted file mode 100644 index f654acd..0000000 --- a/content/.ipynb_checkpoints/openscience-checkpoint.ipynb +++ /dev/null @@ -1,41 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "90e130db", - "metadata": {}, - "source": [ - "## Open and Reproducible Science" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b724c135", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/where_to_find_funds-checkpoint.ipynb b/content/.ipynb_checkpoints/where_to_find_funds-checkpoint.ipynb deleted file mode 100644 index 363fcab..0000000 --- a/content/.ipynb_checkpoints/where_to_find_funds-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/which_cloud-checkpoint.ipynb b/content/.ipynb_checkpoints/which_cloud-checkpoint.ipynb deleted file mode 100644 index 7fde371..0000000 --- a/content/.ipynb_checkpoints/which_cloud-checkpoint.ipynb +++ /dev/null @@ -1,45 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "dbc2e7e6", - "metadata": {}, - "source": [ - "# Choosing Cloud Providers\n", - "\n", - "If you have decided that your research belongs in the cloud, or if you would like to explore cloud computing a bit more, the next step is choosing a public cloud provider. To reiterate, the three major cloud providers that *most* researchers currently use are Micosoft Azure, Amazon Web Services (AWS), and Google Cloud Platform (GCP). \n", - "\n", - " Each cloud provider brings a different set of capabilities that can factor into which cloud provider you ultimately end up choosing. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ff14b2c", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/.ipynb_checkpoints/working with s3-checkpoint.ipynb b/content/.ipynb_checkpoints/working with s3-checkpoint.ipynb deleted file mode 100644 index 32672cb..0000000 --- a/content/.ipynb_checkpoints/working with s3-checkpoint.ipynb +++ /dev/null @@ -1,60 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "56a5c55a", - "metadata": {}, - "source": [ - "# Working with s3 buckets\n", - "\n", - "\n", - "```{admonition} Overview\n", - ":class: tip\n", - "\n", - "\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "502f2360", - "metadata": {}, - "source": [ - "```{code-cell} ipython3\n", - ":tags: [mytag]\n", - "\n", - "print(\"A python cell\")\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c543528f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/AWS/images/acceleratescience.svg b/content/AWS/images/acceleratescience.svg deleted file mode 100644 index 6985786..0000000 --- a/content/AWS/images/acceleratescience.svg +++ /dev/null @@ -1 +0,0 @@ -WILL CLOUD COMPUTING HELP ACCELERATE MYSCIENTIFIC DISCOVERY?Local computeresources taketoo longPersonalcomputersrun of memory orCPUNeed scalablecomputingComputational ProblemCommondatasets are hosted onthe cloudCollaborators arealready using thecloudInterestedinreproduciblescienceExternal motivationDo youalready havecloudcredits?Do you useproprietary softwarethat requires a operatingsystem?Whichprovider doyourcollaboratorsuse?choosing a cloud providerAre theredata securityand privacyconcerns?Do you haveresources toadapt to a cloud-native if necessary?Do you needto make dataavailable for'x' amount ofyears?further assessmentIn your assessment of whether or not your research will from using the cloud, you need to be aware that there is an associated learning curve and a set of bestpractices that you are encouraged to follow! \ No newline at end of file diff --git a/content/AWS/images/is_the_cloud_right.ipynb b/content/AWS/images/is_the_cloud_right.ipynb deleted file mode 100644 index 60aefce..0000000 --- a/content/AWS/images/is_the_cloud_right.ipynb +++ /dev/null @@ -1,72 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1b6fcbc0", - "metadata": {}, - "source": [ - "# Is the Cloud Right for Me?\n", - "\n", - "Now that you have an idea of what cloud computing is, the question that is most pertinent to researchers is whether or not you ***should*** move your work to the cloud. The following chart outlines some fundamental questions to ask in your assessment: \n", - "\n", - "
\n", - "\n", - "Many researchers move to the commercial cloud simply because their local compute resources (local HPC clusters, or departmental clusters) are insufficient to deal with the volume of data and type of computation. With the cloud, there is no wait time to obtain the computing resources you need. With sufficient funds, you may even be able to obtain a near infinite number of CPUs, RAM and GPUs and compute can start as soon as you want it!\n", - "\n", - "```{note}\n", - "Scalable computing is the ability to obtain more computers(horizontal scaling) and/or to obtain more powerful computers(vertical scaling)\n", - "```\n", - "\n", - "With cloud computing, you do not need to purchase or maintain and update hardware, operating systems and a slew of dependencies. For the most part, providers maintain their hardware. Further, cloud providers just keep making new services to keep up with demands the rapidly expanding community building cloud-native workflows. Cloud providers are constantly evolving their tools and resources with a focus on storage, reliability, and security. \n", - "\n", - "\n", - "Other factors that will play in your decision to move to the cloud include knowing where common large datasets are hosted. The core idea in working on the cloud involves a paradigm shift: researchers should no longer bring their data to the compute (i.e. downloading data) but should instead bring their compute to the data! If you think about it, why spend hours on end to download data and find a place to store it when you can work directly with the data?\n", - "\n", - "\n", - "If your collaborators are already working in the cloud i.e. hosting data on the cloud or building cloud-based workflows, it would also make sense for you to bring your work to the cloud. Go here for more on Open Science and Collaboration.\n", - "\n", - "\n", - "## Why *not* to migrate to the cloud?\n", - "\n", - "If you have already identified an adequate-to-your-needs computing environment like XSEDE or you already have the access to the resources you need, it just doesn't make sense to migrate to the cloud!\n", - "\n", - "Using the public cloud requires a learning curve. Sometimes you may just simply not have the time or resources to do this important step. The CLASS program can help you overcome some of these hurdles, but you may prefer to spend your time learning other things or exploring other avenues and it just doesn't seem worth it to invest your time in learning about the cloud. \n", - "\n", - "If you operate your computer(s) at a very high duty cycle i.e. you computer is constantly computing something and you have massive datasets that you work with frequently that is stored on-premise, the cloud may not be a very cost-effective option. \n", - "\n", - "Finally, if there is too much of an administrative drag preventing you from using the cloud (e.g. regulations and compliance such as HIPAA or FERPA), you may want to reconsider staying with your local infrastructure. \n", - "\n", - "In the next chapter, we will discuss the million dollar question: \"But which cloud provider should I choose?!\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ddad00f", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/content/Azure/images/acceleratescience.svg b/content/Azure/images/acceleratescience.svg deleted file mode 100644 index 6985786..0000000 --- a/content/Azure/images/acceleratescience.svg +++ /dev/null @@ -1 +0,0 @@ -WILL CLOUD COMPUTING HELP ACCELERATE MYSCIENTIFIC DISCOVERY?Local computeresources taketoo longPersonalcomputersrun of memory orCPUNeed scalablecomputingComputational ProblemCommondatasets are hosted onthe cloudCollaborators arealready using thecloudInterestedinreproduciblescienceExternal motivationDo youalready havecloudcredits?Do you useproprietary softwarethat requires a operatingsystem?Whichprovider doyourcollaboratorsuse?choosing a cloud providerAre theredata securityand privacyconcerns?Do you haveresources toadapt to a cloud-native if necessary?Do you needto make dataavailable for'x' amount ofyears?further assessmentIn your assessment of whether or not your research will from using the cloud, you need to be aware that there is an associated learning curve and a set of bestpractices that you are encouraged to follow! \ No newline at end of file diff --git a/content/ELM/.ipynb_checkpoints/01_bash_shell-checkpoint.ipynb b/content/ELM/.ipynb_checkpoints/01_bash_shell-checkpoint.ipynb deleted file mode 100644 index d4f0c47..0000000 --- a/content/ELM/.ipynb_checkpoints/01_bash_shell-checkpoint.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "1c75f70d", - "metadata": {}, - "source": [ - "## Learn the Bash Shell" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.5" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}