diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index 2d64955..72780cd 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -10,7 +10,7 @@ "```{admonition} Overview\n", ":class: tip\n", "\n", - "**Teaching:**\n", + "**Teaching:** 80 min\n", "\n", "**Exercises:**\n", "\n", @@ -19,8 +19,6 @@ "\n", "**Objectives:**\n", "* Create a simple workflow using a cloud VM and cloud object storage.\n", - "* Understand the basics of Identity and Access Management (IAM)\n", - "* Add collaborators to a Bucket with appropriate permissions.\n", "\n", "```" ] @@ -116,13 +114,19 @@ "output_type": "stream", "text": [ "Get:1 http://security.debian.org/debian-security buster/updates InRelease [65.4 kB]\n", - "Hit:2 http://deb.debian.org/debian buster InRelease\n", - "Hit:3 http://deb.debian.org/debian buster-updates InRelease\n", - "Hit:4 http://deb.debian.org/debian buster-backports InRelease\n", + "Hit:2 http://deb.debian.org/debian buster InRelease \u001b[0m\u001b[33m\n", + "Get:3 http://deb.debian.org/debian buster-updates InRelease [51.9 kB]\n", + "Get:4 http://deb.debian.org/debian buster-backports InRelease [46.7 kB]\n", "Hit:5 http://packages.cloud.google.com/apt cloud-sdk-buster InRelease\n", "Hit:6 http://packages.cloud.google.com/apt google-cloud-packages-archive-keyring-buster InRelease\n", - "Get:7 http://packages.cloud.google.com/apt google-compute-engine-buster-stable InRelease [5526 B]\n", - "Fetched 70.9 kB in 1s (91.3 kB/s) \u001b[0m\u001b[33m\n", + "Hit:7 http://packages.cloud.google.com/apt google-compute-engine-buster-stable InRelease\n", + "Get:8 http://deb.debian.org/debian buster-backports/main amd64 Packages.diff/Index [27.8 kB]\n", + "Get:9 http://deb.debian.org/debian buster-backports/main Translation-en.diff/Index [27.8 kB]\n", + "Get:10 http://deb.debian.org/debian buster-backports/main amd64 Packages 2022-01-21-2002.00.pdiff [2515 B]\n", + "Get:11 http://deb.debian.org/debian buster-backports/main Translation-en 2022-01-21-2002.00.pdiff [3068 B]\n", + "Get:10 http://deb.debian.org/debian buster-backports/main amd64 Packages 2022-01-21-2002.00.pdiff [2515 B]\n", + "Get:11 http://deb.debian.org/debian buster-backports/main Translation-en 2022-01-21-2002.00.pdiff [3068 B]\n", + "Fetched 225 kB in 1s (278 kB/s)0m\u001b[33m \u001b[0m\u001b[33m\n", "Reading package lists... Done\n", "Building dependency tree \n", "Reading state information... Done\n", @@ -160,7 +164,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "id": "f36cb8c5-f305-4cb2-a5cc-0c9fd8592fb4", "metadata": {}, "outputs": [ @@ -168,7 +172,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "bucket: essentials-learner-2022-01-11 region: us-west2\n" + "bucket: essentials-learner-2022-01-21 region: us-west2\n" ] } ], @@ -180,7 +184,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "id": "c2ae2b74-5e93-4c55-8bd7-63337f7dcbb8", "metadata": {}, "outputs": [ @@ -188,7 +192,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Creating gs://essentials-learner-2022-01-11/...\n" + "Creating gs://essentials-learner-2022-01-21/...\n" ] } ], @@ -206,7 +210,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 4, "id": "10f8c773-97fc-46af-a6b5-1bb832472b33", "metadata": {}, "outputs": [ @@ -214,7 +218,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-11/\n" + "gs://essentials-learner-2022-01-21/\n" ] } ], @@ -234,7 +238,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "05fadd30-4f35-4a93-b713-cc132131b948", "metadata": {}, "outputs": [ @@ -256,7 +260,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "id": "96db6a66-3fbf-419a-b8c8-dbb27639e990", "metadata": {}, "outputs": [], @@ -266,7 +270,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 7, "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7", "metadata": {}, "outputs": [ @@ -295,7 +299,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab", "metadata": {}, "outputs": [], @@ -316,7 +320,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2", "metadata": {}, "outputs": [ @@ -325,13 +329,13 @@ "output_type": "stream", "text": [ "total 28\n", - "-rw-r--r-- 1 learner learner 960 Dec 17 19:00 ReadMe.md\n", - "-rw-r--r-- 1 learner learner 72 Dec 17 19:00 clean.sh\n", - "-rw-r--r-- 1 learner learner 613 Dec 17 19:00 combine.py\n", - "-rw-r--r-- 1 learner learner 280 Dec 17 19:00 download.sh\n", - "-rw-r--r-- 1 learner learner 314 Dec 17 19:00 get-index.sh\n", - "-rw-r--r-- 1 learner learner 76 Dec 17 19:00 search.json\n", - "-rw-r--r-- 1 learner learner 783 Dec 17 19:00 search.py\n" + "-rw-r--r-- 1 learner learner 960 Jan 21 21:32 ReadMe.md\n", + "-rw-r--r-- 1 learner learner 72 Jan 21 21:32 clean.sh\n", + "-rw-r--r-- 1 learner learner 613 Jan 21 21:32 combine.py\n", + "-rw-r--r-- 1 learner learner 280 Jan 21 21:32 download.sh\n", + "-rw-r--r-- 1 learner learner 314 Jan 21 21:32 get-index.sh\n", + "-rw-r--r-- 1 learner learner 76 Jan 21 21:32 search.json\n", + "-rw-r--r-- 1 learner learner 783 Jan 21 21:32 search.py\n" ] } ], @@ -353,7 +357,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 10, "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd", "metadata": {}, "outputs": [ @@ -402,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 11, "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e", "metadata": {}, "outputs": [ @@ -420,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 12, "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38", "metadata": {}, "outputs": [ @@ -434,8 +438,8 @@ "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "\\ [1 files][761.0 MiB/761.0 MiB] 42.0 MiB/s \n", - "Operation completed over 1 objects/761.0 MiB. \n" + "| [1 files][731.9 MiB/731.9 MiB] \n", + "Operation completed over 1 objects/731.9 MiB. \n" ] } ], @@ -453,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 13, "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916", "metadata": {}, "outputs": [], @@ -471,7 +475,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 14, "id": "b005876c-f9af-43d6-80c6-f04295413b9b", "metadata": {}, "outputs": [ @@ -479,8 +483,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 2.6G\n", - "-rw-r--r-- 1 learner learner 2.6G Dec 17 19:01 index.csv\n" + "total 2.5G\n", + "-rw-r--r-- 1 learner learner 2.5G Jan 21 21:32 index.csv\n" ] } ], @@ -498,7 +502,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 15, "id": "ffe969db-d207-44fe-8957-8d129c76ee8f", "metadata": {}, "outputs": [ @@ -507,9 +511,9 @@ "output_type": "stream", "text": [ "SCENE_ID,PRODUCT_ID,SPACECRAFT_ID,SENSOR_ID,DATE_ACQUIRED,COLLECTION_NUMBER,COLLECTION_CATEGORY,SENSING_TIME,DATA_TYPE,WRS_PATH,WRS_ROW,CLOUD_COVER,NORTH_LAT,SOUTH_LAT,WEST_LON,EAST_LON,TOTAL_SIZE,BASE_URL\n", - "LM51940171984262KIS00,LM05_L1GS_194017_19840918_20210913_01_T2,LANDSAT_5,MSS,1984-09-18,01,T2,1984-09-18T09:36:27.0970000Z,L1GS,194,17,51.0,62.58023,60.38359,14.45307,19.13901,34758212,gs://gcp-public-data-landsat/LM05/01/194/017/LM05_L1GS_194017_19840918_20210913_01_T2\n", - "LM10420331972311AAA04,LM01_L1TP_042033_19721106_20180429_01_T2,LANDSAT_1,MSS,1972-11-06,01,T2,1972-11-06T17:49:00.5000000Z,L1TP,42,33,3.0,39.68891,37.707,-115.59896,-112.90344,26932395,gs://gcp-public-data-landsat/LM01/01/042/033/LM01_L1TP_042033_19721106_20180429_01_T2\n", - "LT50150151992246PAC00,LT05_L1TP_015015_19920902_20170121_01_T1,LANDSAT_5,TM,1992-09-02,01,T1,1992-09-02T15:01:34.1390130Z,L1TP,15,15,90.0,65.32118,63.11726,-67.3438,-61.89671,118785510,gs://gcp-public-data-landsat/LT05/01/015/015/LT05_L1TP_015015_19920902_20170121_01_T1\n" + "LE71800592011134ASN00,LE07_L1TP_180059_20110514_20161209_01_T1,LANDSAT_7,ETM,2011-05-14,01,T1,2011-05-14T08:50:07.5251363Z,L1TP,180,59,74.0,2.39913,0.50961,18.09062,20.2487,181281962,gs://gcp-public-data-landsat/LE07/01/180/059/LE07_L1TP_180059_20110514_20161209_01_T1\n", + "LT51360422008226BKT00,LT05_L1GS_136042_20080813_20161030_01_T2,LANDSAT_5,TM,2008-08-13,01,T2,2008-08-13T04:03:49.0450690Z,L1GS,136,42,92.0,26.9495,25.03915,91.40541,93.81099,141994748,gs://gcp-public-data-landsat/LT05/01/136/042/LT05_L1GS_136042_20080813_20161030_01_T2\n", + "LE71760312020339NSG00,LE07_L1TP_176031_20201204_20201230_01_T1,LANDSAT_7,ETM,2020-12-04,01,T1,2020-12-04T07:41:11.6084536Z,L1TP,176,31,3.0,42.75649,40.7935,33.66313,36.65653,188511155,gs://gcp-public-data-landsat/LE07/01/176/031/LE07_L1TP_176031_20201204_20201230_01_T1\n" ] } ], @@ -529,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 16, "id": "c5e300c3-e1f3-4cd4-9679-77725e61c4db", "metadata": {}, "outputs": [ @@ -573,7 +577,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 17, "id": "c9872510-4265-4b0e-aeb5-5a829ff69b24", "metadata": {}, "outputs": [ @@ -603,7 +607,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 18, "id": "6912a9ec-0f9b-4500-ba20-d4280592b323", "metadata": {}, "outputs": [ @@ -611,11 +615,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1\n", - "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1\n", - "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20150127_20170302_01_T1\n", - "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20161028_20170219_01_T1\n", - "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20191224_20200110_01_T1\n" + "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1\n", + "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1\n", + "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20200601_20200608_01_T1\n", + "gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20150111_20170302_01_T2\n" ] } ], @@ -635,7 +638,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 19, "id": "3572c518-df83-4906-bfa6-a37bde2a5063", "metadata": {}, "outputs": [ @@ -670,7 +673,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 20, "id": "cccec3e1-0dcd-4e3b-a059-a884f5219b66", "metadata": { "scrolled": true, @@ -681,50 +684,50 @@ "name": "stdout", "output_type": "stream", "text": [ - "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_ANG.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B3.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B9.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B4.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B1.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B8.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B6.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_BQA.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B11.TIF...\n", + "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_ANG.txt...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B1.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B3.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B10.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B6.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B5.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B4.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B11.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B7.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B2.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B8.TIF...\n", "==> NOTE: You are downloading one or more large file(s), which would\n", "run significantly faster if you enabled sliced object downloads. This\n", "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B10.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_MTL.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B5.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B2.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20211010_20211018_01_T1/LC08_L1TP_025033_20211010_20211018_01_T1_B7.TIF...\n", - "| [14/14 files][997.2 MiB/997.2 MiB] 100% Done 65.3 MiB/s ETA 00:00:00 \n", - "Operation completed over 14 objects/997.2 MiB. \n", - "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_ANG.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B1.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B11.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B10.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B8.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B9.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B3.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_BQA.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B6.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B7.TIF...\n", - "==> NOTE: You are downloading one or more large file(s), which would \n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_B9.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_BQA.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20201007_20201016_01_T1/LC08_L1TP_025033_20201007_20201016_01_T1_MTL.txt...\n", + "- [14/14 files][952.9 MiB/952.9 MiB] 100% Done \n", + "Operation completed over 14 objects/952.9 MiB. \n", + "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_ANG.txt...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B3.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B1.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B10.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B4.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B5.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B6.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B11.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B2.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B7.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B8.TIF...\n", + "==> NOTE: You are downloading one or more large file(s), which would\n", "run significantly faster if you enabled sliced object downloads. This\n", "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B2.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_MTL.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B4.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20170913_20170928_01_T1/LC08_L1TP_025033_20170913_20170928_01_T1_B5.TIF...\n", - "- [14/14 files][979.2 MiB/979.2 MiB] 100% Done 14.9 MiB/s ETA 00:00:00 \n", - "Operation completed over 14 objects/979.2 MiB. \n" + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_B9.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_BQA.TIF...\n", + "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1TP_025033_20210519_20210528_01_T1/LC08_L1TP_025033_20210519_20210528_01_T1_MTL.txt...\n", + "/ [14/14 files][ 1.0 GiB/ 1.0 GiB] 100% Done 30.3 MiB/s ETA 00:00:00 \n", + "Operation completed over 14 objects/1.0 GiB. \n" ] } ], @@ -742,7 +745,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 21, "id": "a37c1567-14b5-4dc7-bc27-d1b84411fce1", "metadata": {}, "outputs": [ @@ -750,10 +753,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 2665792\n", - "drwxr-xr-x 2 learner learner 4096 Dec 17 19:02 \u001b[0m\u001b[01;34mLC08_L1TP_025033_20170913_20170928_01_T1\u001b[0m\n", - "drwxr-xr-x 2 learner learner 4096 Dec 17 19:01 \u001b[01;34mLC08_L1TP_025033_20211010_20211018_01_T1\u001b[0m\n", - "-rw-r--r-- 1 learner learner 2729754843 Dec 17 19:01 index.csv\n" + "total 2564796\n", + "drwxr-xr-x 2 learner learner 4096 Jan 21 21:33 \u001b[0m\u001b[01;34mLC08_L1TP_025033_20201007_20201016_01_T1\u001b[0m\n", + "drwxr-xr-x 2 learner learner 4096 Jan 21 21:33 \u001b[01;34mLC08_L1TP_025033_20210519_20210528_01_T1\u001b[0m\n", + "-rw-r--r-- 1 learner learner 2626336574 Jan 21 21:32 index.csv\n" ] } ], @@ -773,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 22, "id": "0c027e92-ae6f-4152-b8d6-5a70172de3e2", "metadata": {}, "outputs": [ @@ -809,7 +812,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 23, "id": "77999a80-7cfd-46d3-86a6-05d199f7e66d", "metadata": {}, "outputs": [ @@ -840,7 +843,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 24, "id": "b5a2b29b-9c1d-4376-a3eb-4a3dc4bac160", "metadata": { "scrolled": true, @@ -908,75 +911,75 @@ "Get:9 http://deb.debian.org/debian buster/main amd64 liblapack3 amd64 3.8.0-2 [2110 kB]\n", "Get:10 http://deb.debian.org/debian buster/main amd64 libarpack2 amd64 3.7.0-2 [102 kB]\n", "Get:11 http://deb.debian.org/debian buster/main amd64 libsuperlu5 amd64 5.2.1+dfsg1-4 [161 kB]\n", - "Get:12 http://deb.debian.org/debian buster/main amd64 libarmadillo9 amd64 1:9.200.7+dfsg-1 [88.6 kB]\n", - "Get:13 http://deb.debian.org/debian buster/main amd64 libcharls2 amd64 2.0.0+dfsg-1 [64.3 kB]\n", - "Get:14 http://deb.debian.org/debian buster/main amd64 libxml2 amd64 2.9.4+dfsg1-7+deb10u2 [689 kB]\n", - "Get:15 http://security.debian.org/debian-security buster/updates/main amd64 libtiff5 amd64 4.1.0+git191117-2~deb10u3 [271 kB]\n", - "Get:16 http://security.debian.org/debian-security buster/updates/main amd64 libnss3 amd64 2:3.42.1-1+deb10u4 [1161 kB]\n", + "Get:12 http://security.debian.org/debian-security buster/updates/main amd64 libtiff5 amd64 4.1.0+git191117-2~deb10u3 [271 kB]\n", + "Get:13 http://deb.debian.org/debian buster/main amd64 libarmadillo9 amd64 1:9.200.7+dfsg-1 [88.6 kB]\n", + "Get:14 http://security.debian.org/debian-security buster/updates/main amd64 libnss3 amd64 2:3.42.1-1+deb10u4 [1161 kB]\n", + "Get:15 http://deb.debian.org/debian buster/main amd64 libcharls2 amd64 2.0.0+dfsg-1 [64.3 kB]\n", + "Get:16 http://deb.debian.org/debian buster/main amd64 libxml2 amd64 2.9.4+dfsg1-7+deb10u2 [689 kB]\n", "Get:17 http://deb.debian.org/debian buster/main amd64 libdap25 amd64 3.20.3-1 [557 kB]\n", - "Get:18 http://deb.debian.org/debian buster/main amd64 libdapclient6v5 amd64 3.20.3-1 [202 kB]\n", - "Get:19 http://deb.debian.org/debian buster/main amd64 libdapserver7v5 amd64 3.20.3-1 [131 kB]\n", - "Get:20 http://security.debian.org/debian-security buster/updates/main amd64 libpq5 amd64 11.14-0+deb10u1 [171 kB]\n", - "Get:21 http://deb.debian.org/debian buster/main amd64 libepsilon1 amd64 0.9.2+dfsg-4 [42.0 kB]\n", - "Get:22 http://deb.debian.org/debian buster/main amd64 libfontconfig1 amd64 2.13.1-2 [346 kB]\n", - "Get:23 http://deb.debian.org/debian buster/main amd64 libfreexl1 amd64 1.0.5-3 [34.1 kB]\n", - "Get:24 http://deb.debian.org/debian buster/main amd64 libfyba0 amd64 4.1.1-6 [114 kB]\n", - "Get:25 http://deb.debian.org/debian buster/main amd64 libgeos-3.7.1 amd64 3.7.1-1 [735 kB]\n", - "Get:26 http://deb.debian.org/debian buster/main amd64 libgeos-c1v5 amd64 3.7.1-1 [299 kB]\n", - "Get:27 http://deb.debian.org/debian buster/main amd64 proj-data all 5.2.0-1 [6986 kB]\n", - "Get:28 http://deb.debian.org/debian buster/main amd64 libproj13 amd64 5.2.0-1 [225 kB]\n", - "Get:29 http://deb.debian.org/debian buster/main amd64 libjbig0 amd64 2.1-3.1+b2 [31.0 kB]\n", - "Get:30 http://deb.debian.org/debian buster/main amd64 libjpeg62-turbo amd64 1:1.5.2-2+deb10u1 [133 kB]\n", - "Get:31 http://deb.debian.org/debian buster/main amd64 libwebp6 amd64 0.6.1-2+deb10u1 [261 kB]\n", - "Get:32 http://deb.debian.org/debian buster/main amd64 libgeotiff2 amd64 1.4.3-1 [72.0 kB]\n", - "Get:33 http://deb.debian.org/debian buster/main amd64 libgif7 amd64 5.1.4-3 [43.3 kB]\n", - "Get:34 http://deb.debian.org/debian buster/main amd64 libhdf4-0-alt amd64 4.2.13-4 [277 kB]\n", - "Get:35 http://deb.debian.org/debian buster/main amd64 libsz2 amd64 1.0.2-1 [6676 B]\n", - "Get:36 http://deb.debian.org/debian buster/main amd64 libhdf5-103 amd64 1.10.4+repack-10 [1325 kB]\n", - "Get:37 http://deb.debian.org/debian buster/main amd64 libminizip1 amd64 1.1-8+b1 [20.4 kB]\n", - "Get:38 http://deb.debian.org/debian buster/main amd64 liburiparser1 amd64 0.9.1-1 [47.8 kB]\n", - "Get:39 http://deb.debian.org/debian buster/main amd64 libkmlbase1 amd64 1.3.0-7 [48.9 kB]\n", - "Get:40 http://deb.debian.org/debian buster/main amd64 libkmldom1 amd64 1.3.0-7 [172 kB]\n", - "Get:41 http://deb.debian.org/debian buster/main amd64 libkmlengine1 amd64 1.3.0-7 [78.4 kB]\n", - "Get:42 http://deb.debian.org/debian buster/main amd64 libkmlconvenience1 amd64 1.3.0-7 [50.1 kB]\n", - "Get:43 http://deb.debian.org/debian buster/main amd64 libkmlregionator1 amd64 1.3.0-7 [25.1 kB]\n", - "Get:44 http://deb.debian.org/debian buster/main amd64 libkmlxsd1 amd64 1.3.0-7 [34.6 kB]\n", - "Get:45 http://deb.debian.org/debian buster/main amd64 mysql-common all 5.8+1.0.5 [7324 B]\n", - "Get:46 http://deb.debian.org/debian buster/main amd64 mariadb-common all 1:10.3.31-0+deb10u1 [32.7 kB]\n", - "Get:47 http://deb.debian.org/debian buster/main amd64 libmariadb3 amd64 1:10.3.31-0+deb10u1 [173 kB]\n", - "Get:48 http://deb.debian.org/debian buster/main amd64 libnetcdf13 amd64 1:4.6.2-1 [403 kB]\n", - "Get:49 http://deb.debian.org/debian buster/main amd64 libltdl7 amd64 2.4.6-9 [390 kB]\n", - "Get:50 http://deb.debian.org/debian buster/main amd64 libodbc1 amd64 2.3.6-0.1 [223 kB]\n", - "Get:51 http://deb.debian.org/debian buster/main amd64 libogdi3.2 amd64 3.2.1+ds-4 [241 kB]\n", - "Get:52 http://deb.debian.org/debian buster/main amd64 libopenjp2-7 amd64 2.3.0-2+deb10u2 [158 kB]\n", - "Get:53 http://deb.debian.org/debian buster/main amd64 liblcms2-2 amd64 2.9-3 [145 kB]\n", - "Get:54 http://deb.debian.org/debian buster/main amd64 libnspr4 amd64 2:4.20-1 [112 kB]\n", - "Get:55 http://deb.debian.org/debian buster/main amd64 libpoppler82 amd64 0.71.0-5 [1507 kB]\n", - "Get:56 http://deb.debian.org/debian buster/main amd64 libqhull7 amd64 2015.2-4 [196 kB]\n", - "Get:57 http://deb.debian.org/debian buster/main amd64 libspatialite7 amd64 4.3.0a-5+b2 [1256 kB]\n", - "Get:58 http://deb.debian.org/debian buster/main amd64 libxerces-c3.2 amd64 3.2.2+debian-1+deb10u1 [886 kB]\n", - "Get:59 http://deb.debian.org/debian buster/main amd64 odbcinst amd64 2.3.6-0.1 [48.4 kB]\n", - "Get:60 http://deb.debian.org/debian buster/main amd64 odbcinst1debian2 amd64 2.3.6-0.1 [78.1 kB]\n", - "Get:61 http://deb.debian.org/debian buster/main amd64 libgdal20 amd64 2.4.0+dfsg-1+b1 [6171 kB]\n", - "Get:62 http://deb.debian.org/debian buster/main amd64 libxslt1.1 amd64 1.1.32-2.2~deb10u1 [237 kB]\n", - "Get:63 http://deb.debian.org/debian buster/main amd64 proj-bin amd64 5.2.0-1 [105 kB]\n", - "Get:64 http://deb.debian.org/debian buster/main amd64 python3-affine all 2.2.2-1 [14.5 kB]\n", - "Get:65 http://deb.debian.org/debian buster/main amd64 python3-attr all 18.2.0-1 [37.3 kB]\n", - "Get:66 http://deb.debian.org/debian buster/main amd64 python3-soupsieve all 1.8+dfsg-1 [27.6 kB]\n", - "Get:67 http://deb.debian.org/debian buster/main amd64 python3-bs4 all 4.7.1-1 [94.1 kB]\n", - "Get:68 http://deb.debian.org/debian buster/main amd64 python3-chardet all 3.0.4-3 [80.5 kB]\n", - "Get:69 http://deb.debian.org/debian buster/main amd64 python3-colorama all 0.3.7-1 [18.1 kB]\n", - "Get:70 http://deb.debian.org/debian buster/main amd64 python3-click all 7.0-1 [73.6 kB]\n", - "Get:71 http://deb.debian.org/debian buster/main amd64 python3-click-plugins all 1.0.4-1 [7540 B]\n", - "Get:72 http://deb.debian.org/debian buster/main amd64 python3-cligj all 0.5.0-1 [8212 B]\n", - "Get:73 http://deb.debian.org/debian buster/main amd64 python3-webencodings all 0.5.1-1 [10.9 kB]\n", - "Get:74 http://deb.debian.org/debian buster/main amd64 python3-html5lib all 1.0.1-1 [89.5 kB]\n", - "Get:75 http://deb.debian.org/debian buster/main amd64 python3-lxml amd64 4.3.2-1+deb10u3 [1162 kB]\n", + "Get:18 http://security.debian.org/debian-security buster/updates/main amd64 libpq5 amd64 11.14-0+deb10u1 [171 kB]\n", + "Get:19 http://security.debian.org/debian-security buster/updates/main amd64 python3-lxml amd64 4.3.2-1+deb10u4 [1163 kB]\n", + "Get:20 http://deb.debian.org/debian buster/main amd64 libdapclient6v5 amd64 3.20.3-1 [202 kB]\n", + "Get:21 http://deb.debian.org/debian buster/main amd64 libdapserver7v5 amd64 3.20.3-1 [131 kB]\n", + "Get:22 http://deb.debian.org/debian buster/main amd64 libepsilon1 amd64 0.9.2+dfsg-4 [42.0 kB]\n", + "Get:23 http://deb.debian.org/debian buster/main amd64 libfontconfig1 amd64 2.13.1-2 [346 kB]\n", + "Get:24 http://deb.debian.org/debian buster/main amd64 libfreexl1 amd64 1.0.5-3 [34.1 kB]\n", + "Get:25 http://deb.debian.org/debian buster/main amd64 libfyba0 amd64 4.1.1-6 [114 kB]\n", + "Get:26 http://deb.debian.org/debian buster/main amd64 libgeos-3.7.1 amd64 3.7.1-1 [735 kB]\n", + "Get:27 http://deb.debian.org/debian buster/main amd64 libgeos-c1v5 amd64 3.7.1-1 [299 kB]\n", + "Get:28 http://deb.debian.org/debian buster/main amd64 proj-data all 5.2.0-1 [6986 kB]\n", + "Get:29 http://deb.debian.org/debian buster/main amd64 libproj13 amd64 5.2.0-1 [225 kB]\n", + "Get:30 http://deb.debian.org/debian buster/main amd64 libjbig0 amd64 2.1-3.1+b2 [31.0 kB]\n", + "Get:31 http://deb.debian.org/debian buster/main amd64 libjpeg62-turbo amd64 1:1.5.2-2+deb10u1 [133 kB]\n", + "Get:32 http://deb.debian.org/debian buster/main amd64 libwebp6 amd64 0.6.1-2+deb10u1 [261 kB]\n", + "Get:33 http://deb.debian.org/debian buster/main amd64 libgeotiff2 amd64 1.4.3-1 [72.0 kB]\n", + "Get:34 http://deb.debian.org/debian buster/main amd64 libgif7 amd64 5.1.4-3 [43.3 kB]\n", + "Get:35 http://deb.debian.org/debian buster/main amd64 libhdf4-0-alt amd64 4.2.13-4 [277 kB]\n", + "Get:36 http://deb.debian.org/debian buster/main amd64 libsz2 amd64 1.0.2-1 [6676 B]\n", + "Get:37 http://deb.debian.org/debian buster/main amd64 libhdf5-103 amd64 1.10.4+repack-10 [1325 kB]\n", + "Get:38 http://deb.debian.org/debian buster/main amd64 libminizip1 amd64 1.1-8+b1 [20.4 kB]\n", + "Get:39 http://deb.debian.org/debian buster/main amd64 liburiparser1 amd64 0.9.1-1 [47.8 kB]\n", + "Get:40 http://deb.debian.org/debian buster/main amd64 libkmlbase1 amd64 1.3.0-7 [48.9 kB]\n", + "Get:41 http://deb.debian.org/debian buster/main amd64 libkmldom1 amd64 1.3.0-7 [172 kB]\n", + "Get:42 http://deb.debian.org/debian buster/main amd64 libkmlengine1 amd64 1.3.0-7 [78.4 kB]\n", + "Get:43 http://deb.debian.org/debian buster/main amd64 libkmlconvenience1 amd64 1.3.0-7 [50.1 kB]\n", + "Get:44 http://deb.debian.org/debian buster/main amd64 libkmlregionator1 amd64 1.3.0-7 [25.1 kB]\n", + "Get:45 http://deb.debian.org/debian buster/main amd64 libkmlxsd1 amd64 1.3.0-7 [34.6 kB]\n", + "Get:46 http://deb.debian.org/debian buster/main amd64 mysql-common all 5.8+1.0.5 [7324 B]\n", + "Get:47 http://deb.debian.org/debian buster/main amd64 mariadb-common all 1:10.3.31-0+deb10u1 [32.7 kB]\n", + "Get:48 http://deb.debian.org/debian buster/main amd64 libmariadb3 amd64 1:10.3.31-0+deb10u1 [173 kB]\n", + "Get:49 http://deb.debian.org/debian buster/main amd64 libnetcdf13 amd64 1:4.6.2-1 [403 kB]\n", + "Get:50 http://deb.debian.org/debian buster/main amd64 libltdl7 amd64 2.4.6-9 [390 kB]\n", + "Get:51 http://deb.debian.org/debian buster/main amd64 libodbc1 amd64 2.3.6-0.1 [223 kB]\n", + "Get:52 http://deb.debian.org/debian buster/main amd64 libogdi3.2 amd64 3.2.1+ds-4 [241 kB]\n", + "Get:53 http://deb.debian.org/debian buster/main amd64 libopenjp2-7 amd64 2.3.0-2+deb10u2 [158 kB]\n", + "Get:54 http://deb.debian.org/debian buster/main amd64 liblcms2-2 amd64 2.9-3 [145 kB]\n", + "Get:55 http://deb.debian.org/debian buster/main amd64 libnspr4 amd64 2:4.20-1 [112 kB]\n", + "Get:56 http://deb.debian.org/debian buster/main amd64 libpoppler82 amd64 0.71.0-5 [1507 kB]\n", + "Get:57 http://deb.debian.org/debian buster/main amd64 libqhull7 amd64 2015.2-4 [196 kB]\n", + "Get:58 http://deb.debian.org/debian buster/main amd64 libspatialite7 amd64 4.3.0a-5+b2 [1256 kB]\n", + "Get:59 http://deb.debian.org/debian buster/main amd64 libxerces-c3.2 amd64 3.2.2+debian-1+deb10u1 [886 kB]\n", + "Get:60 http://deb.debian.org/debian buster/main amd64 odbcinst amd64 2.3.6-0.1 [48.4 kB]\n", + "Get:61 http://deb.debian.org/debian buster/main amd64 odbcinst1debian2 amd64 2.3.6-0.1 [78.1 kB]\n", + "Get:62 http://deb.debian.org/debian buster/main amd64 libgdal20 amd64 2.4.0+dfsg-1+b1 [6171 kB]\n", + "Get:63 http://deb.debian.org/debian buster/main amd64 libxslt1.1 amd64 1.1.32-2.2~deb10u1 [237 kB]\n", + "Get:64 http://deb.debian.org/debian buster/main amd64 proj-bin amd64 5.2.0-1 [105 kB]\n", + "Get:65 http://deb.debian.org/debian buster/main amd64 python3-affine all 2.2.2-1 [14.5 kB]\n", + "Get:66 http://deb.debian.org/debian buster/main amd64 python3-attr all 18.2.0-1 [37.3 kB]\n", + "Get:67 http://deb.debian.org/debian buster/main amd64 python3-soupsieve all 1.8+dfsg-1 [27.6 kB]\n", + "Get:68 http://deb.debian.org/debian buster/main amd64 python3-bs4 all 4.7.1-1 [94.1 kB]\n", + "Get:69 http://deb.debian.org/debian buster/main amd64 python3-chardet all 3.0.4-3 [80.5 kB]\n", + "Get:70 http://deb.debian.org/debian buster/main amd64 python3-colorama all 0.3.7-1 [18.1 kB]\n", + "Get:71 http://deb.debian.org/debian buster/main amd64 python3-click all 7.0-1 [73.6 kB]\n", + "Get:72 http://deb.debian.org/debian buster/main amd64 python3-click-plugins all 1.0.4-1 [7540 B]\n", + "Get:73 http://deb.debian.org/debian buster/main amd64 python3-cligj all 0.5.0-1 [8212 B]\n", + "Get:74 http://deb.debian.org/debian buster/main amd64 python3-webencodings all 0.5.1-1 [10.9 kB]\n", + "Get:75 http://deb.debian.org/debian buster/main amd64 python3-html5lib all 1.0.1-1 [89.5 kB]\n", "Get:76 http://deb.debian.org/debian buster/main amd64 python3-numpy amd64 1:1.16.2-1 [2119 kB]\n", "Get:77 http://deb.debian.org/debian buster/main amd64 python3-pyparsing all 2.2.0+dfsg1-2 [89.6 kB]\n", "Get:78 http://deb.debian.org/debian buster/main amd64 python3-snuggs all 1.4.3-1 [7228 B]\n", "Get:79 http://deb.debian.org/debian buster/main amd64 python3-rasterio amd64 1.0.21-1 [818 kB]\n", - "Fetched 46.8 MB in 1s (85.4 MB/s) \n", + "Fetched 46.8 MB in 2s (28.9 MB/s) \n", "Extracting templates from packages: 100%\n", "Preconfiguring packages ...\n", "Selecting previously unselected package poppler-data.\n", @@ -1203,8 +1206,8 @@ "Preparing to unpack .../73-python3-html5lib_1.0.1-1_all.deb ...\n", "Unpacking python3-html5lib (1.0.1-1) ...\n", "Selecting previously unselected package python3-lxml:amd64.\n", - "Preparing to unpack .../74-python3-lxml_4.3.2-1+deb10u3_amd64.deb ...\n", - "Unpacking python3-lxml:amd64 (4.3.2-1+deb10u3) ...\n", + "Preparing to unpack .../74-python3-lxml_4.3.2-1+deb10u4_amd64.deb ...\n", + "Unpacking python3-lxml:amd64 (4.3.2-1+deb10u4) ...\n", "Selecting previously unselected package python3-numpy.\n", "Preparing to unpack .../75-python3-numpy_1%3a1.16.2-1_amd64.deb ...\n", "Unpacking python3-numpy (1:1.16.2-1) ...\n", @@ -1293,7 +1296,7 @@ "Setting up libpoppler82:amd64 (0.71.0-5) ...\n", "Setting up libnetcdf13:amd64 (1:4.6.2-1) ...\n", "Setting up python3-numpy (1:1.16.2-1) ...\n", - "Setting up python3-lxml:amd64 (4.3.2-1+deb10u3) ...\n", + "Setting up python3-lxml:amd64 (4.3.2-1+deb10u4) ...\n", "Setting up libarmadillo9 (1:9.200.7+dfsg-1) ...\n", "Setting up python3-snuggs (1.4.3-1) ...\n", "Setting up odbcinst (2.3.6-0.1) ...\n", @@ -1312,7 +1315,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 25, "id": "b9e367a0-26ce-42ce-bb04-6a432f41876e", "metadata": {}, "outputs": [ @@ -1320,10 +1323,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "LC08_L1TP_025033_20211010_20211018_01_T1 1\n", - "LC08_L1TP_025033_20211010_20211018_01_T1 2\n", - "LC08_L1TP_025033_20211010_20211018_01_T1 3\n", - "ERROR 4: output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png: No such file or directory\n" + "LC08_L1TP_025033_20201007_20201016_01_T1 1\n", + "LC08_L1TP_025033_20201007_20201016_01_T1 2\n", + "LC08_L1TP_025033_20201007_20201016_01_T1 3\n", + "ERROR 4: output/result-LC08_L1TP_025033_20201007_20201016_01_T1.png: No such file or directory\n" ] } ], @@ -1349,7 +1352,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 26, "id": "db9f26aa-6317-4834-8bf1-972c8b3cc032", "metadata": {}, "outputs": [ @@ -1357,9 +1360,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 195M\n", - "-rw-r--r-- 1 learner learner 195M Dec 17 19:03 \u001b[0m\u001b[01;35mresult-LC08_L1TP_025033_20211010_20211018_01_T1.png\u001b[0m\n", - "-rw-r--r-- 1 learner learner 910 Dec 17 19:03 result-LC08_L1TP_025033_20211010_20211018_01_T1.png.aux.xml\n" + "total 192M\n", + "-rw-r--r-- 1 learner learner 192M Jan 21 21:34 \u001b[0m\u001b[01;35mresult-LC08_L1TP_025033_20201007_20201016_01_T1.png\u001b[0m\n", + "-rw-r--r-- 1 learner learner 910 Jan 21 21:34 result-LC08_L1TP_025033_20201007_20201016_01_T1.png.aux.xml\n" ] } ], @@ -1379,7 +1382,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 27, "id": "9345472f-4ef3-490b-a80e-2462cd534c89", "metadata": {}, "outputs": [ @@ -1387,7 +1390,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "essentials-learner-2022-01-11\n" + "essentials-learner-2022-01-21\n" ] } ], @@ -1397,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 28, "id": "27dfae96-faf2-4d5d-8a78-97781841f172", "metadata": {}, "outputs": [ @@ -1405,7 +1408,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2022-01-11/\n" + "gs://essentials-learner-2022-01-21/\n" ] } ], @@ -1425,7 +1428,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 29, "id": "681e6b1d-98bb-448a-a57e-f5674214effd", "metadata": {}, "outputs": [ @@ -1433,8 +1436,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Copying file://output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png.aux.xml [Content-Type=application/xml]...\n", - "Copying file://output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png [Content-Type=image/png]...\n", + "Copying file://output/result-LC08_L1TP_025033_20201007_20201016_01_T1.png.aux.xml [Content-Type=application/xml]...\n", + "Copying file://output/result-LC08_L1TP_025033_20201007_20201016_01_T1.png [Content-Type=image/png]...\n", "==> NOTE: You are uploading one or more large file(s), which would run \n", "significantly faster if you enable parallel composite uploads. This\n", "feature can be enabled by editing the\n", @@ -1447,8 +1450,8 @@ "without a compiled crcmod, computing checksums on composite objects is\n", "so slow that gsutil disables downloads of composite objects.\n", "\n", - "\\ [2/2 files][194.0 MiB/194.0 MiB] 100% Done \n", - "Operation completed over 2 objects/194.0 MiB. \n" + "\\ [2/2 files][191.6 MiB/191.6 MiB] 100% Done \n", + "Operation completed over 2 objects/191.6 MiB. \n" ] } ], @@ -1466,7 +1469,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 30, "id": "248c47be-625f-44f5-a6b6-919e8d8baafd", "metadata": {}, "outputs": [ @@ -1474,7 +1477,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "gs://essentials-learner-2021-12-17/output/\n" + "gs://essentials-learner-2022-01-21/output/\n" ] } ], @@ -1484,7 +1487,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 31, "id": "b1ea18e9-5861-4479-9948-3303952dee8a", "metadata": {}, "outputs": [ @@ -1492,9 +1495,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "194.05 MiB 2021-12-17T19:03:54Z gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png\n", - " 910 B 2021-12-17T19:03:52Z gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png.aux.xml\n", - "TOTAL: 2 objects, 203478494 bytes (194.05 MiB)\n" + "191.58 MiB 2022-01-21T21:34:56Z gs://essentials-learner-2022-01-21/output/result-LC08_L1TP_025033_20201007_20201016_01_T1.png\n", + " 910 B 2022-01-21T21:34:54Z gs://essentials-learner-2022-01-21/output/result-LC08_L1TP_025033_20201007_20201016_01_T1.png.aux.xml\n", + "TOTAL: 2 objects, 200890195 bytes (191.58 MiB)\n" ] } ], @@ -1514,79 +1517,19 @@ }, { "cell_type": "markdown", - "id": "a1c11268-f389-405f-8d96-3c319a49b882", + "id": "40d9ab41-8920-45f2-8218-550baac5b069", "metadata": {}, "source": [ - "## Sharing Results (Optional)\n", - "\n", - "In order to share resources outside a project we must use the Identity Access Management service. This is a powerful tool to grant and restrict access to resources, and if not done correctly It can have serious consequences. **Incorrect permissions can lead to exposure of sensitive data, destruction of data, and authorized use of resources that can result in huge bills.** When in doubt, seek help.\n", - "\n", - "The question \"What access is really needed?\" is the **Principal of Least Privilege** and is a major cornerstone of security. We need to determine the lowest set of permissions or roles that is needed. In our case we wish to grant the \"Collaborator\" the \"Viewer\" access to the \"results bucket\". This will allow them to view, list, and download all objects in the bucket. This illustrates that for a **resource** a **member** (identity) is granted a **permission** (think of it in this order). Together this is called a **policy**. Google also uses \"Roles\" as a collection of predefined and managed permissions.\n", + "## Cleanup\n", "\n", - "What we do not want to do is add the collaborator to the project! This would give them access to all storage buckets and all resources." - ] - }, - { - "cell_type": "markdown", - "id": "50990a67-f901-48fd-9295-25ef73245142", - "metadata": { - "tags": [] - }, - "source": [ - "We will now add Members to a Bucket using the Web Console. We will use the Web Console to interactively build the policy binding by doing the following:\n", - " * Navigation Menu -> **Storage/Cloud Storage** -> Browser -> Click on the Bucket Name (**Bucket Details**) -> Select the **Permissions** tab -> Click **Add** next to \"Permissions\" above the permissions list.\n", - " * In the \"New Principals\" box add the Identity for the collaborator (another individual) as directed by the instructor.\n", - " * Select the \"Storage Object Viewer\" by typing \"Storage Object Viewer\" in the filter and then selecting \"Storage Object Viewer\". Do not use any \"Legacy Storage\" roles.\n", - " * Click \"Save\" to save the policy.\n", - " ![iam-storage-object-viewer](img/iam-storage-object-viewer.png)\n", - " * Verify the policy is listed in the \"Permissions\" table on the \"Bucket Details\" page (you should now be on this page).\n", + "We will now leave the resources running in order to learn more about sharing and monitoring costs and will clean up all the resources as the end of Lesson. \n", "\n", - "*Advanced Note: Changes in permissions will show up on the **Home** -> **Activity** page.*" - ] - }, - { - "cell_type": "markdown", - "id": "6a191922-7d5b-4125-9812-181eb056f86c", - "metadata": { - "tags": [] - }, - "source": [ - "Collaborators should now be able to see the contents of the bucket by explicitly naming the bucket. Below shows student321 accessing the bucket (note the prompt).\n", - "```bash\n", - "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-learner-2021-12-17\n", - "gs://essentials-learner-2021-12-17/output/\n", + "```{admonition} Danger\n", + ":class: danger\n", "\n", - "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-learner-2021-12-17/output\n", - "gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png\n", - "gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png.aux.xml\n", + "**Do not forget to do remove the Cloud Storage Bucket and the Compute Engine Instance (Virtual Machine) when you are done with this Lesson!** Running resources and stored data costs money! Cleanup when you are done!\n", "```" ] - }, - { - "cell_type": "markdown", - "id": "ce0909e5-f889-4e96-9947-8706416a5511", - "metadata": {}, - "source": [ - "Now remove the access by selecting the checkbox on the row with the principal identity and click `remove`. Now verify that the collaborator does not have access\n", - "\n", - "```bash\n", - "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-tmiddelkoop-$(date +%F)/output\n", - "AccessDeniedException: 403 student231@class.internet2.edu does not have storage.objects.list access to the Google Cloud Storage bucket.\n", - "```\n", - "\n", - "\n", - "*Instructors: You may want to have students share these buckets for the example to reduce screen flipping and involve the students.*" - ] - }, - { - "cell_type": "markdown", - "id": "40d9ab41-8920-45f2-8218-550baac5b069", - "metadata": {}, - "source": [ - "## Cleanup\n", - "\n", - "We will now leave the resources running in order to learn more about monitoring costs and will clean up all the resources as the end of Lesson. **Don't forget to do remove the Cloud Storage Bucket and the Compute Engine Instance (Virtual Machine) when you are done!**" - ] } ], "metadata": { diff --git a/content/GCP/06b_sharing_results.ipynb b/content/GCP/06b_sharing_results.ipynb new file mode 100644 index 0000000..4537754 --- /dev/null +++ b/content/GCP/06b_sharing_results.ipynb @@ -0,0 +1,116 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "dc57021c", + "metadata": {}, + "source": [ + "# Sharing Data (Optional)\n", + "\n", + "```{admonition} Overview\n", + ":class: tip\n", + "\n", + "**Teaching:** 30 min\n", + "\n", + "**Exercises:** 5 min (optional)\n", + "\n", + "**Questions:**\n", + "* How do I share my results?\n", + "\n", + "**Objectives:**\n", + "* Understand the basics of Identity and Access Management (IAM)\n", + "* Add collaborators to a Bucket with appropriate permissions.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "a1c11268-f389-405f-8d96-3c319a49b882", + "metadata": {}, + "source": [ + "## Sharing Results\n", + "\n", + "In order to share resources outside a project we must use the Identity Access Management service. This is a powerful tool to grant and restrict access to resources, and if not done correctly It can have serious consequences. **Incorrect permissions can lead to exposure of sensitive data, destruction of data, and authorized use of resources that can result in huge bills.** When in doubt, seek help.\n", + "\n", + "The question \"What access is really needed?\" is the **Principal of Least Privilege** and is a major cornerstone of security. We need to determine the lowest set of permissions or roles that is needed. In our case we wish to grant the \"Collaborator\" the \"Viewer\" access to the \"results bucket\". This will allow them to view, list, and download all objects in the bucket. This illustrates that for a **resource** a **member** (identity) is granted a **permission** (think of it in this order). Together this is called a **policy**. Google also uses \"Roles\" as a collection of predefined and managed permissions.\n", + "\n", + "What we do not want to do is add the collaborator to the project! This would give them access to all storage buckets and all resources." + ] + }, + { + "cell_type": "markdown", + "id": "50990a67-f901-48fd-9295-25ef73245142", + "metadata": { + "tags": [] + }, + "source": [ + "We will now add Members to a Bucket using the Web Console. We will use the Web Console to interactively build the policy binding by doing the following:\n", + " * Navigation Menu -> **Storage/Cloud Storage** -> Browser -> Click on the Bucket Name (**Bucket Details**) -> Select the **Permissions** tab -> Click **Add** next to \"Permissions\" above the permissions list.\n", + " * In the \"New Principals\" box add the Identity for the collaborator (another individual) as directed by the instructor.\n", + " * Select the \"Storage Object Viewer\" by typing \"Storage Object Viewer\" in the filter and then selecting \"Storage Object Viewer\". Do not use any \"Legacy Storage\" roles.\n", + " * Click \"Save\" to save the policy.\n", + " ![iam-storage-object-viewer](img/iam-storage-object-viewer.png)\n", + " * Verify the policy is listed in the \"Permissions\" table on the \"Bucket Details\" page (you should now be on this page).\n", + "\n", + "*Advanced Note: Changes in permissions will show up on the **Home** -> **Activity** page.*" + ] + }, + { + "cell_type": "markdown", + "id": "6a191922-7d5b-4125-9812-181eb056f86c", + "metadata": { + "tags": [] + }, + "source": [ + "Collaborators should now be able to see the contents of the bucket by explicitly naming the bucket. Below shows student321 accessing the bucket (note the prompt).\n", + "```bash\n", + "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-learner-2021-12-17\n", + "gs://essentials-learner-2021-12-17/output/\n", + "\n", + "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-learner-2021-12-17/output\n", + "gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png\n", + "gs://essentials-learner-2021-12-17/output/result-LC08_L1TP_025033_20211010_20211018_01_T1.png.aux.xml\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "e35d3a0b-aa8c-4c12-87d9-b218e2c16142", + "metadata": {}, + "source": [ + "Now remove the access by selecting the checkbox on the row with the principal identity and click `remove`. Now verify that the collaborator does not have access\n", + "\n", + "```bash\n", + "student231@cloudshell:~ (t-monument-315019)$ gsutil ls gs://essentials-tmiddelkoop-$(date +%F)/output\n", + "AccessDeniedException: 403 student231@class.internet2.edu does not have storage.objects.list access to the Google Cloud Storage bucket.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "4b93362c-c22b-453c-910b-db4a14ca58af", + "metadata": {}, + "source": [ + "```{admonition} Exercise\n", + "\n", + "*Instructors: You may want to have students share these buckets for the example to reduce screen flipping and involve the students.*\n", + "```" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Bash", + "language": "bash", + "name": "bash" + }, + "language_info": { + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/content/GCP/intro_to_GCP_Essentials.ipynb b/content/GCP/intro_to_GCP_Essentials.ipynb index 2bd2d17..6bff3c8 100644 --- a/content/GCP/intro_to_GCP_Essentials.ipynb +++ b/content/GCP/intro_to_GCP_Essentials.ipynb @@ -15,9 +15,16 @@ "4. [Introduction to the Cloud CLI](./04_intro_to_cli)\n", "4. [Using the Cloud Storage CLI](./05_cli_storage)\n", "6. [Running Analysis on the Cloud](./06_running_analysis)\n", + " and [Sharing Results](./06b_sharing_results)\n", "7. [Monitoring Costs](./07_monitoring_costs)\n", "8. [Cleaning up Resources and Best Practices](./08_cleaning_up_resources)\n" ] + }, + { + "cell_type": "markdown", + "id": "78058910", + "metadata": {}, + "source": [] } ], "metadata": { diff --git a/content/_toc.yml b/content/_toc.yml index 09aa2cf..649e2a9 100644 --- a/content/_toc.yml +++ b/content/_toc.yml @@ -41,6 +41,7 @@ parts: - file: GCP/04_intro_to_cli - file: GCP/05_cli_storage - file: GCP/06_running_analysis + - file: GCP/06b_sharing_results - file: GCP/07_monitoring_costs - file: GCP/08_cleaning_up_resources - file: GCP/glossary