diff --git a/content/GCP/06_running_analysis.ipynb b/content/GCP/06_running_analysis.ipynb index 6a78156..cec1189 100644 --- a/content/GCP/06_running_analysis.ipynb +++ b/content/GCP/06_running_analysis.ipynb @@ -72,7 +72,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 6, "id": "0f2df994-202e-4e3e-96c2-1536d8adb036", "metadata": {}, "outputs": [ @@ -80,10 +80,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "Hit:1 http://security.debian.org/debian-security buster/updates InRelease\n", - "Hit:2 http://deb.debian.org/debian buster InRelease \u001b[0m\n", - "Hit:3 http://deb.debian.org/debian buster-updates InRelease \n", - "Hit:4 http://deb.debian.org/debian buster-backports InRelease \n", + "Hit:1 http://deb.debian.org/debian buster InRelease\n", + "Hit:2 http://deb.debian.org/debian buster-updates InRelease \u001b[0m\n", + "Hit:3 http://deb.debian.org/debian buster-backports InRelease \n", + "Hit:4 http://security.debian.org/debian-security buster/updates InRelease \n", "Hit:5 http://packages.cloud.google.com/apt cloud-sdk-buster InRelease\n", "Hit:6 http://packages.cloud.google.com/apt google-cloud-packages-archive-keyring-buster InRelease\n", "Hit:7 http://packages.cloud.google.com/apt google-compute-engine-buster-stable InRelease\n", @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 7, "id": "05fadd30-4f35-4a93-b713-cc132131b948", "metadata": {}, "outputs": [ @@ -133,7 +133,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 8, "id": "96db6a66-3fbf-419a-b8c8-dbb27639e990", "metadata": {}, "outputs": [], @@ -143,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 9, "id": "36554c99-ba08-4733-8ef2-e68d42d0d2b7", "metadata": {}, "outputs": [ @@ -151,11 +151,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "Cloning into 'CLASS-Examples'...\n", - "remote: Enumerating objects: 54, done.\u001b[K\n", - "remote: Total 54 (delta 0), reused 0 (delta 0), pack-reused 54\u001b[K\n", - "Unpacking objects: 100% (54/54), done.\n" + "fatal: destination path 'CLASS-Examples' already exists and is not an empty directory.\n" ] + }, + { + "ename": "", + "evalue": "128", + "output_type": "error", + "traceback": [] } ], "source": [ @@ -164,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "id": "90c1cda7-60d4-44bb-84f8-e776a77a94ab", "metadata": {}, "outputs": [], @@ -174,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 11, "id": "55b628d5-6e5c-45a5-9cd3-c129db9cdcd2", "metadata": {}, "outputs": [ @@ -182,14 +185,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "total 28\n", - "-rw-r--r-- 1 learner learner 960 Dec 13 15:11 ReadMe.md\n", - "-rw-r--r-- 1 learner learner 72 Dec 13 15:11 clean.sh\n", - "-rw-r--r-- 1 learner learner 514 Dec 13 15:11 combine.py\n", - "-rw-r--r-- 1 learner learner 280 Dec 13 15:11 download.sh\n", - "-rw-r--r-- 1 learner learner 314 Dec 13 15:11 get-index.sh\n", - "-rw-r--r-- 1 learner learner 76 Dec 13 15:11 search.json\n", - "-rw-r--r-- 1 learner learner 783 Dec 13 15:11 search.py\n" + "total 32\n", + "-rw-r--r-- 1 learner learner 960 Dec 13 15:11 ReadMe.md\n", + "-rw-r--r-- 1 learner learner 72 Dec 13 15:11 clean.sh\n", + "-rw-r--r-- 1 learner learner 514 Dec 13 15:11 combine.py\n", + "drwxr-xr-x 4 learner learner 4096 Dec 13 15:12 \u001b[0m\u001b[01;34mdata\u001b[0m\n", + "-rw-r--r-- 1 learner learner 280 Dec 13 15:11 download.sh\n", + "-rw-r--r-- 1 learner learner 314 Dec 13 15:11 get-index.sh\n", + "-rw-r--r-- 1 learner learner 76 Dec 13 15:11 search.json\n", + "-rw-r--r-- 1 learner learner 783 Dec 13 15:11 search.py\n" ] } ], @@ -209,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 12, "id": "e56ab74a-ae6d-4602-a26b-4a2656bd40cd", "metadata": {}, "outputs": [ @@ -246,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "id": "bbe85b75-c7cd-40ed-a3b0-37cbd0a5f52e", "metadata": {}, "outputs": [ @@ -254,8 +258,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "mkdir: created directory 'data'\n" + "mkdir: cannot create directory ‘data’: File exists\n" ] + }, + { + "ename": "", + "evalue": "1", + "output_type": "error", + "traceback": [] } ], "source": [ @@ -264,7 +274,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "id": "18a9b71c-5871-4ce2-a202-b48ad04e8d38", "metadata": {}, "outputs": [ @@ -278,7 +288,7 @@ "feature is enabled by default but requires that compiled crcmod be\n", "installed (see \"gsutil help crcmod\").\n", "\n", - "/ [1 files][760.7 MiB/760.7 MiB] \n", + "| [1 files][760.7 MiB/760.7 MiB] \n", "Operation completed over 1 objects/760.7 MiB. \n" ] } @@ -297,17 +307,17 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 24, "id": "2cdaf24c-c4aa-4e80-9236-939e7c982916", "metadata": {}, "outputs": [], "source": [ - "gzip -d data/index.csv.gz" + "gzip -df data/index.csv.gz" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 25, "id": "b005876c-f9af-43d6-80c6-f04295413b9b", "metadata": {}, "outputs": [ @@ -316,7 +326,9 @@ "output_type": "stream", "text": [ "total 2.6G\n", - "-rw-r--r-- 1 learner learner 2.6G Dec 13 15:11 index.csv\n" + "drwxr-xr-x 2 learner learner 4.0K Dec 13 15:13 \u001b[0m\u001b[01;34mLC08_L1GT_025033_20160215_20170224_01_T2\u001b[0m\n", + "drwxr-xr-x 2 learner learner 4.0K Dec 13 15:12 \u001b[01;34mLC08_L1GT_025033_20210127_20210305_01_T2\u001b[0m\n", + "-rw-r--r-- 1 learner learner 2.6G Dec 13 15:49 index.csv\n" ] } ], @@ -334,7 +346,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 26, "id": "ffe969db-d207-44fe-8957-8d129c76ee8f", "metadata": {}, "outputs": [ @@ -365,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 27, "id": "c5e300c3-e1f3-4cd4-9679-77725e61c4db", "metadata": {}, "outputs": [ @@ -409,7 +421,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 28, "id": "c9872510-4265-4b0e-aeb5-5a829ff69b24", "metadata": {}, "outputs": [ @@ -439,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 29, "id": "6912a9ec-0f9b-4500-ba20-d4280592b323", "metadata": {}, "outputs": [ @@ -468,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 30, "id": "3572c518-df83-4906-bfa6-a37bde2a5063", "metadata": {}, "outputs": [ @@ -503,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 31, "id": "cccec3e1-0dcd-4e3b-a059-a884f5219b66", "metadata": { "scrolled": true, @@ -515,49 +527,35 @@ "output_type": "stream", "text": [ "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B1.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_ANG.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B10.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B4.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B8.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B11.TIF...\n", - "==> NOTE: You are downloading one or more large file(s), which would \n", - "run significantly faster if you enabled sliced object downloads. This\n", - "feature is enabled by default but requires that compiled crcmod be\n", - "installed (see \"gsutil help crcmod\").\n", - "\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B9.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B5.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B3.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_BQA.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B7.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B2.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B6.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_MTL.txt...\n", - "| [14/14 files][841.7 MiB/841.7 MiB] 100% Done \n", - "Operation completed over 14 objects/841.7 MiB. \n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_ANG.txt\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B1.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B10.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B11.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B2.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B3.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B5.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B4.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B8.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B7.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B6.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_BQA.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_B9.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20210127_20210305_01_T2/LC08_L1GT_025033_20210127_20210305_01_T2_MTL.txt\n", "+++ gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_ANG.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B1.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B10.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B11.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B2.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B9.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B4.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B5.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B3.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B8.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_BQA.TIF...\n", - "==> NOTE: You are downloading one or more large file(s), which would\n", - "run significantly faster if you enabled sliced object downloads. This\n", - "feature is enabled by default but requires that compiled crcmod be\n", - "installed (see \"gsutil help crcmod\").\n", - "\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_MTL.txt...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B6.TIF...\n", - "Copying gs://gcp-public-data-landsat/LC08/01/025/033/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B7.TIF...\n", - "/ [14/14 files][976.2 MiB/976.2 MiB] 100% Done \n", - "Operation completed over 14 objects/976.2 MiB. \n" + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B10.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_ANG.txt\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B1.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B11.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B2.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B3.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B7.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B4.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B8.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B5.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B6.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_B9.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_BQA.TIF\n", + "Skipping existing item: file://data/LC08_L1GT_025033_20160215_20170224_01_T2/LC08_L1GT_025033_20160215_20170224_01_T2_MTL.txt\n" ] } ], @@ -575,7 +573,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 32, "id": "a37c1567-14b5-4dc7-bc27-d1b84411fce1", "metadata": {}, "outputs": [ @@ -586,7 +584,7 @@ "total 2664744\n", "drwxr-xr-x 2 learner learner 4096 Dec 13 15:13 \u001b[0m\u001b[01;34mLC08_L1GT_025033_20160215_20170224_01_T2\u001b[0m\n", "drwxr-xr-x 2 learner learner 4096 Dec 13 15:12 \u001b[01;34mLC08_L1GT_025033_20210127_20210305_01_T2\u001b[0m\n", - "-rw-r--r-- 1 learner learner 2728684265 Dec 13 15:11 index.csv\n" + "-rw-r--r-- 1 learner learner 2728684265 Dec 13 15:49 index.csv\n" ] } ], @@ -594,6 +592,172 @@ "ls -l data" ] }, + { + "cell_type": "markdown", + "id": "6073d5f6-68ef-41df-8044-73f221ce8780", + "metadata": {}, + "source": [ + "## Processing the Data\n", + "\n", + "We will now use simple script to combine the color bands and export it as a PNG" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "0c027e92-ae6f-4152-b8d6-5a70172de3e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "#!/usr/bin/python3\n", + "import os\n", + "import rasterio\n", + "\n", + "# Open the first directory in data, could walk entire tree\n", + "for dirname, dirs, files in os.walk('data'):\n", + " source = dirs[0]\n", + " break\n", + "\n", + "# Open band (B2/Blue) and copy metadata for result.\n", + "with rasterio.open(\"data/%s/%s_B2.TIF\" % (source, source)) as band2:\n", + " meta = band2.meta\n", + "\n", + "# Combine bands into PNG\n", + "meta.update(count = 3, driver='PNG')\n", + "with rasterio.open(\"output/result-%s.png\" % source, 'w+', **meta) as output:\n", + " for i in range(1, 4):\n", + " print(i)\n", + " output.write_band(i,rasterio.open(\"data/%s/%s_B%d.TIF\" % (source, source, i+3)).read(1))\n" + ] + } + ], + "source": [ + "cat combine.py" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "77999a80-7cfd-46d3-86a6-05d199f7e66d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Traceback (most recent call last):\n", + " File \"combine.py\", line 3, in \n", + " import rasterio\n", + "ModuleNotFoundError: No module named 'rasterio'\n" + ] + }, + { + "ename": "", + "evalue": "1", + "output_type": "error", + "traceback": [] + } + ], + "source": [ + "python3 combine.py" + ] + }, + { + "cell_type": "markdown", + "id": "52ada3ec-26cd-4e8b-94c2-8ed33e98be6e", + "metadata": {}, + "source": [ + "Oops, let's install the library (note: the output will be slightly different due to how this Lesson is built)\n", + "and while we are at it we also need to create the output directory" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "b5a2b29b-9c1d-4376-a3eb-4a3dc4bac160", + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "install: creating directory 'output/'\n", + "Requirement already satisfied: rasterio in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (1.2.10)\n", + "Requirement already satisfied: affine in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (2.3.0)\n", + "Requirement already satisfied: snuggs>=1.4.1 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (1.4.7)\n", + "Requirement already satisfied: click>=4.0 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (8.0.3)\n", + "Requirement already satisfied: numpy in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (1.21.4)\n", + "Requirement already satisfied: attrs in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (21.2.0)\n", + "Requirement already satisfied: cligj>=0.5 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (0.7.2)\n", + "Requirement already satisfied: certifi in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (2021.10.8)\n", + "Requirement already satisfied: setuptools in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (40.8.0)\n", + "Requirement already satisfied: click-plugins in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from rasterio) (1.1.1)\n", + "Requirement already satisfied: importlib-metadata in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from click>=4.0->rasterio) (4.8.2)\n", + "Requirement already satisfied: pyparsing>=2.1.6 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from snuggs>=1.4.1->rasterio) (3.0.6)\n", + "Requirement already satisfied: typing-extensions>=3.6.4 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->rasterio) (4.0.1)\n", + "Requirement already satisfied: zipp>=0.5 in /home/learner/CLASS-Essentials/.venv/lib/python3.7/site-packages (from importlib-metadata->click>=4.0->rasterio) (3.6.0)\n" + ] + } + ], + "source": [ + "install -dv output/\n", + "python3 -m pip install rasterio" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "b9e367a0-26ce-42ce-bb04-6a432f41876e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "2\n", + "3\n" + ] + } + ], + "source": [ + "python3 combine.py" + ] + }, + { + "cell_type": "markdown", + "id": "91d62964-d0ff-48d1-b215-a74ac233b3ef", + "metadata": {}, + "source": [ + "Verify the results were created." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "db9f26aa-6317-4834-8bf1-972c8b3cc032", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 165M\n", + "-rw-r--r-- 1 learner learner 165M Dec 13 16:09 \u001b[0m\u001b[01;35mresult-LC08_L1GT_025033_20210127_20210305_01_T2.png\u001b[0m\n", + "-rw-r--r-- 1 learner learner 941 Dec 13 16:09 result-LC08_L1GT_025033_20210127_20210305_01_T2.png.aux.xml\n" + ] + } + ], + "source": [ + "ls -lh output" + ] + }, { "cell_type": "markdown", "id": "c0257075-537c-4510-bafd-72e9756db17b", @@ -601,7 +765,7 @@ "source": [ "## Exporting the Results\n", "\n", - "Now that we have the data we will create a bucket to place the results. We will first create a bucket with a reasonable set of options.\n", + "Now that we have the output data we will create a bucket to place the results. We will first create a bucket with a reasonable set of options.\n", "\n", "We fisrt store the bucket name in the `BUCKET` environment variable for future use. This time we will specify a realistic set of options for a private bucket used for computation.\n", "\n", @@ -617,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 52, "id": "f36cb8c5-f305-4cb2-a5cc-0c9fd8592fb4", "metadata": {}, "outputs": [ @@ -637,7 +801,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 51, "id": "c2ae2b74-5e93-4c55-8bd7-63337f7dcbb8", "metadata": {}, "outputs": [ @@ -652,6 +816,86 @@ "source": [ "gsutil mb -b on -l $REGION --pap enforced \"gs://$BUCKET\"" ] + }, + { + "cell_type": "markdown", + "id": "22cc1043-386b-4990-b3dc-62cbdd7ba133", + "metadata": {}, + "source": [ + "Now copy the output data to the bucket. The `-r` flag recursively copies the output directory and `-m` copies the files in parallel. Note the locations of the `-m` and `-r` switches as they apply globally and to the `cp` command respectively." + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "681e6b1d-98bb-448a-a57e-f5674214effd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Copying file://output/result-LC08_L1GT_025033_20210127_20210305_01_T2.png.aux.xml [Content-Type=application/xml]...\n", + "Copying file://output/result-LC08_L1GT_025033_20210127_20210305_01_T2.png [Content-Type=image/png]...\n", + "==> NOTE: You are uploading one or more large file(s), which would run \n", + "significantly faster if you enable parallel composite uploads. This\n", + "feature can be enabled by editing the\n", + "\"parallel_composite_upload_threshold\" value in your .boto\n", + "configuration file. However, note that if you do this large files will\n", + "be uploaded as `composite objects\n", + "`_,which\n", + "means that any user who downloads such objects will need to have a\n", + "compiled crcmod installed (see \"gsutil help crcmod\"). This is because\n", + "without a compiled crcmod, computing checksums on composite objects is\n", + "so slow that gsutil disables downloads of composite objects.\n", + "\n", + "\\ [2/2 files][164.5 MiB/164.5 MiB] 100% Done \n", + "Operation completed over 2 objects/164.5 MiB. \n" + ] + } + ], + "source": [ + "gsutil -m cp -r \"output/\" \"gs://$BUCKET\"" + ] + }, + { + "cell_type": "markdown", + "id": "857861ca-9721-4a1e-a4ae-efffe2143b43", + "metadata": {}, + "source": [ + "Verify that the output was uploaded." + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "248c47be-625f-44f5-a6b6-919e8d8baafd", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "164.53 MiB 2021-12-13T16:19:49Z gs://essentials-learner-2021-12-13/result-LC08_L1GT_025033_20210127_20210305_01_T2.png\n", + " 941 B 2021-12-13T16:19:47Z gs://essentials-learner-2021-12-13/result-LC08_L1GT_025033_20210127_20210305_01_T2.png.aux.xml\n", + " gs://essentials-learner-2021-12-13/output/\n", + "TOTAL: 2 objects, 172521251 bytes (164.53 MiB)\n" + ] + } + ], + "source": [ + "gsutil ls -lh \"gs://$BUCKET\"" + ] + }, + { + "cell_type": "markdown", + "id": "66a41f97-3834-4d09-bfe5-c3f224628842", + "metadata": {}, + "source": [ + "## Viewing the Results\n", + "\n", + "You now can view the results by using the Google Cloud Platform Web Console and navigating to \"Cloud Storage\", selecting the bucket, and then the result object you wish to view (select the `.png` file). You will need to click the \"Preview\" button given the large size of the image." + ] } ], "metadata": {