From 85cbfe43e7beaa2171849a6cbf263e028397a947 Mon Sep 17 00:00:00 2001
From: John Hicks <jhicks@internet2.edu>
Date: Mon, 21 Feb 2022 14:01:40 -0500
Subject: [PATCH 1/2] added azure landsat analysis script
---
azure-landsat/landsat-env | 7 ++
azure-landsat/landsat-hls-azure.py | 169 +++++++++++++++++++++++++++++
2 files changed, 176 insertions(+)
create mode 100644 azure-landsat/landsat-env
create mode 100755 azure-landsat/landsat-hls-azure.py
diff --git a/azure-landsat/landsat-env b/azure-landsat/landsat-env
new file mode 100644
index 0000000..a781529
--- /dev/null
+++ b/azure-landsat/landsat-env
@@ -0,0 +1,7 @@
+sudo apt install python3-pip -y
+pip3 install --upgrade pip
+pip3 install numpy
+pip3 install matplotlib
+pip3 install pandas
+pip3 install rasterio
+pip3 install azure.storage.blob
diff --git a/azure-landsat/landsat-hls-azure.py b/azure-landsat/landsat-hls-azure.py
new file mode 100755
index 0000000..3322ce4
--- /dev/null
+++ b/azure-landsat/landsat-hls-azure.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+# need to install - pip install azure-storage-blob
+# https://nbviewer.org/github/microsoft/AIforEarthDataSets/blob/main/data/hls.ipynb
+
+# Environment setup
+import requests
+import numpy as np
+import io
+import matplotlib.pyplot as plt
+import pandas as pd
+
+import rasterio
+from azure.storage.blob import ContainerClient
+
+container_name = 'hls'
+storage_account_name = 'hlssa'
+storage_account_url = 'https://' + storage_account_name + '.blob.core.windows.net/'
+hls_blob_root = storage_account_url + container_name
+
+# This file is provided by NASA; it indicates the lat/lon extents of each
+# hls tile.
+#
+# The file originally comes from:
+#
+# https://hls.gsfc.nasa.gov/wp-content/uploads/2016/10/S2_TilingSystem2-1.txt
+#
+# ...but as of 8/2019, there is a bug with the column names in the original file, so we
+# access a copy with corrected column names.
+hls_tile_extents_url = 'https://ai4edatasetspublicassets.blob.core.windows.net/assets/S2_TilingSystem2-1.txt'
+
+# Load this file into a table, where each row is:
+#
+# Tile ID, Xstart, Ystart, UZ, EPSG, MinLon, MaxLon, MinLon, MaxLon
+s = requests.get(hls_tile_extents_url).content
+hls_tile_extents = pd.read_csv(io.StringIO(s.decode('utf-8')),delimiter=r'\s+')
+print('Read tile extents for {} tiles'.format(len(hls_tile_extents)))
+
+hls_container_client = ContainerClient(account_url=storage_account_url,
+ container_name=container_name,
+ credential=None)
+
+# Functions
+
+def list_available_tiles(prefix):
+ """
+ List all blobs in an Azure blob container matching a prefix.
+
+ We'll use this to query tiles by location and year.
+ """
+
+ files = []
+ generator = hls_container_client.list_blobs(name_starts_with=prefix)
+ for blob in generator:
+ files.append(blob.name)
+ return files
+
+
+def lat_lon_to_hls_tile_id(lat,lon):
+ """
+ Get the hls tile ID for a given lat/lon coordinate pair.
+ """
+
+ found_matching_tile = False
+
+ for i_row,row in hls_tile_extents.iterrows():
+ found_matching_tile = lat >= row.MinLat and lat <= row.MaxLat \
+ and lon >= row.MinLon and lon <= row.MaxLon
+ if found_matching_tile:
+ break
+
+ if not found_matching_tile:
+ return None
+ else:
+ return row.TilID
+
+# Build a tile path from components, including lat/lon lookup
+# Near Bear Lake, UT
+# lat = 41.89655047211277; lon = -111.4132464403312
+lat = 39.768402; lon = -86.158066
+
+tile_id = lat_lon_to_hls_tile_id(lat,lon)
+print('Using Tile ID {}'.format(tile_id))
+
+year = '2019'
+daynum = '001' # 1-indexed day-of-year
+product = 'S30' # 'S30' for Sentinel, 'L30' for Landsat
+sband = '_01'
+version = 'v1.4' # Currently always v1.4
+
+blob_name = product + '/HLS.' + product + '.T' + tile_id + '.' + year + daynum + '.' + version \
+ + sband + '.tif'
+sentinel_url = hls_blob_root + '/' + blob_name
+print('Constructed tile URL:\n{}'.format(sentinel_url))
+
+# Display an RGB composite of the Sentinel-2 image we just chose
+# Bands 2, 3, and 4 are B, G, and R in Sentinel-2 HLS images
+composite_bands = [4,3,2]
+
+urls = [sentinel_url.replace(sband,'_' + str(b).zfill(2)) for b in composite_bands]
+print('Rendering URLs:')
+for s in urls:
+ print(s)
+
+image_data = []
+
+composite_downsample_factor = 5
+composite_norm_value = 15000
+
+for fn in urls:
+ with rasterio.open(fn,'r') as raster:
+ h = int(raster.height/composite_downsample_factor)
+ w = int(raster.width/composite_downsample_factor)
+ band_array = raster.read(1, out_shape=(1, h, w))
+ raster.close()
+ band_array = band_array / composite_norm_value
+ image_data.append(band_array)
+
+rgb = np.dstack((image_data[0],image_data[1],image_data[2]))
+np.clip(rgb,0,1,rgb)
+
+dpi = 100; fig = plt.figure(frameon=False,figsize=(w/dpi,h/dpi),dpi=dpi)
+ax = plt.Axes(fig,[0., 0., 1., 1.]); ax.set_axis_off(); fig.add_axes(ax)
+plt.imshow(rgb);
+plt.imsave('test.png', rgb)
+
+i_daynum = int(daynum)
+product = 'L30'
+
+# Try days sequentially until we find one where there's a Landsat image for this day
+while True:
+
+ daynum = str(i_daynum).zfill(3)
+ prefix = product + '/HLS.' + product + '.T' + tile_id + '.' + year + daynum
+ print('Finding tiles with prefix {}'.format(prefix))
+ matches = list_available_tiles(prefix)
+
+ if len(matches) == 0:
+ print('No matching tiles')
+ i_daynum += 1
+ else:
+ break
+
+blob_name = matches[0]
+landsat_url = hls_blob_root + '/' + blob_name
+print('Found {} matching tiles, e.g.:\n{}'.format(len(matches),landsat_url))
+
+# Display a composite of the Landsat image we found
+urls = [landsat_url.replace(sband,'_' + str(b).zfill(2)) for b in composite_bands]
+print('Rendering URLs:')
+for s in urls:
+ print(s)
+
+image_data = []
+for fn in urls:
+ with rasterio.open(fn,'r') as raster:
+ h = int(raster.height/composite_downsample_factor)
+ w = int(raster.width/composite_downsample_factor)
+ band_array = raster.read(1, out_shape=(1, h, w))
+ raster.close()
+ band_array = band_array / composite_norm_value
+ image_data.append(band_array)
+
+rgb = np.dstack((image_data[0],image_data[1],image_data[2]))
+np.clip(rgb,0,1,rgb)
+
+dpi = 100; fig = plt.figure(frameon=False,figsize=(w/dpi,h/dpi),dpi=dpi)
+ax = plt.Axes(fig,[0., 0., 1., 1.]); ax.set_axis_off(); fig.add_axes(ax)
+plt.imshow(rgb);
+plt.show()
From 02b94436587cb8457fe7430526f1060ff23b0723 Mon Sep 17 00:00:00 2001
From: John Hicks <jhicks@internet2.edu>
Date: Fri, 25 Feb 2022 16:31:08 -0500
Subject: [PATCH 2/2] added to setup.sh azure-landset
---
azure-landsat/{landsat-env => setup.sh} | 2 ++
1 file changed, 2 insertions(+)
rename azure-landsat/{landsat-env => setup.sh} (93%)
mode change 100644 => 100755
diff --git a/azure-landsat/landsat-env b/azure-landsat/setup.sh
old mode 100644
new mode 100755
similarity index 93%
rename from azure-landsat/landsat-env
rename to azure-landsat/setup.sh
index a781529..85b49e7
--- a/azure-landsat/landsat-env
+++ b/azure-landsat/setup.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
sudo apt install python3-pip -y
pip3 install --upgrade pip
pip3 install numpy