import rasterio as rio
from rasterio.session import AWSSession
import rioxarray
import boto3
import json
from pystac_client import Client
import hvplot.xarrayUSGS Landsat Cloud Data - Search and Access
Seach for LS Scenes using STAC API
stac_url = 'https://landsatlook.usgs.gov/stac-server'ls_cat = Client.open(stac_url)ls_cat<Client id=stac-server>
[f'{x.title} --- {x.id}' for x in ls_cat.get_children()]['Landsat Collection 1 Level-2 Albers Surface Temperature (ST) Product --- landsat-c1l2alb-st',
'Landsat Collection 2 Level-2 UTM Surface Reflectance (SR) Product --- landsat-c2l2-sr',
'Landsat Collection 2 Level-2 UTM Surface Temperature (ST) Product --- landsat-c2l2-st',
'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Surface Temperature (ST) Product --- landsat-c2ard-st',
'Landsat Collection 2 Level-2 Albers Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c2l2alb-bt',
'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c2ard-bt',
'Landsat Collection 2 Level-3 Fractional Snow Covered Area (fSCA) Product --- landsat-c2l3-fsca',
'Landsat Collection 1 Level-2 Albers Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c1l2alb-bt',
'Landsat Collection 2 Level-1 Product --- landsat-c2l1',
'Landsat Collection 2 Level-3 Burned Area (BA) Product --- landsat-c2l3-ba',
'Landsat Collection 1 Level-2 Albers Top of Atmosphere (TA) Reflectance Product --- landsat-c1l2alb-ta',
'Landsat Collection 2 Level-2 Albers Surface Temperature (ST) Product --- landsat-c2l2alb-st',
'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Surface Reflectance (SR) Product --- landsat-c2ard-sr',
'Landsat Collection 1 Level-1 Product --- landsat-c1l1',
'Landsat Collection 1 Level-2 Albers Surface Reflectance (SR) Product --- landsat-c1l2alb-sr',
'Landsat Collection 2 Level-2 Albers Top of Atmosphere (TA) Reflectance Product --- landsat-c2l2alb-ta',
'Landsat Collection 2 Level-2 Albers Surface Reflectance (SR) Product --- landsat-c2l2alb-sr',
'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Top of Atmosphere (TA) Reflectance Product --- landsat-c2ard-ta',
'Landsat Collection 2 Level-3 Dynamic Surface Water Extent (DSWE) Product --- landsat-c2l3-dswe']
list(ls_cat.get_all_collections())[<CollectionClient id=landsat-c1l2alb-st>,
<CollectionClient id=landsat-c2l2-sr>,
<CollectionClient id=landsat-c2l2-st>,
<CollectionClient id=landsat-c2ard-st>,
<CollectionClient id=landsat-c2l2alb-bt>,
<CollectionClient id=landsat-c2ard-bt>,
<CollectionClient id=landsat-c2l3-fsca>,
<CollectionClient id=landsat-c1l2alb-bt>,
<CollectionClient id=landsat-c2l1>,
<CollectionClient id=landsat-c2l3-ba>,
<CollectionClient id=landsat-c1l2alb-ta>,
<CollectionClient id=landsat-c2l2alb-st>,
<CollectionClient id=landsat-c2ard-sr>,
<CollectionClient id=landsat-c1l1>,
<CollectionClient id=landsat-c1l2alb-sr>,
<CollectionClient id=landsat-c2l2alb-ta>,
<CollectionClient id=landsat-c2l2alb-sr>,
<CollectionClient id=landsat-c2ard-ta>,
<CollectionClient id=landsat-c2l3-dswe>]
bbox = [-81.96624,26.38141,-81.80351,26.56457]datetime_range = '2021-10-17T00:00:00Z/2021-10-31T23:59:59Z'
datetime_range'2021-10-17T00:00:00Z/2021-10-31T23:59:59Z'
search = ls_cat.search(
collections = ['landsat-c2l1'], # Landsat C2 L1 - Copied ID from above. Must be in a list
bbox = bbox, # Specified above
datetime = datetime_range # Specified above
)search.matched()3
items = list(search.get_items())Print first 5 item ids and associated datetime
[f'{x.id} --- {x.datetime}' for x in items][:5]['LC08_L1TP_016042_20211027_20211104_02_T1 --- 2021-10-27 15:56:38.874611+00:00',
'LC08_L1TP_016041_20211027_20211104_02_T1 --- 2021-10-27 15:56:14.983570+00:00',
'LE07_L1TP_016042_20211019_20211114_02_T1 --- 2021-10-19 14:50:36.703778+00:00']
http_links = []
for i in items:
for a in i.assets.values():
a_link = json.loads(json.dumps(a.to_dict()))
if a_link['href'].endswith('.TIF'):
http_links.append(a_link['href'])
len(http_links)49
http_links[:10]['https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF',
'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF']
s3_links = []
for i in items:
for a in i.assets.values():
a_link = json.loads(json.dumps(a.to_dict()))
if 'alternate' in a_link:
s3_asset = a_link['alternate']['s3']['href']
if s3_asset.endswith('.TIF'):
s3_links.append(s3_asset)len(s3_links)49
s3_links[:10]['s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF',
's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF']
Data Access - rioxarray
Set working environment
session = boto3.Session()rio_env = rio.Env(AWSSession(session, requester_pays=True),
AWS_NO_SIGN_REQUEST='NO',
GDAL_DISABLE_READDIR_ON_OPEN='TRUE')
rio_env.__enter__()<rasterio.env.Env at 0x7f6e5276eb50>
s3_url = 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF'Read S3 asset as xarray datarray
da = rioxarray.open_rasterio(s3_url, chunks='auto').squeeze('band', drop=True)da<xarray.DataArray (y: 7821, x: 7661)>
dask.array<getitem, shape=(7821, 7661), dtype=uint16, chunksize=(7821, 7661), chunktype=numpy.ndarray>
Coordinates:
* x (x) float64 2.889e+05 2.889e+05 ... 5.187e+05 5.187e+05
* y (y) float64 2.992e+06 2.992e+06 ... 2.758e+06 2.758e+06
spatial_ref int64 0
Attributes:
_FillValue: 0.0
scale_factor: 1.0
add_offset: 0.0Plot scene
#da.hvplot(x='x', y='y', crs='epsg:32617', rasterize=True, width=800, height=600, tiles = 'ESRI') # TODO: how to exclude fill values
da.hvplot(x='x', y='y', crs='epsg:32617', rasterize=True, width=800, height=600)AWS CLI
List content in bucket
! aws s3 ls s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/ --request-payer requester2021-11-05 08:37:33 116979 LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt
2021-11-05 08:37:33 77713726 LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF
2021-11-05 08:37:34 76108899 LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF
2021-11-05 08:37:36 75282069 LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF
2021-11-05 08:37:37 77374352 LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF
2021-11-05 08:37:38 77527685 LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF
2021-11-05 08:37:39 76485245 LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF
2021-11-05 08:37:40 78304185 LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF
2021-11-05 08:37:42 75847018 LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF
2021-11-05 08:37:44 73697164 LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF
2021-11-05 08:37:45 304168225 LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF
2021-11-05 08:37:48 60967770 LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF
2021-11-05 08:37:49 15036 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json
2021-11-05 08:37:49 12369 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt
2021-11-05 08:37:49 18065 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml
2021-11-05 08:37:49 3527153 LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF
2021-11-05 08:37:49 222983 LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF
2021-11-05 08:37:49 2052489 LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF
2021-11-05 08:37:49 1783702 LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF
2021-11-05 08:37:50 8239344 LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF
2021-11-05 08:37:50 2930826 LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF
2021-11-05 08:37:50 26399 LC08_L1TP_016042_20211027_20211104_02_T1_stac.json
2021-11-05 08:37:50 46747 LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg
2021-11-05 08:37:50 6015 LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg
Download content in bucket to local
! aws s3 cp s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/ . --recursive --request-payer download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt to ./LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_stac.json to ./LC08_L1TP_016042_20211027_20211104_02_T1_stac.json
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg to ./LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg to ./LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF