USGS Landsat Cloud Data - Search and Access

import rasterio as rio
from rasterio.session import AWSSession
import rioxarray
import boto3
import json
from pystac_client import Client
import hvplot.xarray

Seach for LS Scenes using STAC API

stac_url = 'https://landsatlook.usgs.gov/stac-server'
ls_cat = Client.open(stac_url)
ls_cat
<Client id=stac-server>
[f'{x.title} --- {x.id}' for x in ls_cat.get_children()]
['Landsat Collection 1 Level-2 Albers Surface Temperature (ST) Product --- landsat-c1l2alb-st',
 'Landsat Collection 2 Level-2 UTM Surface Reflectance (SR) Product --- landsat-c2l2-sr',
 'Landsat Collection 2 Level-2 UTM Surface Temperature (ST) Product --- landsat-c2l2-st',
 'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Surface Temperature (ST) Product --- landsat-c2ard-st',
 'Landsat Collection 2 Level-2 Albers Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c2l2alb-bt',
 'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c2ard-bt',
 'Landsat Collection 2 Level-3 Fractional Snow Covered Area (fSCA) Product --- landsat-c2l3-fsca',
 'Landsat Collection 1 Level-2 Albers Top of Atmosphere Brightness Temperature (BT) Product --- landsat-c1l2alb-bt',
 'Landsat Collection 2 Level-1 Product --- landsat-c2l1',
 'Landsat Collection 2 Level-3 Burned Area (BA) Product --- landsat-c2l3-ba',
 'Landsat Collection 1 Level-2 Albers Top of Atmosphere (TA) Reflectance Product --- landsat-c1l2alb-ta',
 'Landsat Collection 2 Level-2 Albers Surface Temperature (ST) Product --- landsat-c2l2alb-st',
 'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Surface Reflectance (SR) Product --- landsat-c2ard-sr',
 'Landsat Collection 1 Level-1 Product --- landsat-c1l1',
 'Landsat Collection 1 Level-2 Albers Surface Reflectance (SR) Product --- landsat-c1l2alb-sr',
 'Landsat Collection 2 Level-2 Albers Top of Atmosphere (TA) Reflectance Product --- landsat-c2l2alb-ta',
 'Landsat Collection 2 Level-2 Albers Surface Reflectance (SR) Product --- landsat-c2l2alb-sr',
 'Landsat Collection 2 Analysis Ready Data (ARD) Level-2 UTM Top of Atmosphere (TA) Reflectance Product --- landsat-c2ard-ta',
 'Landsat Collection 2 Level-3 Dynamic Surface Water Extent (DSWE) Product --- landsat-c2l3-dswe']
list(ls_cat.get_all_collections())
[<CollectionClient id=landsat-c1l2alb-st>,
 <CollectionClient id=landsat-c2l2-sr>,
 <CollectionClient id=landsat-c2l2-st>,
 <CollectionClient id=landsat-c2ard-st>,
 <CollectionClient id=landsat-c2l2alb-bt>,
 <CollectionClient id=landsat-c2ard-bt>,
 <CollectionClient id=landsat-c2l3-fsca>,
 <CollectionClient id=landsat-c1l2alb-bt>,
 <CollectionClient id=landsat-c2l1>,
 <CollectionClient id=landsat-c2l3-ba>,
 <CollectionClient id=landsat-c1l2alb-ta>,
 <CollectionClient id=landsat-c2l2alb-st>,
 <CollectionClient id=landsat-c2ard-sr>,
 <CollectionClient id=landsat-c1l1>,
 <CollectionClient id=landsat-c1l2alb-sr>,
 <CollectionClient id=landsat-c2l2alb-ta>,
 <CollectionClient id=landsat-c2l2alb-sr>,
 <CollectionClient id=landsat-c2ard-ta>,
 <CollectionClient id=landsat-c2l3-dswe>]
bbox = [-81.96624,26.38141,-81.80351,26.56457]
datetime_range = '2021-10-17T00:00:00Z/2021-10-31T23:59:59Z'
datetime_range
'2021-10-17T00:00:00Z/2021-10-31T23:59:59Z'
search = ls_cat.search(
    collections = ['landsat-c2l1'],       # Landsat C2 L1 - Copied ID from above. Must be in a list
    bbox = bbox,                          # Specified above
    datetime = datetime_range             # Specified above
)
search.matched()
3
items = list(search.get_items())

Print first 5 item ids and associated datetime

[f'{x.id} --- {x.datetime}' for x in items][:5]
['LC08_L1TP_016042_20211027_20211104_02_T1 --- 2021-10-27 15:56:38.874611+00:00',
 'LC08_L1TP_016041_20211027_20211104_02_T1 --- 2021-10-27 15:56:14.983570+00:00',
 'LE07_L1TP_016042_20211019_20211114_02_T1 --- 2021-10-19 14:50:36.703778+00:00']
http_links = []

for i in items:
    for a in i.assets.values():
        a_link = json.loads(json.dumps(a.to_dict()))
        if a_link['href'].endswith('.TIF'):
            http_links.append(a_link['href'])
        
len(http_links)
49
http_links[:10]
['https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF',
 'https://landsatlook.usgs.gov/data/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF']
s3_links = []

for i in items:
    for a in i.assets.values():
        a_link = json.loads(json.dumps(a.to_dict()))
        if 'alternate' in a_link:
            s3_asset = a_link['alternate']['s3']['href']
            if s3_asset.endswith('.TIF'):
                s3_links.append(s3_asset)
len(s3_links)
49
s3_links[:10]
['s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF',
 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF']

Data Access - rioxarray

Set working environment

session = boto3.Session()
rio_env = rio.Env(AWSSession(session, requester_pays=True), 
                  AWS_NO_SIGN_REQUEST='NO',
                  GDAL_DISABLE_READDIR_ON_OPEN='TRUE')
rio_env.__enter__()
<rasterio.env.Env at 0x7f6e5276eb50>
s3_url = 's3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF'

Read S3 asset as xarray datarray

da = rioxarray.open_rasterio(s3_url, chunks='auto').squeeze('band', drop=True)
da
<xarray.DataArray (y: 7821, x: 7661)>
dask.array<getitem, shape=(7821, 7661), dtype=uint16, chunksize=(7821, 7661), chunktype=numpy.ndarray>
Coordinates:
  * x            (x) float64 2.889e+05 2.889e+05 ... 5.187e+05 5.187e+05
  * y            (y) float64 2.992e+06 2.992e+06 ... 2.758e+06 2.758e+06
    spatial_ref  int64 0
Attributes:
    _FillValue:    0.0
    scale_factor:  1.0
    add_offset:    0.0

Plot scene

#da.hvplot(x='x', y='y', crs='epsg:32617', rasterize=True, width=800, height=600, tiles = 'ESRI') # TODO: how to exclude fill values
da.hvplot(x='x', y='y', crs='epsg:32617', rasterize=True, width=800, height=600)

AWS CLI

List content in bucket

! aws s3 ls s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/ --request-payer requester
2021-11-05 08:37:33     116979 LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt
2021-11-05 08:37:33   77713726 LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF
2021-11-05 08:37:34   76108899 LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF
2021-11-05 08:37:36   75282069 LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF
2021-11-05 08:37:37   77374352 LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF
2021-11-05 08:37:38   77527685 LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF
2021-11-05 08:37:39   76485245 LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF
2021-11-05 08:37:40   78304185 LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF
2021-11-05 08:37:42   75847018 LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF
2021-11-05 08:37:44   73697164 LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF
2021-11-05 08:37:45  304168225 LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF
2021-11-05 08:37:48   60967770 LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF
2021-11-05 08:37:49      15036 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json
2021-11-05 08:37:49      12369 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt
2021-11-05 08:37:49      18065 LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml
2021-11-05 08:37:49    3527153 LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF
2021-11-05 08:37:49     222983 LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF
2021-11-05 08:37:49    2052489 LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF
2021-11-05 08:37:49    1783702 LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF
2021-11-05 08:37:50    8239344 LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF
2021-11-05 08:37:50    2930826 LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF
2021-11-05 08:37:50      26399 LC08_L1TP_016042_20211027_20211104_02_T1_stac.json
2021-11-05 08:37:50      46747 LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg
2021-11-05 08:37:50       6015 LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg

Download content in bucket to local

! aws s3 cp s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/ . --recursive --request-payer 
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt to ./LC08_L1TP_016042_20211027_20211104_02_T1_ANG.txt
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B10.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B2.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B1.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B11.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B6.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B3.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B4.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B5.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.json
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.txt
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml to ./LC08_L1TP_016042_20211027_20211104_02_T1_MTL.xml
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B7.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_SZA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_SAA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_QA_RADSAT.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_QA_PIXEL.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_VAA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_stac.json to ./LC08_L1TP_016042_20211027_20211104_02_T1_stac.json
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg to ./LC08_L1TP_016042_20211027_20211104_02_T1_thumb_large.jpeg
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg to ./LC08_L1TP_016042_20211027_20211104_02_T1_thumb_small.jpeg
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B9.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_VZA.TIF
download: s3://usgs-landsat/collection02/level-1/standard/oli-tirs/2021/016/042/LC08_L1TP_016042_20211027_20211104_02_T1/LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF to ./LC08_L1TP_016042_20211027_20211104_02_T1_B8.TIF