!mamba install kerchunk -y --quiet
!mamba install h5py=3.2 -y --quiet
import numpy as np
import xarray as xr
import matplotlib.pyplot as plt
import s3fs
import datetime as dt
import logging
import fsspec
import ujson
import requests
from tqdm import tqdm
from glob import glob
import kerchunk.hdf
= {
s3_cred_endpoint 'podaac':'https://archive.podaac.earthdata.nasa.gov/s3credentials',
'lpdaac':'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials',
'ornldaac':'https://data.ornldaac.earthdata.nasa.gov/s3credentials'
}
def get_temp_creds():
= s3_cred_endpoint['ornldaac']
temp_creds_url return requests.get(temp_creds_url).json()
= get_temp_creds() temp_creds_req
#fs = fsspec.filesystem('s3', anon=True)
= s3fs.S3FileSystem(anon=False, key=temp_creds_req['accessKeyId'], secret=temp_creds_req['secretAccessKey'], token=temp_creds_req['sessionToken']) fs
= ['s3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2015.nc',
urls 's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2016.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2017.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2018.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2019.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2020.nc']
def gen_json(u):
= dict(
so ="rb", anon=False, default_fill_cache=False,
mode="none"
default_cache_type
)with fs.open(u, **so) as infile:
print(u)
= kerchunk.hdf.SingleHdf5ToZarr(infile, u, inline_threshold=300)
h5chunks with open(f"jsons/{u.split('/')[-1]}.json", 'wb') as outfile:
outfile.write(ujson.dumps(h5chunks.translate()).encode())
for u in urls:
gen_json(u)
= 's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2015.nc' u
= dict(
so ="rb", anon=False, default_fill_cache=False, default_cache_type="none"
mode )
= fs.open(u, **so) inf
= kerchunk.hdf.SingleHdf5ToZarr(inf, u, inline_threshold=300) h5chunks
h5chunks.translate()
PermissionError: The provided token has expired.