!mamba install kerchunk -y --quiet!mamba install h5py=3.2 -y --quietimport numpy as np
import xarray as xr
import matplotlib.pyplot as plt
import s3fs
import datetime as dt
import logging
import fsspec
import ujson
import requests
from tqdm import tqdm
from glob import globimport kerchunk.hdfs3_cred_endpoint = {
'podaac':'https://archive.podaac.earthdata.nasa.gov/s3credentials',
'lpdaac':'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials',
'ornldaac':'https://data.ornldaac.earthdata.nasa.gov/s3credentials'
}def get_temp_creds():
temp_creds_url = s3_cred_endpoint['ornldaac']
return requests.get(temp_creds_url).json()temp_creds_req = get_temp_creds()#fs = fsspec.filesystem('s3', anon=True)
fs = s3fs.S3FileSystem(anon=False, key=temp_creds_req['accessKeyId'], secret=temp_creds_req['secretAccessKey'], token=temp_creds_req['sessionToken'])urls = ['s3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2015.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2016.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2017.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2018.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2019.nc',
's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2020.nc']def gen_json(u):
so = dict(
mode="rb", anon=False, default_fill_cache=False,
default_cache_type="none"
)
with fs.open(u, **so) as infile:
print(u)
h5chunks = kerchunk.hdf.SingleHdf5ToZarr(infile, u, inline_threshold=300)
with open(f"jsons/{u.split('/')[-1]}.json", 'wb') as outfile:
outfile.write(ujson.dumps(h5chunks.translate()).encode())for u in urls:
gen_json(u)u = 's3://ornl-cumulus-prod-protected/daymet/Daymet_Daily_V4/data/daymet_v4_daily_na_tmax_2015.nc'so = dict(
mode="rb", anon=False, default_fill_cache=False, default_cache_type="none"
)inf = fs.open(u, **so)h5chunks = kerchunk.hdf.SingleHdf5ToZarr(inf, u, inline_threshold=300)h5chunks.translate()PermissionError: The provided token has expired.