CAMELS Brazil

This notebook explores CAMELS Brazil dataset which was introduced in VP Changas et al., 2020.

from ai4water.eda import EDA
from ai4water.datasets import CAMELS_BR
from ai4water.utils.utils import get_version_info

import pandas as pd
import numpy as np

import matplotlib.pyplot as plt

from easy_mpl import hist, ridge
from easy_mpl import scatter
from easy_mpl.utils import process_cbar

**********Tensorflow models could not be imported **********

C:\Users\ather\.conda\envs\cat_aware\lib\site-packages\sklearn\experimental\enable_hist_gradient_boosting.py:15: UserWarning: Since version 1.0, it is not needed to import enable_hist_gradient_boosting anymore. HistGradientBoostingClassifier and HistGradientBoostingRegressor are now stable and can be normally imported from sklearn.ensemble.
  warnings.warn(

for k,v in get_version_info().items():
    print(k, v)

python 3.8.17 (default, Jul  5 2023, 20:44:21) [MSC v.1916 64 bit (AMD64)]
os nt
ai4water 1.07
easy_mpl 0.21.3
SeqMetrics 1.3.4
numpy 1.24.3
pandas 1.3.4
matplotlib 3.6.0
sklearn 1.3.0
xarray 0.20.1
netCDF4 1.5.7
seaborn 0.12.2

dataset = CAMELS_BR(
    path=r"E:\data\gscad\CAMELS\CAMELS_BR"   # set path to None if you have not already downloaded data
)

    Not downloading the data since the directory
    E:\data\gscad\CAMELS\CAMELS_BR already exists.
    Use overwrite=True to remove previously saved files and download again

dataset.start

'19800101'

dataset.end

'20181231'

stations = dataset.stations()
len(stations)

593

Static Data

dataset.static_features

['p_mean',
 'pet_mean',
 'et_mean',
 'aridity',
 'p_seasonality',
 'asynchronicity',
 'frac_snow',
 'high_prec_freq',
 'high_prec_dur',
 'high_prec_timing',
 'low_prec_freq',
 'low_prec_dur',
 'low_prec_timing',
 'geol_class_1st',
 'geol_class_1st_perc',
 'geol_class_2nd',
 'geol_class_2nd_perc',
 'carb_rocks_perc',
 'geol_porosity',
 'geol_permeability',
 'consumptive_use',
 'consumptive_use_perc',
 'reservoirs_vol',
 'regulation_degree',
 'q_mean',
 'runoff_ratio',
 'stream_elas',
 'slope_fdc',
 'baseflow_index',
 'hfd_mean',
 'Q5',
 'Q95',
 'high_q_freq',
 'high_q_dur',
 'low_q_freq',
 'low_q_dur',
 'zero_q_freq',
 'crop_perc',
 'crop_mosaic_perc',
 'forest_perc',
 'shrub_perc',
 'grass_perc',
 'barren_perc',
 'imperv_perc',
 'wet_perc',
 'snow_perc',
 'dom_land_cover',
 'dom_land_cover_perc',
 'gauge_name',
 'gauge_region',
 'gauge_lat',
 'gauge_lon',
 'area_ana',
 'area_gsim',
 'area_gsim_quality',
 'q_quality_control_perc',
 'q_stream_stage_perc',
 'sand_perc',
 'silt_perc',
 'clay_perc',
 'org_carbon_content',
 'bedrock_depth',
 'water_table_depth',
 'elev_gauge',
 'elev_mean',
 'slope_mean',
 'area']

hfd = ''

static = dataset.fetch_static_features(stn_id=stations)
static.shape

(593, 67)

EDA(data=static, save=False).heatmap()

<AxesSubplot: ylabel='Examples'>

static = static.dropna(axis=1)
static.shape

(593, 61)

coords = dataset.stn_coords()
coords

	lat	long
gauge_id
36020000	-6.3244	-40.1617
36045000	-6.6461	-39.9617
87270000	-29.7011	-51.4411
29100000	-5.4867	-49.2214
84071000	-27.4142	-48.9528
...	...	...
26350000	-11.6181	-50.6625
50380000	-10.8450	-40.1128
26720000	-12.4178	-49.5892
74700000	-27.6742	-54.4625
45880000	-13.5586	-44.3031

593 rows × 2 columns

dataset.plot_stations()

<AxesSubplot: >

lat = coords['lat'].astype(float).values.reshape(-1,)
long = coords['long'].astype(float).values.reshape(-1,)
height = static['elev_mean']
slope = static['slope_mean']

idx = 0
ax_num = 0

fig, axes = plt.subplots(5, 5, figsize=(15, 12))
axes = axes.flatten()

while ax_num < 25 and idx<static.shape[1]:

    val = static.iloc[:, idx]
    idx += 1

    try:
        c = val.astype(float).values.reshape(-1,)

        en = static.shape[0]
        ax = axes[ax_num]
        ax, sc = scatter(long[0:en], lat[0:en], c=c[0:en], cmap="hot", show=False, ax=ax)

        process_cbar(ax, sc, border=False, title=val.name, #title_kws ={"fontsize": 14}
                    )
        ax_num += 1
    except ValueError:
        continue


plt.tight_layout()
plt.show()
print(idx)

27

idx = 27
ax_num = 0

fig, axes = plt.subplots(5, 5, figsize=(15, 12))
axes = axes.flatten()

while ax_num < 25 and idx<static.shape[1]:

    val = static.iloc[:, idx]
    idx += 1

    try:
        c = val.astype(float).values.reshape(-1,)

        en = static.shape[0]
        ax = axes[ax_num]
        ax, sc = scatter(long[0:en], lat[0:en], c=c[0:en], cmap="hot", show=False, ax=ax)

        process_cbar(ax, sc, border=False, title=val.name, #title_kws ={"fontsize": 14}
                    )
        ax_num += 1
    except ValueError:
        continue


plt.tight_layout()
plt.show()
print(idx)

56

idx = 56
ax_num = 0

fig, axes = plt.subplots(5, 5, figsize=(15, 12))
axes = axes.flatten()

while ax_num < 25 and idx<static.shape[1]:

    val = static.iloc[:, idx]
    idx += 1

    try:
        c = val.astype(float).values.reshape(-1,)

        en = static.shape[0]
        ax = axes[ax_num]
        ax, sc = scatter(long[0:en], lat[0:en], c=c[0:en], cmap="hot", show=False, ax=ax)

        process_cbar(ax, sc, border=False, title=val.name, #title_kws ={"fontsize": 14}
                    )
        ax_num += 1
    except ValueError:
        continue


plt.tight_layout()
plt.show()
print(idx)

61

Dynamic Features

dataset.dynamic_features

['streamflow_m3s',
 'streamflow_mm',
 'simulated_streamflow_m3s',
 'precipitation_cpc',
 'precipitation_mswep',
 'precipitation_chirps',
 'evapotransp_gleam',
 'evapotransp_mgb',
 'potential_evapotransp_gleam',
 'temperature_min',
 'temperature_mean',
 'temperature_max']

# difference betwen different pcp ???
# meaning of gleam ???

Streamflow

streamflow = dataset.q_mmd()
print(streamflow.shape)

(14245, 593)

EDA(data=streamflow, save=False).heatmap()

<AxesSubplot: ylabel='Examples'>

st = 0

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):

    hist(streamflow.iloc[:, idx].values.reshape(-1,),
         bins=20,
         ax=ax,
         show=False
        )

    idx += 1

plt.show()
print(idx)

49

st = 49

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):

    hist(streamflow.iloc[:, idx].values.reshape(-1,),
         bins=20,
         ax=ax,
         show=False
        )

    idx += 1

plt.show()
print(idx)

98

st = 98

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):

    hist(streamflow.iloc[:, idx].values.reshape(-1,),
         bins=20,
         ax=ax,
         show=False
        )

    idx += 1

plt.show()
print(idx)

147

st = 147

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):

    hist(streamflow.iloc[:, idx].values.reshape(-1,),
         bins=20,
         ax=ax,
         show=False
        )

    idx += 1

plt.show()
print(idx)

196

st = 196

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

245

st = 245

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

294

st = 294

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

343

st = 343

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

392

st = 392

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

441

st = 441

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

490

st = 490

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

539

st = 539

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = st
for _, ax in enumerate(axes.flat):


    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

588

fig, axes = plt.subplots(7, 7, figsize=(10, 10), sharey="all")

idx = 588
for _, ax in enumerate(axes.flat):

    if idx>=593:
        break

    hist(streamflow.iloc[:, idx].values.reshape(-1,),
     bins=20,
     ax=ax,
     show=False
    )

    idx += 1

plt.show()
print(idx)

593

# skewness of streamflow

_ = hist(streamflow.skew().values.reshape(-1,), bins=50)

potential evapotranspiration

pet = dataset.fetch(dynamic_features = 'potential_evapotransp_gleam', as_dataframe=True)
pet = pet.reset_index()
pet.index = pd.to_datetime(pet.pop('time'))
pet.pop('dynamic_features')
print(pet.shape)
EDA(data=pet, save=False).heatmap()

(14245, 593)

<AxesSubplot: ylabel='Examples'>

_ = hist(pet.skew().values.reshape(-1,), bins=50)

Air Temperature

temp = dataset.fetch(dynamic_features = 'temperature_mean', as_dataframe=True)
temp = temp.reset_index()
temp.index = pd.to_datetime(temp.pop('time'))
temp.pop('dynamic_features')

print(temp.shape)

EDA(data=temp, save=False).heatmap()

(14245, 593)

<AxesSubplot: ylabel='Examples'>

_ = hist(temp.skew().values.reshape(-1,), bins=50)

Precipitation

pcp = dataset.fetch(dynamic_features = 'precipitation_mswep', as_dataframe=True)
pcp = pcp.reset_index()
pcp.index = pd.to_datetime(pcp.pop('time'))
pcp.pop('dynamic_features')
print(pcp.shape)
EDA(data=pcp, save=False).heatmap()

(14245, 593)

<AxesSubplot: ylabel='Examples'>

_ = hist(pcp.skew().values.reshape(-1,), bins=50)