LETID Outdoor Geospatial Demo (HPC)

LETID Outdoor Geospatial Demo (HPC)#

PVDeg Logo

import matplotlib.pyplot as plt
import pandas as pd
import pvdeg
from pvdeg import DATA_DIR
import getpass
import os

# This information helps with debugging and getting support :)
import sys
import platform

print("Working on a ", platform.system(), platform.release())
print("Python version ", sys.version)
print("Pandas version ", pd.__version__)
print("pvdeg version ", pvdeg.__version__)

Working on a  Linux 4.18.0-477.10.1.el8_8.x86_64
Python version  3.13.14 | packaged by Anaconda, Inc. | (main, Jun 17 2026, 20:12:46) [GCC 14.3.0]
Pandas version  3.0.3
pvdeg version  0.7.2.dev4+g321beb370

Single location example#

weather_file = os.path.join(DATA_DIR, "psm3_demo.csv")
WEATHER, META = pvdeg.weather.read(weather_file, "psm")

kwargs = {
    "tau_0": 115,  # us, carrier lifetime in non-degraded states, e.g. LETID/LID states A or C
    "tau_deg": 55,  # us, carrier lifetime in fully-degraded state, e.g. LETID/LID state B
    "wafer_thickness": 180,  # um
    "s_rear": 46,  # cm/s
    "cell_area": 243,  # cm^2
    "na_0": 100,
    "nb_0": 0,
    "nc_0": 0,
    "mechanism_params": "D037",
}

pvdeg.letid.calc_letid_outdoors(weather_df=WEATHER, meta=META, **kwargs)

/projects/pvdeg/mspringe/envs/pvdeg_313/lib/python3.13/site-packages/scipy/optimize/_chandrupatla.py:437: RuntimeWarning: invalid value encountered in divide
  C = A / (A + B)

	Temperature	Injection	NA	NB	NC	tau	Jsc	Voc	Isc	FF	Pmp	Pmp_norm
time
1999-01-01 00:30:00-07:00	0.0	NaN	100.000000	0.000000e+00	0.000000e+00	115.000000	41.590997	0.666327	10.106612	0.840987	5.663467	1.000000
1999-01-01 01:30:00-07:00	0.0	NaN	100.000000	1.702422e-15	0.000000e+00	115.000000	41.590997	0.666327	10.106612	0.840987	5.663467	1.000000
1999-01-01 02:30:00-07:00	0.0	NaN	100.000000	3.404843e-15	5.403329e-36	115.000000	41.590997	0.666327	10.106612	0.840987	5.663467	1.000000
1999-01-01 03:30:00-07:00	0.0	NaN	100.000000	5.107265e-15	1.620999e-35	115.000000	41.590997	0.666327	10.106612	0.840987	5.663467	1.000000
1999-01-01 04:30:00-07:00	0.0	NaN	100.000000	6.809686e-15	3.241997e-35	115.000000	41.590997	0.666327	10.106612	0.840987	5.663467	1.000000
...	...	...	...	...	...	...	...	...	...	...	...	...
1999-12-31 19:30:00-07:00	0.0	NaN	27.833654	6.778463e+01	4.381716e+00	66.112142	41.333851	0.654974	10.044126	0.838966	5.519257	0.974537
1999-12-31 20:30:00-07:00	0.0	NaN	27.833654	6.778463e+01	4.381716e+00	66.112142	41.333851	0.654974	10.044126	0.838966	5.519257	0.974537
1999-12-31 21:30:00-07:00	0.0	NaN	27.833654	6.778463e+01	4.381716e+00	66.112142	41.333851	0.654974	10.044126	0.838966	5.519257	0.974537
1999-12-31 22:30:00-07:00	0.0	NaN	27.833654	6.778463e+01	4.381716e+00	66.112142	41.333851	0.654974	10.044126	0.838966	5.519257	0.974537
1999-12-31 23:30:00-07:00	0.0	NaN	27.833654	6.778463e+01	4.381716e+00	66.112142	41.333851	0.654974	10.044126	0.838966	5.519257	0.974537

8760 rows × 12 columns

Start distributed compute cluster - DASK#

local = {
    "manager": "local",
    "n_workers": 1,
    "threads_per_worker": 8,  # Number of CPUs
}

# kestrel = {
#     "manager": "slurm",
#     "n_jobs": 1,  # Number of nodes used for parallel processing
#     "cores": 104,
#     "memory": "230GB",
#     "account": "pvdeg",
#     "queue": "debug",
#     "walltime": "1:00:00",
#     "processes": 104,
#     "job_extra_directives": ["-o ./logs/slurm-%j.out"],
# }

kestrel = {
    "manager": "slurm",
    "n_jobs": 1,  # Number of nodes used for parallel processing
    "cores": 100,
    "memory": "230GB",
    "account": "pvdeg",
    "queue": "debug",
    "walltime": "1:00:00",
    "interface": "hsn0",
    "processes": 100,  # workers per node; 104 single-thread procs overwhelm startup
    "local_directory": f"/scratch/{getpass.getuser()}/",  # node-local scratch for fast worker startup
    "death_timeout": 600,  # secs a worker waits to reach the scheduler before exiting
    "job_extra_directives": ["-o ./logs/slurm-%j.out"],
}

pvdeg.geospatial.start_dask(hpc=kestrel)

Dashboard: http://10.150.16.29:8787/status

Client

Client-63d2d72b-8479-11f1-a388-0040a6880611

Connection method: Cluster object	Cluster type: dask_jobqueue.SLURMCluster
Dashboard: http://10.150.16.29:8787/status

Cluster Info

SLURMCluster

c64ea8af

Dashboard: http://10.150.16.29:8787/status	Workers: 0
Total threads: 0	Total memory: 0 B

Scheduler Info

Scheduler

Scheduler-0a9d9304-c29e-40d5-85cc-b1d316c26936

Comm: tcp://10.150.16.29:45161	Workers: 0
Dashboard: http://10.150.16.29:8787/status	Total threads: 0
Started: Just now	Total memory: 0 B

Workers

# Get weather data
weather_db = "NSRDB"

weather_arg = {
    "satellite": "Americas",
    "names": 2022,
    "NREL_HPC": True,
    "attributes": [
        "air_temperature",
        "wind_speed",
        "dhi",
        "ghi",
        "dni",
        "relative_humidity",
    ],
}

weather_ds, meta_df = pvdeg.weather.get(weather_db, geospatial=True, **weather_arg)

# Define geographical region
meta_SW = meta_df[meta_df["State"].isin(["Colorado", "New Mexico", "Utah", "Arizona"])]
meta_SW_sub, gids_SW_sub = pvdeg.utilities.gid_downsampling(meta_SW, 6)

# Chunk along gid so geospatial.analysis (weather_ds.map_blocks) emits one task per
# location instead of a single serial block. time stays one chunk (-1) because each
# location needs its full timeseries.
weather_SW_sub = weather_ds.sel(gid=meta_SW_sub.index).chunk({"time": -1, "gid": 1})

/kfs2/projects/pvdeg/mspringe/repos/PVDegradationTools/pvdeg/weather.py:642: UserWarning: The specified chunks separate the stored chunks along dimension "phony_dim_1" starting at index 500. This could degrade performance. Instead, consider rechunking after loading.
  ds = xr.open_dataset(

weather_SW_sub

<xarray.Dataset> Size: 409MB
Dimensions:            (time: 17520, gid: 486)
Coordinates:
  * time               (time) datetime64[us] 140kB 2022-01-01 ... 2022-12-31T...
  * gid                (gid) int64 4kB 277461 277473 277485 ... 554496 554508
Data variables:
    temp_air           (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
    wind_speed         (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
    dhi                (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
    ghi                (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
    dni                (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
    relative_humidity  (time, gid) float64 68MB dask.array<chunksize=(17520, 1), meta=np.ndarray>
Attributes:
    full_version_record:   {"rex": "0.2.80", "pandas": "2.0.0", "numpy": "1.2...
    package:               rex
    version:               4.0.0
    kestrel_nsrdb_fnames:  ['/datasets/NSRDB/current/nsrdb_2022.h5']

meta_df

	latitude	longitude	altitude	tz	Country	State	County	wind_height
0	-15.950000	-179.979996	0	12	NaN	NaN	NaN	2
1	-15.990000	-179.979996	0	12	NaN	NaN	NaN	2
2	-16.030001	-179.979996	0	12	NaN	NaN	NaN	2
3	-16.070000	-179.979996	0	12	NaN	NaN	NaN	2
4	-16.110001	-179.979996	0	12	NaN	NaN	NaN	2
...	...	...	...	...	...	...	...	...
2018262	16.010000	-22.500000	0	-1	NaN	NaN	NaN	2
2018263	15.970000	-22.500000	0	-1	NaN	NaN	NaN	2
2018264	15.930000	-22.500000	0	-1	NaN	NaN	NaN	2
2018265	15.890000	-22.500000	0	-1	NaN	NaN	NaN	2
2018266	15.850000	-22.500000	0	-1	NaN	NaN	NaN	2

2018267 rows × 8 columns

# Define desired analysis
geo = {
    "func": pvdeg.letid.calc_letid_outdoors,
    "weather_ds": weather_SW_sub,
    "meta_df": meta_SW_sub,
    "tau_0": 115,  # us, carrier lifetime in non-degraded states, e.g. LETID/LID states A or C
    "tau_deg": 55,  # us, carrier lifetime in fully-degraded state, e.g. LETID/LID state B
    "wafer_thickness": 180,  # um
    "s_rear": 46,  # cm/s
    "cell_area": 243,  # cm^2
    "na_0": 100,
    "nb_0": 0,
    "nc_0": 0,
    "mechanism_params": "D037",
}

letid_res = pvdeg.geospatial.analysis(**geo)

/projects/pvdeg/mspringe/envs/pvdeg_313/lib/python3.13/site-packages/distributed/client.py:3429: UserWarning: Sending large graph of size 11.15 MiB.
This may cause some slowdown.
Consider loading the data with Dask directly
 or using futures or delayed objects to embed the data into the graph without repetition.
See also https://docs.dask.org/en/stable/best-practices.html#load-data-with-dask for more information.
  warnings.warn(
/kfs2/projects/pvdeg/mspringe/repos/PVDegradationTools/pvdeg/utilities.py:1567: FutureWarning: dropping variables using `drop` is deprecated; use drop_vars.
  stacked = ds_gids.drop(["gid"])

import datetime

ims = []
for n in range(1, 13):
    for i, np_t in enumerate(letid_res.time):
        t = pd.Timestamp(np_t.values).time()
        d = pd.Timestamp(np_t.values).day
        m = pd.Timestamp(np_t.values).month
        if m == n:
            if d == 15:
                if t == datetime.time(12):
                    fig, ax = pvdeg.geospatial.plot_USA(
                        letid_res["Pmp_norm"].sel(time=np_t),
                        cmap="viridis",
                        vmin=0.95,
                        vmax=1,
                        title=f"Normalized Power  - 2022-{m}-{d} 12:00",
                        cb_title="Normalized Power",
                    )
                    plt.savefig(f"../../TEMP/RH_animation_{n}.png", dpi=600)

# import imageio
# ims = [imageio.imread(f'../../TEMP/RH_animation_{n}.png') for n in range(1, 13)]
# imageio.mimwrite('../../TEMP/RH_animation.gif', ims, format='GIF', duration=1000, loop=10)

../_images/2c1b604d9c7c19831a1fd3db8913b4957679def993a07f9ec6d139c1a6e5b430.png

../_images/6cff8c06ef940a3fb8746c16d4b1ebc6f9e83e9f9f5b9e57c1a955ef3b07d6a3.png

../_images/3f128f2f1dc28177f6bafbff9f914ef4e12aefe49388dc84ae3a472964942d9a.png

../_images/4b41554e004ba3fae2432ea11dafe0af080c241cd6e9fe4af569bc09c969db5f.png

../_images/b986aed3f59904dc166f1bec37cc926407cbb6435d0fb27e6494bb6cfb9ba4fb.png

../_images/99eed62db3e7bef7a0a007c58846d4cca230328eb6df5c7078e17ab4974d13bf.png

../_images/4e8b79c9104b1d72bd883babc73f306bd794cae0975bf890229d7a8a625f0b91.png

../_images/3a01c32cedf45a35bf227aa08d66bc237473770d73da582b9a0a5c27655ae4d7.png

../_images/fa1368344459ca1568a0b30c1506b377f6db4046b5d3684a7e9168e7a9bb19fb.png

../_images/3dc39df3f06a31dc28d46d16f79daed1932df2a56b3f7cfcc38abcd87361cd81.png

../_images/448c2ca16be336f75319595a7ed28f6afa67b6bf8932d059a6dbcf8e7a982f74.png

../_images/e0d2c26f767ef997cb5b681eb528b354f55d1de66e5cf5e441ece6ebb4e003d3.png

import datetime

ims = []
dates = []
subarctics = []
coldsemiarids = []
hotdeserts = []

for n in range(1, 13):
    for i, np_t in enumerate(letid_res.time):
        t = pd.Timestamp(np_t.values).time()
        d = pd.Timestamp(np_t.values).day
        m = pd.Timestamp(np_t.values).month
        if m == n:
            if d == 15:
                if t == datetime.time(12):
                    dates.append(np_t.values)

                    # subartic: near Crested Butte CO
                    # cold semi-arid: near Springfield CO
                    # hot desert: near Yuma AZ

                    subarctic = letid_res.sel(
                        time=np_t, latitude=39.01, longitude=-107.1
                    )
                    subarctics.append(subarctic["Pmp_norm"])

                    coldsemiarid = letid_res.sel(
                        time=np_t, latitude=37.57, longitude=-102.3
                    )
                    coldsemiarids.append(coldsemiarid["Pmp_norm"])

                    hotdesert = letid_res.sel(
                        time=np_t, latitude=32.77, longitude=-114.3
                    )
                    hotdeserts.append(hotdesert["Pmp_norm"])

                    fig, ax = plt.subplots()
                    ax.plot(
                        dates,
                        subarctics,
                        marker="o",
                        c="C0",
                        label="Central CO - Subarctic Dfc",
                    )
                    ax.plot(
                        dates,
                        coldsemiarids,
                        marker="o",
                        c="C1",
                        label="Southeast CO - Cold Semi-Arid BSk",
                    )
                    ax.plot(
                        dates,
                        hotdeserts,
                        marker="o",
                        c="C2",
                        label="Southwest AZ - Hot Desert BWh",
                    )

                    ax.legend(loc="upper right")

                    ax.set_xlim([datetime.date(2022, 1, 1), datetime.date(2023, 1, 1)])

                    ax.set_ylim([0.945, 1.005])
                    ax.set_ylabel("Normalized Power")

                    plt.savefig(f"../../TEMP/LETID_plot_animation_{n}.png", dpi=600)

../_images/50a9c6d92a8b59ffac2fadd09fc237d891ea4f3ba086de367e0e081af3bcefef.png

../_images/af1678069ab1046c385db3f21bc8aaef6f3dfad2b20198b1f7f81345b2835d02.png

../_images/3fa7c2429d257e23fb4a167a7ba02f64716a841b5d3a6bb1fb8cf7e730ed4598.png

../_images/bf254407245b435d64a5faa220a8ecd68b41fe9599faeb022732991a4ef0eff2.png

../_images/3eb03149cdab8952feca0be2ceae552f6cc462de885bc38600d93e24f8d53d20.png

../_images/c323593baaa48f9569e3d6abcd041cccb02cdda52b6292a72516daff5dad2177.png

../_images/b3681a2975f31cbdf6ed67ee865210bc76fc9d091de9f95efe2eafc33a2ada40.png

../_images/d13c72880e8fe782a24d0adee48e7caec8d58de6043c464bceb3b59610b53da2.png

../_images/2e4a16448a4d30bf47073627beee735abe7ece415510daa93e73f7f48fd95cf1.png

../_images/aa53c518418a149ac7fe7669ea31f25812450a476635db1555675773712b7ff3.png

../_images/a9b41e49a78bf8ee8a74aa35ad85db536ca389fa5048cbd27ceb4e27e4fcd13d.png

../_images/879a8c32a895387b905d0b72852bdd4d9d801a8c7d3bf231019099bb3cb601ab.png

# import imageio

# ims = [imageio.imread(f"../../TEMP/LETID_plot_animation_{n}.png") for n in range(1, 13)]
# imageio.mimwrite(
#     "../../TEMP/LETID_plot_animation.gif", ims, format="GIF", duration=1000, loop=10
# )

LETID Outdoor Geospatial Demo (HPC)

Contents

LETID Outdoor Geospatial Demo (HPC)#

Single location example#

Start distributed compute cluster - DASK#

Client

Cluster Info

SLURMCluster

Scheduler Info

Scheduler

Workers