Skip to content

Test & improve perfomance in .tools.add_year #494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
khaeru wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

Test & improve perfomance in .tools.add_year #494

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9e92ffb
Add rudimentary tests of .add_year.interpolate_1d
khaeru Jul 18, 2021
373c7af
Type hint .add_year.interpolate_1d
khaeru Jul 18, 2021
ce88d10
Return early in interpolate_1d() for empty input
khaeru Jul 18, 2021
37bddd7
Clarify documentation of interpolate_1d()
khaeru Jul 18, 2021
0351efe
Add .add_year.i1d_genno, a partial replacement for interpolate_1d
khaeru Jul 18, 2021
913be5a
Sort imports to satisfy isort
glatterf42 Mar 3, 2023
91cdca4
Fix some typos in add_year/README
glatterf42 Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Return early in interpolate_1d() for empty input 
This dedents most of the code one level.
  • Loading branch information
@khaeru @glatterf42
khaeru authored and glatterf42 committed Nov 21, 2025
commit ce88d1069144caf3ab856af4db14271439e42fb4
193 changes: 95 additions & 98 deletions message_ix/tools/add_year/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -592,124 +592,121 @@ def interpolate_1d( # noqa: C901
Allow extrapolation when a new year is outside the parameter years.
extrapol_neg : bool
Allow negative values obtained by extrapolation.

- Appears to have no effect when extrapolating *before* `horizon`.
bound_extend : bool
Allow extrapolation of bounds for new years
"""
horizon_new = sorted(horizon + yrs_new)
idx = [x for x in df.columns if x not in [year_col, value_col]]
if not df.empty:
df2 = df.pivot_table(index=idx, columns=year_col, values=value_col)
df2_int_column_list = [int(column) for column in df2.columns]

# To sort the new years smaller than the first year for
# extrapolation (e.g. 2025 values are calculated first; then
# values of 2015 based on 2020 and 2025)
year_before = sorted(
[x for x in yrs_new if x < min(df2_int_column_list)], reverse=True
)
if year_before and extrapolate:
for y in year_before:
yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y)))

for yr in yrs_new:
extrapol = True if yr > max(horizon) else extrapolate

# a) If this new year greater than modeled years, do extrapolation
if yr > max(df2_int_column_list) and extrapol:
if yr == horizon_new[horizon_new.index(max(df2_int_column_list)) + 1]:
year_pre = max([x for x in df2_int_column_list if x < yr])

if len([x for x in df2_int_column_list if x < yr]) >= 2:
year_pp = max([x for x in df2_int_column_list if x < year_pre])
df2[yr] = intpol(
df2[year_pre], df2[year_pp], year_pre, year_pp, yr
)

if bound_extend:
df2[yr] = df2[yr].fillna(df2[year_pre])

df2[yr][np.isinf(df2[year_pre])] = df2[year_pre]
if (
not df2[yr].loc[(df2[yr] < 0) & (df2[year_pre] >= 0)].empty
and extrapol_neg
):
df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), year_pre]
* extrapol_neg
)
else:
df2[yr] = df2[year_pre]
if df.empty:
log.warning("The submitted dataframe is empty, so returned empty results")
return df

# b) If the new year is smaller than modeled years, extrapolate
elif yr < min(df2_int_column_list) and extrapol:
year_next = min([x for x in df2_int_column_list if x > yr])
df2 = df.pivot_table(index=idx, columns=year_col, values=value_col)
df2_int_column_list = [int(column) for column in df2.columns]

# To make sure the new year is not two steps smaller
cond = year_next == horizon_new[horizon_new.index(yr) + 1]
# To sort the new years smaller than the first year for extrapolation (e.g. 2025
# values are calculated first; then values of 2015 based on 2020 and 2025)
year_before = sorted(
[x for x in yrs_new if x < min(df2_int_column_list)], reverse=True
)
if year_before and extrapolate:
for y in year_before:
yrs_new.insert(len(yrs_new), yrs_new.pop(yrs_new.index(y)))

if len([x for x in df2_int_column_list if x > yr]) >= 2 and cond:
year_nn = min([x for x in df2_int_column_list if x > year_next])
df2[yr] = intpol(
df2[year_next], df2[year_nn], year_next, year_nn, yr
)
df2[yr][np.isinf(df2[year_next])] = df2[year_next]
for yr in yrs_new:
extrapol = True if yr > max(horizon) else extrapolate

# a) If this new year greater than modeled years, do extrapolation
if yr > max(df2_int_column_list) and extrapol:
if yr == horizon_new[horizon_new.index(max(df2_int_column_list)) + 1]:
year_pre = max([x for x in df2_int_column_list if x < yr])

if len([x for x in df2_int_column_list if x < yr]) >= 2:
year_pp = max([x for x in df2_int_column_list if x < year_pre])
df2[yr] = intpol(df2[year_pre], df2[year_pp], year_pre, year_pp, yr)

if bound_extend:
df2[yr] = df2[yr].fillna(df2[year_pre])

df2[yr][np.isinf(df2[year_pre])] = df2[year_pre]
if (
not df2[yr].loc[(df2[yr] < 0) & (df2[year_next] >= 0)].empty
not df2[yr].loc[(df2[yr] < 0) & (df2[year_pre] >= 0)].empty
and extrapol_neg
):
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), year_next]
df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_pre] >= 0), year_pre]
* extrapol_neg
)
else:
df2[yr] = df2[year_pre]

elif bound_extend and cond:
df2[yr] = df2[year_next]
# b) If the new year is smaller than modeled years, extrapolate
elif yr < min(df2_int_column_list) and extrapol:
year_next = min([x for x in df2_int_column_list if x > yr])

# c) Otherise, do intrapolation
elif yr > min(df2_int_column_list) and yr < max(df2_int_column_list):
year_pre = max([x for x in df2_int_column_list if x < yr])
year_next = min([x for x in df2_int_column_list if x > yr])
df2[yr] = intpol(df2[year_pre], df2[year_next], year_pre, year_next, yr)
# To make sure the new year is not two steps smaller
cond = year_next == horizon_new[horizon_new.index(yr) + 1]

if len([x for x in df2_int_column_list if x > yr]) >= 2 and cond:
year_nn = min([x for x in df2_int_column_list if x > year_next])
df2[yr] = intpol(df2[year_next], df2[year_nn], year_next, year_nn, yr)
df2[yr][np.isinf(df2[year_next])] = df2[year_next]
if (
not df2[yr].loc[(df2[yr] < 0) & (df2[year_next] >= 0)].empty
and extrapol_neg
):
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), year_next]
* extrapol_neg
)

# Extrapolate for new years if the value exists for the
# previous year but not for the next years
# TODO: here is the place that should be changed if the
# new year should go to the time step before the existing one
if [x for x in df2_int_column_list if x > year_next]:
year_nn = min([x for x in df2_int_column_list if x > year_next])
df2[yr] = df2[yr].fillna(
intpol(df2[year_next], df2[year_nn], year_next, year_nn, yr)
elif bound_extend and cond:
df2[yr] = df2[year_next]

# c) Otherise, do intrapolation
elif yr > min(df2_int_column_list) and yr < max(df2_int_column_list):
year_pre = max([x for x in df2_int_column_list if x < yr])
year_next = min([x for x in df2_int_column_list if x > yr])
df2[yr] = intpol(df2[year_pre], df2[year_next], year_pre, year_next, yr)

# Extrapolate for new years if the value exists for the previous year but
# not for the next years
# TODO: here is the place that should be changed if the new year should go
# to the time step before the existing one
if [x for x in df2_int_column_list if x > year_next]:
year_nn = min([x for x in df2_int_column_list if x > year_next])
df2[yr] = df2[yr].fillna(
intpol(df2[year_next], df2[year_nn], year_next, year_nn, yr)
)
if (
not df2[yr].loc[(df2[yr] < 0) & (df2[year_next] >= 0)].empty
and extrapol_neg
):
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), year_next]
* extrapol_neg
)
if (
not df2[yr].loc[(df2[yr] < 0) & (df2[year_next] >= 0)].empty
and extrapol_neg
):
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), yr] = (
df2.loc[(df2[yr] < 0) & (df2[year_next] >= 0), year_next]
* extrapol_neg
)

if bound_extend:
df2[yr] = df2[yr].fillna(df2[year_pre])
mask = np.isinf(df2[year_pre])
df2.loc[mask, str(yr)] = df2[year_pre]

df2 = (
pd.melt(
df2.reset_index(),
id_vars=idx,
value_vars=[x for x in df2.columns if x not in idx],
var_name=year_col,
value_name=value_col,
)
.dropna(subset=[value_col])
.reset_index(drop=True)
if bound_extend:
df2[yr] = df2[yr].fillna(df2[year_pre])
df2[yr][np.isinf(df2[year_pre])] = df2[year_pre]

return (
pd.melt(
df2.reset_index(),
id_vars=idx,
value_vars=[x for x in df2.columns if x not in idx],
var_name=year_col,
value_name=value_col,
)
df2 = df2.sort_values(idx).reset_index(drop=True)
else:
log.warning("The submitted dataframe is empty, so returned empty results")
df2 = df
return df2
.dropna(subset=[value_col])
.reset_index(drop=True)
.sort_values(idx)
.reset_index(drop=True)
)


# %% VI.B) Interpolating parameters with two dimensions related to time
Expand Down