394 multi yoy - allows multiple slope calculations across multi-year periods

.github/workflows/nbval.yaml

@@ -29,7 +29,7 @@ jobs:
     - name: Run notebook and check output
       run: |
         # --sanitize-with:  pre-process text to remove irrelevant differences (e.g. warning filepaths)
-        pytest --nbval docs/${{ matrix.notebook-file }} --sanitize-with docs/nbval_sanitization_rules.cfg
+        pytest --nbval --nbval-sanitize-with docs/nbval_sanitization_rules.cfg docs/${{ matrix.notebook-file }}
     - name: Run notebooks again, save files
       run: |
         pip install nbconvert[webpdf]

docs/TrendAnalysis_example.ipynb

@@ -62160,7 +62160,7 @@
     "# Visualize the results\n",
     "ta_new_filter.plot_degradation_summary('sensor', summary_title='Sensor-based degradation results',\n",
     "                                   scatter_ymin=0.5, scatter_ymax=1.1,\n",
-    "                                   hist_xmin=-30, hist_xmax=45);\n",
+    "                                   hist_xmin=-30, hist_xmax=45)\n",
     "plt.show()"
    ]
   },
@@ -62247,7 +62247,7 @@
     "# Visualize the results\n",
     "ta_stuck_filter.plot_degradation_summary('sensor', summary_title='Sensor-based degradation results',\n",
     "                                         scatter_ymin=0.5, scatter_ymax=1.1,\n",
-    "                                         hist_xmin=-30, hist_xmax=45);\n",
+    "                                         hist_xmin=-30, hist_xmax=45)\n",
     "plt.show()"
    ]
   },

docs/TrendAnalysis_example_NSRDB.ipynb

@@ -158,7 +158,7 @@
     "ax.plot(df.index, df.soiling, 'o', alpha=0.01)\n",
     "#ax.set_ylim(0,1500)\n",
     "fig.autofmt_xdate()\n",
-    "ax.set_ylabel('soiling signal');\n",
+    "ax.set_ylabel('soiling signal')\n",
     "df['power'] = df['power_ac'] * df['soiling']\n",
     "\n",
     "plt.show()"

docs/notebook_requirements.txt

@@ -15,7 +15,7 @@ ipython==8.26.0
 ipython-genutils==0.2.0
 ipywidgets==8.1.3
 jedi==0.19.1
-Jinja2==3.1.5
+Jinja2==3.1.6
 jsonschema==4.23.0
 jupyter==1.0.0
 jupyter-client==8.6.2
@@ -29,7 +29,7 @@ nbclient==0.10.0
 nbconvert==7.16.4
 nbformat==5.10.4
 nest-asyncio==1.6.0
-notebook==7.2.1
+notebook==7.2.2
 numexpr==2.10.1
 pandocfilters==1.5.1
 parso==0.8.4
@@ -48,7 +48,7 @@ soupsieve==2.6
 terminado==0.18.1
 testpath==0.6.0
 tinycss2==1.3.0
-tornado==6.4.2
+tornado==6.5.1
 traitlets==5.14.3
 wcwidth==0.2.13
 webencodings==0.5.1

docs/sphinx/source/changelog.rst

@@ -1,5 +1,7 @@
 RdTools Change Log
 ==================
+.. include:: changelog/pending.rst
+.. include:: changelog/v3.0.1.rst
 .. include:: changelog/v3.0.0.rst
 .. include:: changelog/v2.1.8.rst
 .. include:: changelog/v2.1.7.rst

docs/sphinx/source/changelog/pending.rst

@@ -0,0 +1,24 @@
+*************************
+v3.0.x (X, X, 2025)
+*************************
+
+Enhancements
+------------
+* :py:func:`~rdtools.degradation.degradation_year_on_year` has new parameter ``label=`` 
+  to return the calc_info['YoY_values'] as either right labeled (default), left or
+  center labeled. (:issue:`459`)
+* :py:func:`~rdtools.plotting.degradation_timeseries_plot` has new parameter ``label=`` 
+  to allow the timeseries plot to have right labeling (default), center or left labeling. 
+  (:issue:`455`)
+* :py:func:`~rdtools.degradation.degradation_year_on_year` has new parameter ``multi_yoy`` 
+  (default False) to trigger multiple YoY degradation calculations similar to Hugo Quest et
+  al 2023. In this mode, instead of a series of 1-year duration slopes, 2-year, 3-year etc
+  slopes are also included.  calc_info['YoY_values'] returns a non-monotonic index
+  in this mode due to multiple overlapping annual slopes.  (:issue:`394`)  
+
+
+
+Contributors
+------------
+* Chris Deline (:ghuser:`cdeline`)
+

docs/sphinx/source/changelog/v3.0.1.rst

@@ -0,0 +1,11 @@
+*************************
+v3.0.1 (August 21, 2025)
+*************************
+
+Requirements
+------------
+* Updated Jinja2==3.1.6 in ``notebook_requirements.txt`` (:pull:`465`)
+* Updated tornado==6.5.1 in ``notebook_requirements.txt`` (:pull:`465`)
+* Updated requests==2.32.4 in ``requirements.txt`` (:pull:`465`)
+* Updated urllib3==2.5.0 in ``requirements.txt`` (:pull:`465`)
+* Removed constraint that scipy<1.16.0 (:pull:`465`)

docs/system_availability_example.ipynb

@@ -618,7 +618,7 @@
     }
    ],
    "source": [
-    "aa2.plot();"
+    "plt.show(aa2.plot())"
    ]
   },
   {

rdtools/degradation.py

@@ -179,7 +179,8 @@ def degradation_classical_decomposition(energy_normalized,
 
 def degradation_year_on_year(energy_normalized, recenter=True,
                              exceedance_prob=95, confidence_level=68.2,
-                             uncertainty_method='simple', block_length=30):
+                             uncertainty_method='simple', block_length=30,
+                             label='right', multi_yoy=False):
     '''
     Estimate the trend of a timeseries using the year-on-year decomposition
     approach and calculate a Monte Carlo-derived confidence interval of slope.
@@ -208,6 +209,13 @@ def degradation_year_on_year(energy_normalized, recenter=True,
         If `uncertainty_method` is 'circular_block', `block_length`
         determines the length of the blocks used in the circular block bootstrapping
         in number of days. Must be shorter than a third of the time series.
+    label    : {'right', 'center', 'left'}, default 'right'
+        Which Year-on-Year slope edge to label.
+    multi_yoy : bool, default False
+        Whether to return the standard Year-on-Year slopes where each slope
+        is calculated over points separated by 365 days (default) or
+        multi_year-on-year where points can be separated by N * 365 days
+        where N is an integer from 1 to the length of the dataset in years.
 
     Returns
     -------
@@ -218,7 +226,8 @@ def degradation_year_on_year(energy_normalized, recenter=True,
         degradation rate estimate
     calc_info : dict
 
-        * `YoY_values` - pandas series of right-labeled year on year slopes
+        * `YoY_values` - pandas series of year on year slopes, either right
+            or center labeled, depending on the `label` parameter.
         * `renormalizing_factor` - float of value used to recenter data
         * `exceedance_level` - the degradation rate that was outperformed with
           probability of `exceedance_prob`
@@ -233,6 +242,12 @@ def degradation_year_on_year(energy_normalized, recenter=True,
     energy_normalized.name = 'energy'
     energy_normalized.index.name = 'dt'
 
+    if label not in {None, "right", "left", "center"}:
+        raise ValueError(f"Unsupported value {label} for `label`."
+                         " Must be 'right', 'left' or 'center'.")
+    if label is None:
+        label = "right"
+
     # Detect less than 2 years of data. This is complicated by two things:
     #   - leap days muddle the precise meaning of "two years of data".
     #   - can't just check the number of days between the first and last
@@ -269,37 +284,74 @@ def degradation_year_on_year(energy_normalized, recenter=True,
     energy_normalized = energy_normalized.reset_index()
     energy_normalized['energy'] = energy_normalized['energy'] / renorm
 
-    energy_normalized['dt_shifted'] = energy_normalized.dt + pd.DateOffset(years=1)
-
-    # Merge with what happened one year ago, use tolerance of 8 days to allow
-    # for weekly aggregated data
-    df = pd.merge_asof(energy_normalized[['dt', 'energy']],
-                       energy_normalized.sort_values('dt_shifted'),
-                       left_on='dt', right_on='dt_shifted',
-                       suffixes=['', '_right'],
-                       tolerance=pd.Timedelta('8D')
-                       )
-
-    df['time_diff_years'] = (df.dt - df.dt_right) / pd.Timedelta('365d')
-    df['yoy'] = 100.0 * (df.energy - df.energy_right) / (df.time_diff_years)
-    df.index = df.dt
+    # dataframe container for combined year-over-year changes
+    df = pd.DataFrame()
+    if multi_yoy:
+        year_range = range(1, int((energy_normalized.iloc[-1]['dt'] -
+                                   energy_normalized.iloc[0]['dt']).days/365)+1)
+    else:
+        year_range = [1]
+    for y in year_range:
+        energy_normalized['dt_shifted'] = energy_normalized.dt + pd.DateOffset(years=y)
+        # Merge with what happened one year ago, use tolerance of 8 days to allow
+        # for weekly aggregated data
+        df_temp = pd.merge_asof(energy_normalized[['dt', 'energy']],
+                                energy_normalized.sort_values('dt_shifted'),
+                                left_on='dt', right_on='dt_shifted',
+                                suffixes=['', '_left'],
+                                tolerance=pd.Timedelta('8D')
+                                )
+        df = pd.concat([df, df_temp], ignore_index=True)
+
+    df['time_diff_years'] = (df.dt - df.dt_left) / pd.Timedelta('365d')
+    df['yoy'] = 100.0 * (df.energy - df.energy_left) / (df.time_diff_years)
+    # df.index = df.dt
 
     yoy_result = df.yoy.dropna()
 
-    df_right = df.set_index(df.dt_right).drop_duplicates('dt_right')
-    df['usage_of_points'] = df.yoy.notnull().astype(int).add(
-                df_right.yoy.notnull().astype(int), fill_value=0)
-
     if not len(yoy_result):
         raise ValueError('no year-over-year aggregated data pairs found')
 
     Rd_pct = yoy_result.median()
 
+    YoY_times = df.dropna(subset=['yoy'], inplace=False).copy()
+
+    # calculate usage of points.
+    df_left = YoY_times.set_index(YoY_times.dt_left)  # .drop_duplicates('dt_left')
+    df_right = YoY_times.set_index(YoY_times.dt)  # .drop_duplicates('dt')
+    usage_of_points = df_right.yoy.notnull().astype(int).add(
+                df_left.yoy.notnull().astype(int),
+                fill_value=0).groupby(level=0).sum()
+    usage_of_points.name = 'usage_of_points'
+
+    if pd.__version__ < '2.0.0':
+        # For old Pandas versions < 2.0.0, time columns cannot be averaged
+        # with each other, so we use a custom function to calculate center label
+        YoY_times['dt_center'] = _avg_timestamp_old_Pandas(YoY_times['dt'], YoY_times['dt_left'])
+    else:
+        YoY_times['dt_center'] = pd.to_datetime(YoY_times[['dt', 'dt_left']].mean(axis=1))
+
+    YoY_times = YoY_times[['dt', 'dt_center', 'dt_left']]
+    YoY_times = YoY_times.rename(columns={'dt': 'dt_right'})
+
+    YoY_times.set_index(YoY_times[f'dt_{label}'], inplace=True)
+    # YoY_times = YoY_times.rename_axis(None, axis=1)
+    YoY_times.index.name = None
+    yoy_result.index = YoY_times[f'dt_{label}']
+    yoy_result.index.name = None
+
+    # the following is throwing a futurewarning if infer_objects() isn't included here.
+    # see https://github.com/pandas-dev/pandas/issues/57734
+    energy_normalized = energy_normalized.merge(usage_of_points, how='left', left_on='dt',
+                                                right_index=True, left_index=False
+                                                ).infer_objects().fillna(0.0)
+
     if uncertainty_method == 'simple':  # If we need the full results
         calc_info = {
             'YoY_values': yoy_result,
             'renormalizing_factor': renorm,
-            'usage_of_points': df['usage_of_points']
+            'usage_of_points': energy_normalized.set_index('dt')['usage_of_points'],
+            'YoY_times': YoY_times[['dt_right', 'dt_center', 'dt_left']]
         }
 
         # bootstrap to determine 68% CI and exceedance probability
@@ -345,17 +397,84 @@ def degradation_year_on_year(energy_normalized, recenter=True,
 
         # Save calculation information
         calc_info = {
+            'YoY_values': yoy_result,
             'renormalizing_factor': renorm,
             'exceedance_level': exceedance_level,
-            'usage_of_points': df['usage_of_points'],
+            'usage_of_points': energy_normalized.set_index('dt')['usage_of_points'],
+            'YoY_times': YoY_times[['dt_right', 'dt_center', 'dt_left']],
             'bootstrap_rates': bootstrap_rates}
 
         return (Rd_pct, Rd_CI, calc_info)
 
     else:  # If we do not need confidence intervals and exceedance level
+        """ # TODO: return tuple just like all other cases. Issue: test_bootstrap_module
+        return (Rd_pct, None, {
+            'YoY_values': yoy_result,
+            'usage_of_points': energy_normalized.set_index('dt')['usage_of_points'],
+            'YoY_times': YoY_times[['dt_right', 'dt_center', 'dt_left']]}
+        })
+        """
         return Rd_pct
 
 
+def _avg_timestamp_old_Pandas(dt, dt_left):
+    '''
+    For old Pandas versions < 2.0.0, time columns cannot be averaged
+    together.  From https://stackoverflow.com/questions/57812300/
+    python-pandas-to-calculate-mean-of-datetime-of-multiple-columns
+
+    Parameters
+    ----------
+    dt : pandas.Series
+        First series with datetime values
+    dt_left : pandas.Series
+        Second series with datetime values.
+
+    Returns
+    -------
+    pandas.Series
+        Series with the average timestamp of df1 and df2.
+    '''
+    import calendar
+
+    # allow for numeric index
+    try:
+        temp_df = pd.DataFrame({'dt' : dt.dt.tz_localize(None),
+                                'dt_left' : dt_left.dt.tz_localize(None)
+                                }).tz_localize(None)
+    except TypeError:  # in case numeric index passed
+        temp_df = pd.DataFrame({'dt' : dt.dt.tz_localize(None),
+                                'dt_left' : dt_left.dt.tz_localize(None)
+                                })
+
+    # conversion from dates to seconds since epoch (unix time)
+    def to_unix(s):
+        if type(s) is pd.Timestamp:
+            return calendar.timegm(s.timetuple())
+        else:
+            return pd.NaT
+
+    # sum the seconds since epoch, calculate average, and convert back to readable date
+    averages = []
+    for index, row in temp_df.iterrows():
+        unix = [to_unix(i) for i in row]
+        # unix = [pd.Timestamp(i).timestamp() for i in row]
+        try:
+            average = sum(unix) / len(unix)
+            # averages.append(datetime.datetime.utcfromtimestamp(average).strftime('%Y-%m-%d'))
+            averages.append(pd.to_datetime(average, unit='s'))
+        except TypeError:
+            averages.append(pd.NaT)
+    temp_df['averages'] = averages
+
+    try:
+        dt_center = (temp_df['averages'].tz_localize(dt.dt.tz)).dt.tz_localize(dt.dt.tz)
+    except TypeError:  # not a timeseries index
+        dt_center = (temp_df['averages']).dt.tz_localize(dt.dt.tz)
+
+    return dt_center
+
+
 def _mk_test(x, alpha=0.05):
     '''
     Mann-Kendall test of significance for trend (used in classical