From b649f3522bd31e234da688a831f1a102f165115f Mon Sep 17 00:00:00 2001 From: lorenzo riches Date: Fri, 30 Oct 2026 11:28:36 +1100 Subject: [PATCH 1/2] explicit parsing of timestamps --- pvlib/iotools/solcast.py | 26 +++++++++++++++- tests/iotools/test_solcast.py | 56 ++++++++++++++++++++++++++++++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/pvlib/iotools/solcast.py b/pvlib/iotools/solcast.py index 96ed677097..eeeb928c9d 100644 --- a/pvlib/iotools/solcast.py +++ b/pvlib/iotools/solcast.py @@ -78,6 +78,8 @@ def get_solcast_tmy( data : pandas.DataFrame containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. + The index is timezone-aware, using the UTC offset returned by + the API, or UTC if the returned timestamps have mixed offsets. metadata: dict latitude and longitude of the request. @@ -179,6 +181,8 @@ def get_solcast_historic( data : pandas.DataFrame containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. + The index is timezone-aware, using the UTC offset returned by + the API, or UTC if the returned timestamps have mixed offsets. metadata: dict latitude and longitude of the request. @@ -263,6 +267,8 @@ def get_solcast_forecast( data : pandas.DataFrame Contains the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. + The index is timezone-aware, using the UTC offset returned by + the API, or UTC if the returned timestamps have mixed offsets. metadata: dict latitude and longitude of the request. @@ -341,6 +347,8 @@ def get_solcast_live( data : pandas.DataFrame containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. + The index is timezone-aware, using the UTC offset returned by + the API, or UTC if the returned timestamps have mixed offsets. metadata: dict latitude and longitude of the request. @@ -400,6 +408,22 @@ def get_solcast_live( return data, {"latitude": latitude, "longitude": longitude} +def _parse_period_end(period_end): + """Parse Solcast ``period_end`` timestamps to a tz-aware DatetimeIndex. + + Supports both the legacy format (``2025-04-16T06:00:00.0000000Z``) and + the ISO 8601 UTC offset format (``2025-04-16T14:00:00+01:30``). + If all timestamps share the same UTC offset, that offset is kept; + otherwise the timestamps are converted to UTC. + """ + period_end = pd.Series(period_end).astype(str) + times = pd.DatetimeIndex(pd.to_datetime(period_end, utc=True)) + offsets = period_end.str.extract(r'(Z|[+-]\d{2}:?\d{2})$')[0] + if len(times) > 0 and offsets.nunique(dropna=False) == 1: + times = times.tz_convert(pd.Timestamp(period_end.iloc[0]).tz) + return times + + def _solcast2pvlib(data): """Format the data from Solcast to pvlib's conventions. @@ -414,7 +438,7 @@ def _solcast2pvlib(data): """ # move from period_end to period_middle as per pvlib convention - data["period_mid"] = pd.to_datetime( + data["period_mid"] = _parse_period_end( data.period_end) - pd.to_timedelta(data.period.values) / 2 data = data.set_index("period_mid").drop(columns=["period_end", "period"]) diff --git a/tests/iotools/test_solcast.py b/tests/iotools/test_solcast.py index d6df24c222..8dbf7d088f 100644 --- a/tests/iotools/test_solcast.py +++ b/tests/iotools/test_solcast.py @@ -185,11 +185,65 @@ def test_get_solcast_tmy( ['2023-09-20 01:45:00+00:00', '2023-09-20 01:15:00+00:00'], dtype='datetime64[ns, UTC]', name='period_mid', freq=None) ) - ) + ), + ( + # new standard API format: float air_temp, ISO 8601 UTC offset + pd.DataFrame( + [[942, 843, 1017.4, 30.2, 7.8, 316, 1010, -2, 4.6, 16.4, + '2023-09-20T12:00:00+10:00', 'PT30M', 90], + [936, 832, 1017.9, 29.8, 7.9, 316, 996, -14, 4.5, 16.3, + '2023-09-20T11:30:00+10:00', 'PT30M', 0]], + columns=[ + 'dni', 'ghi', 'surface_pressure', 'air_temp', 'wind_speed_10m', + 'wind_direction_10m', 'gti', 'azimuth', 'dewpoint_temp', + 'precipitable_water', 'period_end', 'period', 'zenith'], + index=pd.RangeIndex(start=0, stop=2, step=1) + ), + pd.DataFrame( + [[9.4200e+02, 8.4300e+02, 1.0174e+05, 3.0200e+01, 7.8000e+00, + 3.1600e+02, 1.0100e+03, 2.0000e+00, 4.6000e+00, 1.6400e+00, 90], + [9.3600e+02, 8.3200e+02, 1.0179e+05, 2.9800e+01, 7.9000e+00, + 3.1600e+02, 9.9600e+02, 1.4000e+01, 4.5000e+00, 1.6300e+00, 0]], + columns=[ + 'dni', 'ghi', 'pressure', 'temp_air', 'wind_speed', + 'wind_direction', 'poa_global', 'solar_azimuth', + 'temp_dew', 'precipitable_water', 'solar_zenith'], + index=pd.DatetimeIndex( + ['2023-09-20 11:45:00+10:00', '2023-09-20 11:15:00+10:00'], + name='period_mid', freq=None) + ) + ), ]) def test_solcast2pvlib(in_df, out_df): df = pvlib.iotools.solcast._solcast2pvlib(in_df) pd.testing.assert_frame_equal(df.astype(float), out_df.astype(float)) + assert df.index.tz is not None + + +@pytest.mark.parametrize("period_end,expected", [ + # legacy format, 7 fractional-second digits and Z suffix + (['2025-04-16T06:00:00.0000000Z', '2025-04-16T06:30:00.0000000Z'], + ['2025-04-16T06:00:00+00:00', '2025-04-16T06:30:00+00:00']), + # new standard format, explicit UTC offset + (['2025-04-16T14:00:00+01:30', '2025-04-16T14:30:00+01:30'], + ['2025-04-16T14:00:00+01:30', '2025-04-16T14:30:00+01:30']), + (['2025-04-16T06:00:00+00:00'], ['2025-04-16T06:00:00+00:00']), +]) +def test_parse_period_end(period_end, expected): + out = pvlib.iotools.solcast._parse_period_end(pd.Series(period_end)) + expected = pd.DatetimeIndex(pd.to_datetime(expected)) + assert out.equals(expected) + assert str(out.tz) == str(expected.tz) + + +def test_parse_period_end_mixed_offsets(): + # e.g. a local time series spanning a daylight saving transition + out = pvlib.iotools.solcast._parse_period_end(pd.Series( + ['2025-04-06T01:00:00+11:00', '2025-04-06T03:00:00+10:00'])) + expected = pd.DatetimeIndex( + ['2025-04-05T14:00:00', '2025-04-05T17:00:00'], tz='UTC') + assert str(out.tz) == 'UTC' + assert out.equals(expected) @pytest.mark.parametrize("map_variables", [True, False]) From 578270ada4e2948a49060e248142536e7d17576e Mon Sep 17 00:00:00 2001 From: lorenzo riches Date: Tue, 17 Nov 2026 11:09:08 +1100 Subject: [PATCH 2/2] review --- pvlib/iotools/solcast.py | 39 +++++++++++++++-------------------- tests/iotools/test_solcast.py | 37 ++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/pvlib/iotools/solcast.py b/pvlib/iotools/solcast.py index eeeb928c9d..6c6dc9e496 100644 --- a/pvlib/iotools/solcast.py +++ b/pvlib/iotools/solcast.py @@ -79,7 +79,7 @@ def get_solcast_tmy( containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. The index is timezone-aware, using the UTC offset returned by - the API, or UTC if the returned timestamps have mixed offsets. + the API. metadata: dict latitude and longitude of the request. @@ -117,8 +117,7 @@ def get_solcast_tmy( params = dict( latitude=latitude, longitude=longitude, - format="json", - **kwargs + **{"format": "json", **kwargs} ) data = _get_solcast( @@ -182,7 +181,7 @@ def get_solcast_historic( containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. The index is timezone-aware, using the UTC offset returned by - the API, or UTC if the returned timestamps have mixed offsets. + the API. metadata: dict latitude and longitude of the request. @@ -225,8 +224,7 @@ def get_solcast_historic( end=end, duration=duration, api_key=api_key, - format="json", - **kwargs + **{"format": "json", **kwargs} ) data = _get_solcast( @@ -260,7 +258,9 @@ def get_solcast_forecast( from Solcast's "period end" convention. kwargs: Optional parameters passed to the API. - See [2]_ for full list of parameters. + See [2]_ for full list of parameters. ``format`` defaults to + ``"json"``; use ``format="json;version=3"`` to opt in to the + Solcast response format version 3 before it becomes the default. Returns ------- @@ -268,7 +268,7 @@ def get_solcast_forecast( Contains the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. The index is timezone-aware, using the UTC offset returned by - the API, or UTC if the returned timestamps have mixed offsets. + the API. metadata: dict latitude and longitude of the request. @@ -305,8 +305,7 @@ def get_solcast_forecast( params = dict( latitude=latitude, longitude=longitude, - format="json", - **kwargs + **{"format": "json", **kwargs} ) data = _get_solcast( @@ -340,7 +339,9 @@ def get_solcast_live( from Solcast's "period end" convention. kwargs: Optional parameters passed to the API. - See [2]_ for full list of parameters. + See [2]_ for full list of parameters. ``format`` defaults to + ``"json"``; use ``format="json;version=3"`` to opt in to the + Solcast response format version 3 before it becomes the default. Returns ------- @@ -348,7 +349,7 @@ def get_solcast_live( containing the values for the parameters requested. The times in the DataFrame index indicate the midpoint of each interval. The index is timezone-aware, using the UTC offset returned by - the API, or UTC if the returned timestamps have mixed offsets. + the API. metadata: dict latitude and longitude of the request. @@ -394,8 +395,7 @@ def get_solcast_live( params = dict( latitude=latitude, longitude=longitude, - format="json", - **kwargs + **{"format": "json", **kwargs} ) data = _get_solcast( @@ -413,15 +413,10 @@ def _parse_period_end(period_end): Supports both the legacy format (``2025-04-16T06:00:00.0000000Z``) and the ISO 8601 UTC offset format (``2025-04-16T14:00:00+01:30``). - If all timestamps share the same UTC offset, that offset is kept; - otherwise the timestamps are converted to UTC. + The API returns all timestamps with a single fixed UTC offset + (set by the ``time_zone`` parameter), which is kept. """ - period_end = pd.Series(period_end).astype(str) - times = pd.DatetimeIndex(pd.to_datetime(period_end, utc=True)) - offsets = period_end.str.extract(r'(Z|[+-]\d{2}:?\d{2})$')[0] - if len(times) > 0 and offsets.nunique(dropna=False) == 1: - times = times.tz_convert(pd.Timestamp(period_end.iloc[0]).tz) - return times + return pd.DatetimeIndex(pd.to_datetime(period_end)) def _solcast2pvlib(data): diff --git a/tests/iotools/test_solcast.py b/tests/iotools/test_solcast.py index 8dbf7d088f..6fb0c790dd 100644 --- a/tests/iotools/test_solcast.py +++ b/tests/iotools/test_solcast.py @@ -236,16 +236,6 @@ def test_parse_period_end(period_end, expected): assert str(out.tz) == str(expected.tz) -def test_parse_period_end_mixed_offsets(): - # e.g. a local time series spanning a daylight saving transition - out = pvlib.iotools.solcast._parse_period_end(pd.Series( - ['2025-04-06T01:00:00+11:00', '2025-04-06T03:00:00+10:00'])) - expected = pd.DatetimeIndex( - ['2025-04-05T14:00:00', '2025-04-05T17:00:00'], tz='UTC') - assert str(out.tz) == 'UTC' - assert out.equals(expected) - - @pytest.mark.parametrize("map_variables", [True, False]) @pytest.mark.parametrize("endpoint,function,params,json_response", [ ( @@ -356,6 +346,33 @@ def test_get_solcast_forecast( ) +@pytest.mark.parametrize("endpoint,function,key", [ + ("forecast/radiation_and_weather", + pvlib.iotools.get_solcast_forecast, "forecasts"), + ("live/radiation_and_weather", + pvlib.iotools.get_solcast_live, "estimated_actuals"), +]) +def test_get_solcast_format_version_3(requests_mock, endpoint, function, key): + # response format version 3: ISO 8601 UTC offset and float air_temp + requests_mock.get( + f"https://api.solcast.com.au/data/{endpoint}", + json={key: [ + {'air_temp': 20.1, 'period_end': '2025-04-16T14:00:00+01:30', + 'period': 'PT30M'}, + {'air_temp': 20.6, 'period_end': '2025-04-16T14:30:00+01:30', + 'period': 'PT30M'}, + ]}) + df, _ = function(latitude=0, longitude=0, api_key="1234", + format="json;version=3") + assert requests_mock.last_request.qs["format"] == ["json;version=3"] + expected_index = pd.DatetimeIndex( + ['2025-04-16T13:45:00+01:30', '2025-04-16T14:15:00+01:30'], + name='period_mid') + assert df.index.equals(expected_index) + assert str(df.index.tz) == 'UTC+01:30' + assert df['temp_air'].tolist() == [20.1, 20.6] + + @pytest.mark.parametrize( "function", [