Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -1387,6 +1387,8 @@ response = await fga_client.execute_api_request(

If a network request fails with a 429 or 5xx error from the server, the SDK will automatically retry the request up to 3 times with a minimum wait time of 100 milliseconds between each attempt.

The SDK honors `Retry-After` delay seconds or HTTP dates up to 30 minutes. Unparseable or out-of-range values fall back to exponential backoff. In the sync client, urllib3's own `Retry-After` handling is disabled unless you set `Configuration.retries`.

To customize this behavior, create a `RetryParams` object and assign values to the `max_retry` and `min_wait_in_ms` constructor parameters. `max_retry` determines the maximum number of retries (up to 15), while `min_wait_in_ms` sets the minimum wait time between retries in milliseconds.

Apply your custom retry values by passing the object to the `ClientConfiguration` constructor's `retry_params` parameter.
Expand Down
26 changes: 15 additions & 11 deletions openfga_sdk/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import time
import urllib

from email.utils import parsedate_to_datetime
from multiprocessing.pool import ThreadPool

from dateutil.parser import parse # type: ignore[import-untyped]
Expand All @@ -19,6 +20,7 @@
from openfga_sdk.configuration import Configuration
from openfga_sdk.constants import (
MAX_BACKOFF_TIME_IN_SEC,
RETRY_AFTER_HEADER_NAME,
RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC,
)
from openfga_sdk.exceptions import (
Expand Down Expand Up @@ -299,6 +301,7 @@ async def __call_api(
configuration=self.configuration.telemetry,
)

wait_time_in_sec: float
try:
wait_time_in_sec = self._parse_retry_after_header(e.header)
except ValueError:
Expand Down Expand Up @@ -423,23 +426,24 @@ async def __call_api(
return (return_data, response_data.status, response_data.headers)

def _parse_retry_after_header(self, headers) -> int:
retry_after_header = headers.get("retry-after")
retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower())
if not retry_after_header:
raise ValueError("Retry-After header is not present")

# RFC 9110: Retry-After is either delay-seconds or an HTTP-date
try:
parsed_http_date = self.__deserialize_datetime(retry_after_header).replace(
tzinfo=datetime.timezone.utc
)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (parsed_http_date - now).total_seconds()
except ApiException:
wait_time_in_sec = int(retry_after_header)
except ValueError:
try:
retry_at = parsedate_to_datetime(retry_after_header)
Comment thread
SoulPancake marked this conversation as resolved.
Resolved
except (OverflowError, ValueError) as e:
raise ValueError("Retry-After header is invalid") from e
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=datetime.timezone.utc)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (retry_at - now).total_seconds()

if (
wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC
or wait_time_in_sec < 1
):
if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC:
Comment thread
SoulPancake marked this conversation as resolved.
Resolved
raise ValueError("Retry-After header is invalid")

return math.ceil(wait_time_in_sec)
Expand Down
3 changes: 2 additions & 1 deletion openfga_sdk/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@ def __init__(
"""Safe chars for path_param
"""
self.retries = None
"""Adding retries to override urllib3 default value 3
"""urllib3 Retry for the sync client. Defaults to 3 retries with
Retry-After handling disabled; the SDK retries 429/5xx itself.
"""
# Enable client side validation
self.client_side_validation = True
Expand Down
7 changes: 4 additions & 3 deletions openfga_sdk/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@
Documentation: https://openfga.dev/docs
Support: https://openfga.dev/community
License: [Apache-2.0](https://github.com/openfga/python-sdk/blob/main/LICENSE)

NOTE: This file was auto generated by OpenAPI Generator (https://openapi-generator.tech). DO NOT EDIT.
"""

from openfga_sdk.constants import RETRY_AFTER_HEADER_NAME


# Specific FGA header to be parsed
X_RATELIMIT_LIMIT = "x-ratelimit-limit"
X_RATELIMIT_REMAINING = "x_ratelimit_remaining"
X_RATELIMIT_RESET = "x_ratelimit_reset"
FGA_REQUEST_ID = "fga-request-id"
FGA_QUERY_DURATION_MS = "fga-query-duration-ms"
OPENFGA_AUTHORIZATION_MODEL_ID = "openfga_authorization_model_id"
RETRY_AFTER = "retry-after"
RETRY_AFTER = RETRY_AFTER_HEADER_NAME.lower()
RESPONSE_HEADERS_TO_KEEP = [
X_RATELIMIT_LIMIT,
X_RATELIMIT_REMAINING,
Expand Down
26 changes: 15 additions & 11 deletions openfga_sdk/sync/api_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import time
import urllib

from email.utils import parsedate_to_datetime
from multiprocessing.pool import ThreadPool

from dateutil.parser import parse # type: ignore[import-untyped]
Expand All @@ -17,6 +18,7 @@
from openfga_sdk.configuration import Configuration
from openfga_sdk.constants import (
MAX_BACKOFF_TIME_IN_SEC,
RETRY_AFTER_HEADER_NAME,
RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC,
)
from openfga_sdk.exceptions import (
Expand Down Expand Up @@ -299,6 +301,7 @@ def __call_api(
configuration=self.configuration.telemetry,
)

wait_time_in_sec: float
try:
wait_time_in_sec = self._parse_retry_after_header(e.header)
except ValueError:
Expand Down Expand Up @@ -424,23 +427,24 @@ def __call_api(
return (return_data, response_data.status, response_data.headers)

def _parse_retry_after_header(self, headers) -> int:
retry_after_header = headers.get("retry-after")
retry_after_header = headers.get(RETRY_AFTER_HEADER_NAME.lower())
if not retry_after_header:
raise ValueError("Retry-After header is not present")

# RFC 9110: Retry-After is either delay-seconds or an HTTP-date
try:
parsed_http_date = self.__deserialize_datetime(retry_after_header).replace(
tzinfo=datetime.timezone.utc
)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (parsed_http_date - now).total_seconds()
except ApiException:
wait_time_in_sec = int(retry_after_header)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

THIS IS NOT A BLOCKING ISSUE

Now that we're first checking if Retry-After header contains seconds first before checking for date, the HTTP spec writes the seconds form as delay-seconds = 1*DIGIT, which means "one or more of the digits 0–9 and nothing else."

But int() also accepts +60, 1_0 (read as 10) and non-ASCII digits. I checked over real HTTP: +60 and 1_0 are honored by both clients. ٦٠ is honored as 60s by the async client, but the sync client falls back to backoff, so the two clients behave differently.

The 1–1800s cap limits the impact, but checking for ASCII digits first would match the spec and keep both clients consistent, something like this

if re.fullmatch(r"[0-9]+", retry_after_header):
            wait_time_in_sec = int(retry_after_header)
else:

(and indent the HTTP-date branch under else:, in both api_client.py and sync/api_client.py). Adding "+60" and "1_0" to test_parse_retry_after_rejects_invalid would cover it.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@SoulPancake take a look at this also. If you want to go by this approach or not.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like no real server sends non-aSCII digits though 🤔

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer to keep int() here since the JS, Go, Java and .NET SDKs all accept +60 too, and the 1 to 1800s range limits the impact

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can standardise across SDKs later if we want to have this @cluelessno1

except ValueError:
try:
retry_at = parsedate_to_datetime(retry_after_header)
Comment thread
SoulPancake marked this conversation as resolved.
Resolved
except (OverflowError, ValueError) as e:
raise ValueError("Retry-After header is invalid") from e
if retry_at.tzinfo is None:
retry_at = retry_at.replace(tzinfo=datetime.timezone.utc)
now = datetime.datetime.now(datetime.timezone.utc)
wait_time_in_sec = (retry_at - now).total_seconds()

if (
wait_time_in_sec > RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC
or wait_time_in_sec < 1
):
if not 1 <= wait_time_in_sec <= RETRY_HEADER_MAX_ALLOWABLE_DURATION_IN_SEC:
Comment thread
SoulPancake marked this conversation as resolved.
Resolved
raise ValueError("Retry-After header is invalid")

return math.ceil(wait_time_in_sec)
Expand Down
6 changes: 6 additions & 0 deletions openfga_sdk/sync/rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ def __init__(

if hasattr(configuration, "retries") and configuration.retries is not None:
addition_pool_args["retries"] = configuration.retries
else:
# The SDK retries 429/5xx itself; keep urllib3 from also retrying
# idempotent requests and sleeping for an uncapped Retry-After.
addition_pool_args["retries"] = urllib3.Retry(
total=3, respect_retry_after_header=False
Comment thread
SoulPancake marked this conversation as resolved.
Resolved
)

if (
hasattr(configuration, "socket_options")
Expand Down
23 changes: 14 additions & 9 deletions test/api/open_fga_api_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1456,7 +1456,7 @@ async def test_429_error_retry_configured_unparseable_retry_after(
):
"""
Test to ensure 429 errors are handled properly.
For this case, retry is configured and the Retry-After header is provided as an HTTP date.
For this case, the Retry-After header is invalid.
"""
response_body = '{"allowed": true, "resolution": "1234"}'
error_response_body = """
Expand All @@ -1465,20 +1465,22 @@ async def test_429_error_retry_configured_unparseable_retry_after(
"message": "Rate Limit exceeded"
}
"""
retry_after_in_sec = 5
five_seconds_from_now = f"{retry_after_in_sec}s"
min_wait_in_ms = 10
max_wait_in_sec = 1
mock_http_response = http_mock_response(
body=error_response_body,
status=429,
headers={"Retry-After": five_seconds_from_now},
headers={"Retry-After": "not-a-date"},
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
)
mock_request.side_effect = [
RateLimitExceededError(http_resp=mock_http_response),
mock_response(response_body, 200),
]

retry = openfga_sdk.configuration.RetryParams(
max_retry=1, min_wait_in_ms=10, max_wait_in_sec=1
max_retry=1,
min_wait_in_ms=min_wait_in_ms,
max_wait_in_sec=max_wait_in_sec,
)
configuration = self.configuration
configuration.store_id = store_id
Expand All @@ -1499,7 +1501,10 @@ async def test_429_error_retry_configured_unparseable_retry_after(
self.assertTrue(api_response.allowed)
mock_request.assert_called()
self.assertEqual(mock_request.call_count, 2)
self.assertNotEqual(mock_sleep.call_args[0][0], retry_after_in_sec)
# an invalid header falls back to exponential backoff
self.assertTrue(
min_wait_in_ms / 1000 <= mock_sleep.call_args[0][0] <= max_wait_in_sec
)

@patch("asyncio.sleep")
@patch.object(rest.RESTClientObject, "request")
Expand Down Expand Up @@ -1566,7 +1571,7 @@ async def test_429_error_retry_configured_with_delay_seconds(
):
"""
Test to ensure 429 errors are handled properly.
For this case, retry is configured and the Retry-After header is provided as an HTTP date.
For this case, the Retry-After header is provided as delay seconds.
"""
response_body = '{"allowed": true, "resolution": "1234"}'
error_response_body = """
Expand All @@ -1575,11 +1580,11 @@ async def test_429_error_retry_configured_with_delay_seconds(
"message": "Rate Limit exceeded"
}
"""
retry_after_in_sec = 10
retry_after_in_sec = 60
mock_http_response = http_mock_response(
body=error_response_body,
status=429,
headers={"Retry-After": retry_after_in_sec},
headers={"Retry-After": str(retry_after_in_sec)},
)
mock_request.side_effect = [
RateLimitExceededError(http_resp=mock_http_response),
Expand Down
Loading