Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions scripts/postprocess_generated_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from __future__ import annotations

import ast
import keyword
import re
import subprocess
from pathlib import Path
Expand Down Expand Up @@ -102,6 +103,16 @@ def _base_names(node: ast.ClassDef) -> set[str]:
return {b.id for b in node.bases if isinstance(b, ast.Name)}


def _escape_docstring(docstring: str) -> str:
"""Escape `docstring` for use between triple double quotes.

Escapes every backslash, every NUL byte (Python source cannot contain one), and every double quote that would
close the literal: one followed by two more double quotes, or the last character.
"""
escaped = docstring.replace('\\', '\\\\').replace('\0', '\\x00')
return re.sub(r'"(?=""|\Z)', r'\\"', escaped)


def fix_discriminators(content: str) -> str:
"""Replace camelCase discriminator values with their snake_case equivalents."""
for camel, snake in DISCRIMINATOR_FIXES.items():
Expand Down Expand Up @@ -156,12 +167,18 @@ def convert_enums_to_literals(content: str) -> str:
new_lines.extend(f' {v!r},' for v in values)
new_lines.append('] | str')
if docstring is not None:
escaped = _escape_docstring(docstring)
if '\n' in docstring:
new_lines.append('"""')
new_lines.extend(docstring.splitlines())
new_lines.append('"""')
docstring_lines = ['"""', *escaped.split('\n'), '"""']
expected = f'\n{docstring}\n'
else:
new_lines.append(f'"""{docstring}"""')
docstring_lines = [f'"""{escaped}"""']
expected = docstring
# The docstring text comes from the downloaded specification, so check that the emitted literal reads back
# as the docstring value.
if ast.literal_eval('\n'.join(docstring_lines)) != expected:
raise ValueError(f'Cannot emit the docstring of {node.name} as a string literal: {docstring!r}')
new_lines.extend(docstring_lines)

assert node.end_lineno is not None # noqa: S101
replacements.append((node.lineno - 1, node.end_lineno, new_lines))
Expand Down Expand Up @@ -518,6 +535,10 @@ def _rename_fields_in_class_block(block: list[str], field_aliases: dict[str, str
Matches lines of the form `<indent><snake_ident>:` and substitutes the identifier when an alias is present.
Multi-line annotations and trailing default values are preserved verbatim because only the field name
on the first line is replaced.

Raises:
ValueError: If an alias is not a valid Python identifier. Aliases come from the downloaded specification and
are written into the generated source as field names.
"""
field_decl = re.compile(r'^(\s+)([a-z_][a-z0-9_]*)(\s*:)')
out: list[str] = []
Expand All @@ -531,6 +552,8 @@ def _rename_fields_in_class_block(block: list[str], field_aliases: dict[str, str
if api_name is None or api_name == name:
out.append(line)
continue
if not api_name.isidentifier() or keyword.iskeyword(api_name):
raise ValueError(f'Cannot emit the alias of field {name} as a field name: {api_name!r}')
out.append(f'{indent}{api_name}{colon}{line[m.end() :]}')
return out

Expand Down
60 changes: 60 additions & 0 deletions tests/unit/test_postprocess_generated_models.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
from __future__ import annotations

import ast
import textwrap

import pytest

from scripts.postprocess_generated_models import (
add_camel_case_typeddicts,
add_docs_group_decorators,
Expand Down Expand Up @@ -214,6 +217,41 @@ class Status(StrEnum):
assert alias_idx < docstring_idx


@pytest.mark.parametrize(
'docstring',
[
pytest.param('Use """ here; import os', id='triple quote on single line'),
pytest.param('Summary.\n"""\nimport os\n"""', id='triple quotes on multiple lines'),
pytest.param('Four """" quotes', id='run of four quotes'),
pytest.param('Ends with a "quote"', id='trailing quote'),
pytest.param('Match \\d+ or a literal \\n.', id='backslashes'),
pytest.param('Summary.\n\nMatch \\d+ or a literal \\n.', id='backslashes on multiple lines'),
pytest.param('Summary.\nLine\u2028separator and form\x0cfeed', id='non-newline line boundaries'),
pytest.param('Null \x00 byte', id='null byte'),
],
)
def test_convert_enums_to_literals_round_trips_docstring(docstring: str) -> None:
"""The trailing docstring keeps the exact value of the enum docstring and adds no statements."""
content = f"from typing import Literal\n\nclass Status(StrEnum):\n {docstring!r}\n\n READY = 'READY'\n"
result = convert_enums_to_literals(content)

tree = ast.parse(result)
assert [type(stmt) for stmt in tree.body] == [ast.ImportFrom, ast.Assign, ast.Expr]
trailing_docstring = tree.body[2]
assert isinstance(trailing_docstring, ast.Expr)
# A multi-line docstring is emitted with its delimiters on their own lines.
expected = f'\n{docstring}\n' if '\n' in docstring else docstring
assert ast.literal_eval(trailing_docstring.value) == expected


def test_convert_enums_to_literals_rejects_docstring_it_cannot_round_trip() -> None:
"""A docstring that would read back with a different value raises instead of being emitted."""
docstring = 'Summary.\nCarriage\rreturn'
content = f"from typing import Literal\n\nclass Status(StrEnum):\n {docstring!r}\n\n READY = 'READY'\n"
with pytest.raises(ValueError, match='docstring of Status'):
convert_enums_to_literals(content)


def test_convert_enums_to_literals_preserves_hyphenated_values() -> None:
"""The string values are taken verbatim — hyphenated values like `'TIMED-OUT'` survive intact."""
content = textwrap.dedent("""\
Expand Down Expand Up @@ -595,6 +633,28 @@ class FooDict(TypedDict):
assert 'retryCount: NotRequired[int]' in result


@pytest.mark.parametrize(
'alias',
[
pytest.param('x: int\n import os\n userId', id='code on new lines'),
pytest.param('user-id', id='hyphen'),
pytest.param('class', id='keyword'),
pytest.param('', id='empty'),
],
)
def test_add_camel_case_typeddicts_rejects_alias_that_is_not_identifier(alias: str) -> None:
"""An alias that is not a valid Python identifier raises instead of being written into the source."""
content = textwrap.dedent("""\
from typing import NotRequired, TypedDict

class FooDict(TypedDict):
user_id: NotRequired[str]
""")
alias_map = {'Foo': {'user_id': alias}}
with pytest.raises(ValueError, match='alias of field user_id'):
add_camel_case_typeddicts(content, alias_map)


def test_add_camel_case_typeddicts_rewires_nested_refs() -> None:
"""References to other snake TypedDicts in a cloned annotation are renamed to their camel form."""
content = textwrap.dedent("""\
Expand Down
Loading