diff --git a/scripts/postprocess_generated_models.py b/scripts/postprocess_generated_models.py index 4591d53f..204ac2d0 100644 --- a/scripts/postprocess_generated_models.py +++ b/scripts/postprocess_generated_models.py @@ -25,6 +25,7 @@ from __future__ import annotations import ast +import keyword import re import subprocess from pathlib import Path @@ -102,6 +103,16 @@ def _base_names(node: ast.ClassDef) -> set[str]: return {b.id for b in node.bases if isinstance(b, ast.Name)} +def _escape_docstring(docstring: str) -> str: + """Escape `docstring` for use between triple double quotes. + + Escapes every backslash, every NUL byte (Python source cannot contain one), and every double quote that would + close the literal: one followed by two more double quotes, or the last character. + """ + escaped = docstring.replace('\\', '\\\\').replace('\0', '\\x00') + return re.sub(r'"(?=""|\Z)', r'\\"', escaped) + + def fix_discriminators(content: str) -> str: """Replace camelCase discriminator values with their snake_case equivalents.""" for camel, snake in DISCRIMINATOR_FIXES.items(): @@ -156,12 +167,18 @@ def convert_enums_to_literals(content: str) -> str: new_lines.extend(f' {v!r},' for v in values) new_lines.append('] | str') if docstring is not None: + escaped = _escape_docstring(docstring) if '\n' in docstring: - new_lines.append('"""') - new_lines.extend(docstring.splitlines()) - new_lines.append('"""') + docstring_lines = ['"""', *escaped.split('\n'), '"""'] + expected = f'\n{docstring}\n' else: - new_lines.append(f'"""{docstring}"""') + docstring_lines = [f'"""{escaped}"""'] + expected = docstring + # The docstring text comes from the downloaded specification, so check that the emitted literal reads back + # as the docstring value. + if ast.literal_eval('\n'.join(docstring_lines)) != expected: + raise ValueError(f'Cannot emit the docstring of {node.name} as a string literal: {docstring!r}') + new_lines.extend(docstring_lines) assert node.end_lineno is not None # noqa: S101 replacements.append((node.lineno - 1, node.end_lineno, new_lines)) @@ -518,6 +535,10 @@ def _rename_fields_in_class_block(block: list[str], field_aliases: dict[str, str Matches lines of the form `:` and substitutes the identifier when an alias is present. Multi-line annotations and trailing default values are preserved verbatim because only the field name on the first line is replaced. + + Raises: + ValueError: If an alias is not a valid Python identifier. Aliases come from the downloaded specification and + are written into the generated source as field names. """ field_decl = re.compile(r'^(\s+)([a-z_][a-z0-9_]*)(\s*:)') out: list[str] = [] @@ -531,6 +552,8 @@ def _rename_fields_in_class_block(block: list[str], field_aliases: dict[str, str if api_name is None or api_name == name: out.append(line) continue + if not api_name.isidentifier() or keyword.iskeyword(api_name): + raise ValueError(f'Cannot emit the alias of field {name} as a field name: {api_name!r}') out.append(f'{indent}{api_name}{colon}{line[m.end() :]}') return out diff --git a/tests/unit/test_postprocess_generated_models.py b/tests/unit/test_postprocess_generated_models.py index 9087587f..f7187880 100644 --- a/tests/unit/test_postprocess_generated_models.py +++ b/tests/unit/test_postprocess_generated_models.py @@ -1,7 +1,10 @@ from __future__ import annotations +import ast import textwrap +import pytest + from scripts.postprocess_generated_models import ( add_camel_case_typeddicts, add_docs_group_decorators, @@ -214,6 +217,41 @@ class Status(StrEnum): assert alias_idx < docstring_idx +@pytest.mark.parametrize( + 'docstring', + [ + pytest.param('Use """ here; import os', id='triple quote on single line'), + pytest.param('Summary.\n"""\nimport os\n"""', id='triple quotes on multiple lines'), + pytest.param('Four """" quotes', id='run of four quotes'), + pytest.param('Ends with a "quote"', id='trailing quote'), + pytest.param('Match \\d+ or a literal \\n.', id='backslashes'), + pytest.param('Summary.\n\nMatch \\d+ or a literal \\n.', id='backslashes on multiple lines'), + pytest.param('Summary.\nLine\u2028separator and form\x0cfeed', id='non-newline line boundaries'), + pytest.param('Null \x00 byte', id='null byte'), + ], +) +def test_convert_enums_to_literals_round_trips_docstring(docstring: str) -> None: + """The trailing docstring keeps the exact value of the enum docstring and adds no statements.""" + content = f"from typing import Literal\n\nclass Status(StrEnum):\n {docstring!r}\n\n READY = 'READY'\n" + result = convert_enums_to_literals(content) + + tree = ast.parse(result) + assert [type(stmt) for stmt in tree.body] == [ast.ImportFrom, ast.Assign, ast.Expr] + trailing_docstring = tree.body[2] + assert isinstance(trailing_docstring, ast.Expr) + # A multi-line docstring is emitted with its delimiters on their own lines. + expected = f'\n{docstring}\n' if '\n' in docstring else docstring + assert ast.literal_eval(trailing_docstring.value) == expected + + +def test_convert_enums_to_literals_rejects_docstring_it_cannot_round_trip() -> None: + """A docstring that would read back with a different value raises instead of being emitted.""" + docstring = 'Summary.\nCarriage\rreturn' + content = f"from typing import Literal\n\nclass Status(StrEnum):\n {docstring!r}\n\n READY = 'READY'\n" + with pytest.raises(ValueError, match='docstring of Status'): + convert_enums_to_literals(content) + + def test_convert_enums_to_literals_preserves_hyphenated_values() -> None: """The string values are taken verbatim — hyphenated values like `'TIMED-OUT'` survive intact.""" content = textwrap.dedent("""\ @@ -595,6 +633,28 @@ class FooDict(TypedDict): assert 'retryCount: NotRequired[int]' in result +@pytest.mark.parametrize( + 'alias', + [ + pytest.param('x: int\n import os\n userId', id='code on new lines'), + pytest.param('user-id', id='hyphen'), + pytest.param('class', id='keyword'), + pytest.param('', id='empty'), + ], +) +def test_add_camel_case_typeddicts_rejects_alias_that_is_not_identifier(alias: str) -> None: + """An alias that is not a valid Python identifier raises instead of being written into the source.""" + content = textwrap.dedent("""\ + from typing import NotRequired, TypedDict + + class FooDict(TypedDict): + user_id: NotRequired[str] + """) + alias_map = {'Foo': {'user_id': alias}} + with pytest.raises(ValueError, match='alias of field user_id'): + add_camel_case_typeddicts(content, alias_map) + + def test_add_camel_case_typeddicts_rewires_nested_refs() -> None: """References to other snake TypedDicts in a cloned annotation are renamed to their camel form.""" content = textwrap.dedent("""\