Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions sqlmesh/core/context_diff.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,10 @@ class ContextDiff(PydanticModel):
"""Environment statements."""
diff_rendered: bool = False
"""Whether the diff should compare raw vs rendered models"""
stored_parent_forward_only: t.Dict[SnapshotId, bool] = {}
"""Whether the earlier versions of the stored parents of new indirectly modified snapshots were
forward-only. A past version doesn't record this, so it is read from the snapshot stored for that
version."""

@classmethod
def create(
Expand Down Expand Up @@ -233,6 +237,9 @@ def create(

return ContextDiff(
environment=environment,
stored_parent_forward_only=_get_stored_parent_forward_only(
state_reader, merged_snapshots, new_snapshots, modified_snapshots
),
is_new_environment=is_new_environment,
is_unfinalized_environment=bool(env and not env.finalized_ts),
normalize_environment_name=is_new_environment or bool(env and env.normalize_name),
Expand Down Expand Up @@ -491,6 +498,57 @@ def text_diff(self, name: str) -> str:
return ""


def _get_stored_parent_forward_only(

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This gate is true whenever any stored direct parent changed, including a metadata-only or non-breaking change. A stored parent with no breaking history then switches the forward-only rule for the new parents too, which I think is the both-new behavior the description says is left for a follow-up.

Repro: child c has parents a (stored, changed only owner), b (new, forward-only BREAKING) and d (new, NON_BREAKING). On main, and on this PR when a is unchanged, c is rebuilt. On this PR, when a has only the metadata change, c is forward-only and keeps its old version. So the same b + d change gives the opposite result depending on whether a third parent had an owner change applied in dev.

Gating on stored_parent_forward_only instead fixes it. That set is only non-empty when a stored parent has a breaking version, so initialize it to set(). With this change all of test_plan.py and test_change_scenarios.py still pass, and the repro matches main. Could you add the repro as a test too? (The comment includes the code change and the full parametrized repro test.)

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same issue as the builder.py comment (the gate), addressed there in f4a943c.

state_reader: StateReader,
snapshots: t.Dict[SnapshotId, Snapshot],
new_snapshots: t.Dict[SnapshotId, Snapshot],
modified_snapshots: t.Dict[str, t.Tuple[Snapshot, Snapshot]],
) -> t.Dict[SnapshotId, bool]:
"""Returns whether the past versions of stored parents were forward-only.

A stored parent already exists in state and is a direct parent of a new indirectly modified snapshot
that also has a new direct parent, but it has changed since that snapshot's previous version. These
are the only snapshots for which the plan builder reads the flags. For each such parent, the earlier
versions in its history (`all_versions`, as used by the plan builder) created after the version the
previous snapshot was built from are returned. Nothing is returned for a parent whose history doesn't
reach back to that version, and versions that can't be found in state are left out.
"""
version_ids: t.Set[SnapshotId] = set()
for snapshot, previous_snapshot in modified_snapshots.values():
if (
snapshot.snapshot_id not in new_snapshots
or not any(parent_id in new_snapshots for parent_id in snapshot.parents)
or not snapshot.is_indirectly_modified(previous_snapshot)
):
continue
previous_parent_ids = {p.name: p for p in previous_snapshot.parents}
for parent_id in snapshot.parents:
parent = snapshots.get(parent_id)
previous_parent_id = previous_parent_ids.get(parent_id.name)
if (
not parent
or parent_id in new_snapshots
or previous_parent_id is None
or previous_parent_id == parent_id
):
continue
# The latest version is the parent itself, whose flag is known.
parent_version_ids: t.List[SnapshotId] = []
for version in reversed(parent.all_versions[:-1]):
version_id = version.snapshot_id(parent.name)
if version_id == previous_parent_id:
version_ids.update(parent_version_ids)
break
parent_version_ids.append(version_id)

if not version_ids:
return {}
return {
snapshot_id: snapshot.is_forward_only
for snapshot_id, snapshot in state_reader.get_snapshots(version_ids).items()
}


def _build_requirements(
provided_requirements: t.Dict[str, str],
excluded_requirements: t.Set[str],
Expand Down
168 changes: 154 additions & 14 deletions sqlmesh/core/plan/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
SnapshotChangeCategory,
)
from sqlmesh.core.snapshot.categorizer import categorize_change
from sqlmesh.core.snapshot.definition import Interval, SnapshotId
from sqlmesh.core.snapshot.definition import Interval, SnapshotDataVersion, SnapshotId
from sqlmesh.utils import columns_to_types_all_known, random_id
from sqlmesh.utils.dag import DAG
from sqlmesh.utils.date import (
Expand All @@ -52,6 +52,14 @@
logger = logging.getLogger(__name__)


# Changes in these categories don't require the history of downstream snapshots to be recomputed.
DOWNSTREAM_REBUILD_NOT_REQUIRED_CATEGORIES = {
SnapshotChangeCategory.METADATA,
SnapshotChangeCategory.NON_BREAKING,
SnapshotChangeCategory.INDIRECT_NON_BREAKING,
}


class PlanBuilder:
"""Plan Builder constructs a Plan based on user choices for how they want to backfill, preview, etc. their changes.

Expand Down Expand Up @@ -726,6 +734,7 @@ def _categorize_snapshot(
return

all_upstream_forward_only = set()
new_upstream_requires_rebuild = False
all_upstream_categories = set()
direct_parent_categories = set()

Expand All @@ -735,30 +744,168 @@ def _categorize_snapshot(
if parent and self._is_new_snapshot(parent):
all_upstream_categories.add(parent.change_category)
all_upstream_forward_only.add(parent.is_forward_only)
if parent.change_category not in DOWNSTREAM_REBUILD_NOT_REQUIRED_CATEGORIES:
new_upstream_requires_rebuild = True
if p_id in snapshot.parents:
direct_parent_categories.add(parent.change_category)

# Direct parents that already exist in state but changed relative to the previous version
# of this snapshot (eg. they were categorized in another plan) don't show up above.
stored_parent_categories: t.Set[SnapshotChangeCategory] = set()
stored_parent_forward_only: t.Set[bool] = set()
if direct_parent_categories:
stored_parent_categories, stored_parent_forward_only = (
self._get_stored_direct_parent_changes(snapshot)
)

if stored_parent_forward_only:
# A stored parent has a breaking version since the previous version of this snapshot (or
# its history can't be traced back that far), so that change is part of this one too and
# its forward-only flags are added to the vote. The new parents keep the rule used when
# only new parents changed, so the same changes to them give the same result. The only
# exception is when none of the new upstream changes require downstream history to be
# recomputed (metadata or non-breaking changes only): then they don't vote, so they can't
# undo a forward-only change to the stored parent. Without such a stored parent the
# decision is the same as when only new parents changed.
all_upstream_forward_only = (
all_upstream_forward_only if new_upstream_requires_rebuild else set()
) | stored_parent_forward_only

if all_upstream_forward_only == {True} or (
snapshot.is_model and snapshot.model.forward_only
):
forward_only = True

if direct_parent_categories.intersection(
if (direct_parent_categories | stored_parent_categories).intersection(
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
{SnapshotChangeCategory.BREAKING, SnapshotChangeCategory.INDIRECT_BREAKING}
):
snapshot.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING, forward_only)
elif not direct_parent_categories:
snapshot.categorize_as(
self._get_orphaned_indirect_change_category(snapshot), forward_only
)
elif all_upstream_categories == {SnapshotChangeCategory.METADATA}:
elif (all_upstream_categories | stored_parent_categories) == {
SnapshotChangeCategory.METADATA
}:
snapshot.categorize_as(SnapshotChangeCategory.METADATA, forward_only)
else:
snapshot.categorize_as(SnapshotChangeCategory.INDIRECT_NON_BREAKING, forward_only)
else:
# Metadata updated.
snapshot.categorize_as(SnapshotChangeCategory.METADATA, forward_only)

def _get_parent_versions_since(
self, parent: Snapshot, previous_parent_id: SnapshotId
) -> t.Optional[t.List[SnapshotDataVersion]]:
"""Returns all versions of the parent created after the previous parent, including the current
version. The previous parent itself is excluded.

Returns None if the previous parent is not found in the parent's version history.
"""
versions: t.List[SnapshotDataVersion] = []
for pv in reversed(parent.all_versions):
if pv.snapshot_id(parent.name) == previous_parent_id:
return versions
versions.append(pv)
return None

def _get_parent_categories_since(
self, parent: Snapshot, previous_parent_id: SnapshotId
) -> t.Optional[t.Set[SnapshotChangeCategory]]:
"""Returns the categories of the versions returned by `_get_parent_versions_since`."""
versions = self._get_parent_versions_since(parent, previous_parent_id)
if versions is None:
return None
return {t.cast(SnapshotChangeCategory, v.change_category) for v in versions}

@staticmethod
def _get_known_category(version: SnapshotDataVersion) -> SnapshotChangeCategory:
"""Returns the category of the version, or BREAKING if it isn't one that is assigned today.

A missing category or the deprecated FORWARD_ONLY category may still be found in older state. Both
changed data, so they are treated as breaking, and whether they were forward-only is read from the
stored snapshot like for any other breaking version.
"""
if version.change_category in (
SnapshotChangeCategory.BREAKING,
SnapshotChangeCategory.INDIRECT_BREAKING,
SnapshotChangeCategory.NON_BREAKING,
SnapshotChangeCategory.INDIRECT_NON_BREAKING,
SnapshotChangeCategory.METADATA,
):
return version.change_category
return SnapshotChangeCategory.BREAKING

def _was_applied_forward_only(self, parent: Snapshot, version: SnapshotDataVersion) -> bool:
"""Returns True if the given version of the parent was applied forward-only.

A past version doesn't record this, so it is read from the snapshot stored for that version.
If that snapshot can't be found, we can't tell, so the version is not considered forward-only.
"""
version_id = version.snapshot_id(parent.name)
if version_id == parent.snapshot_id:
return parent.is_forward_only
return self._context_diff.stored_parent_forward_only.get(version_id, False)

def _get_stored_direct_parent_changes(
self, snapshot: Snapshot
) -> t.Tuple[t.Set[SnapshotChangeCategory], t.Set[bool]]:
"""Returns the change categories and forward-only flags of direct parents that already exist in
state (they are not new in this plan) but changed since the previous version of the snapshot
was built.

A forward-only flag is only returned for a parent with a breaking version since then (True if
every such version was applied forward-only), or False for a parent whose history can't be traced
back that far. Metadata and non-breaking changes don't contribute a flag. The flags are added to
the forward-only vote of the new parents, which otherwise follows the same rule as when only new
parents changed.

For example, a parent was changed in a dev plan and is now combined with another parent
that is changed in the current plan. The change to the first parent is not visible to the
categorization of new parents, so it is recovered from the parent's version history.
"""
previous_snapshot = self._context_diff.modified_snapshots[snapshot.name][1]
previous_parent_ids = {p.name: p for p in previous_snapshot.parents}

categories: t.Set[SnapshotChangeCategory] = set()
forward_only: t.Set[bool] = set()
for p_id in snapshot.parents:
parent = self._context_diff.snapshots.get(p_id)
previous_parent_id = previous_parent_ids.get(p_id.name)
if not parent or self._is_new_snapshot(parent):
# Not stored, or categorized as part of this plan.
continue
if previous_parent_id is None or previous_parent_id == p_id:
# Not a parent of the previous version, or unchanged since then.
continue

parent_versions = self._get_parent_versions_since(parent, previous_parent_id)
if parent_versions is None:
# The history doesn't reach back to the previous parent (it only keeps the latest
# versions), so we can't rule out a breaking change. A rebuild is safer than silently
# reusing stale data.
categories.add(SnapshotChangeCategory.INDIRECT_BREAKING)
forward_only.add(False)
continue

version_categories = [(v, self._get_known_category(v)) for v in parent_versions]
categories.update(category for _, category in version_categories)

# Every breaking version of the parent since the previous child requires downstream history to
# be recomputed, unless it was applied forward-only. The parent's latest version being
# forward-only is not enough, an earlier version may not have been.
breaking_versions = [
v
for v, category in version_categories
if category
in (SnapshotChangeCategory.BREAKING, SnapshotChangeCategory.INDIRECT_BREAKING)
]
if breaking_versions:
forward_only.add(
all(self._was_applied_forward_only(parent, v) for v in breaking_versions)
)
return categories, forward_only

def _get_orphaned_indirect_change_category(
self, indirect_snapshot: Snapshot
) -> SnapshotChangeCategory:
Expand Down Expand Up @@ -790,17 +937,10 @@ def _get_orphaned_indirect_change_category(
# so we can skip it
continue

# Find the previous snapshot ID of the same parent in the historical chain
previous_parent_found = False
previous_parent_categories = set()
for pv in reversed(current_parent_snapshot.all_versions):
pv_snapshot_id = pv.snapshot_id(current_parent_snapshot.name)
if pv_snapshot_id == pevious_parent_snapshot_id:
previous_parent_found = True
break
previous_parent_categories.add(pv.change_category)

if not previous_parent_found:
previous_parent_categories = self._get_parent_categories_since(
current_parent_snapshot, pevious_parent_snapshot_id
)
if previous_parent_categories is None:
# The previous parent is not in the historical chain so falling back to INDIRECT_BREAKING
return SnapshotChangeCategory.INDIRECT_BREAKING

Expand Down
Loading