diff --git a/docs/concepts/plans.md b/docs/concepts/plans.md index c3e12652ee..593afcd30d 100644 --- a/docs/concepts/plans.md +++ b/docs/concepts/plans.md @@ -458,6 +458,24 @@ These examples demonstrate how to select which models to restate based on model sqlmesh plan --restate-model "db.model_a" --start "2024-01-01" --end "2024-01-10" ``` +### Restatement after moving a model's start earlier + +When you move an incremental model's `start` earlier, SQLMesh backfills that model for the newly added date range, as usual. Downstream models may already hold intervals for that same range, though, and those were computed while the upstream model was empty there. + +To keep that stale data from lingering, SQLMesh also restates the downstream models' existing intervals for the added range. The range is widened to each downstream model's interval unit. The plan output lists these models under the heading "Models restated because the start of an upstream model moved earlier". + +No restatement happens when the new `start` is later than before or when the change is [breaking](#breaking-change), because downstream models are rebuilt anyway. + +SQLMesh skips a model and prints a single warning in these cases. Models downstream of a skipped model are not restated through it, since its data doesn't change, but they can still be restated through another upstream model: + +- The model has `disable_restatement` set to `true` (this is the default for SCD Type 2 models). +- The model's kind only supports restating its full history (`INCREMENTAL_BY_UNIQUE_KEY`, `INCREMENTAL_UNMANAGED`, `INCREMENTAL_BY_PARTITION`, and the SCD Type 2 kinds). The warning prints the `sqlmesh plan --restate-model ...` command to run manually. +- The plan targets a development environment and the downstream model's table is one that `prod` uses or will use. These models are restated when the change is applied to `prod`. + +If you use `--select-model`, `--skip-backfill` or `--empty-backfill`, stale models that the plan does not backfill are not restated either. The same applies to the part of the added range before the plan's `--start`, when it is set later than the new model `start` outside a development environment. In both cases the warning prints the `--restate-model` command to run manually. + +Unlike a plan with `--restate-model`, these automatic restatements do not make the plan a restatement plan (`context.is_restatement` remains `False`). Built-in signals like `freshness()` automatically evaluate intervals restated due to upstream start extensions while still respecting signals for un-restated intervals. + ### Restating production vs development Restatement plans behave differently depending on if you're targeting the `prod` environment or a [development environment](./environments.md#how-to-use-environments). diff --git a/docs/reference/model_configuration.md b/docs/reference/model_configuration.md index 47dcaaced1..89e7e6faf9 100644 --- a/docs/reference/model_configuration.md +++ b/docs/reference/model_configuration.md @@ -20,7 +20,7 @@ Configuration options for SQLMesh model properties. Supported by all model kinds | `tags` | Arbitrary strings used to organize or classify a model | array[str] | N | | `cron` | The cron expression specifying how often the model should be refreshed. (Default: `@daily`) | str | N | | `interval_unit` | The temporal granularity of the model's data intervals. Supported values: `year`, `month`, `day`, `hour`, `half_hour`, `quarter_hour`, `five_minute`. (Default: inferred from `cron`) | str | N | -| `start` | The date/time that determines the earliest date interval that should be processed by a model. Can be a datetime string, epoch time in milliseconds, or a relative datetime such as `1 year ago`. (Default: `yesterday`) | str \| int | N | +| `start` | The date/time that determines the earliest date interval that should be processed by a model. Can be a datetime string, epoch time in milliseconds, or a relative datetime such as `1 year ago`. (Default: `yesterday`). Moving `start` earlier also restates the existing intervals of downstream models for the added range; see [restatement after moving a model's start earlier](../concepts/plans.md#restatement-after-moving-a-models-start-earlier). | str \| int | N | | `end` | The date/time that determines the latest date interval that should be processed by a model. Can be a datetime string, epoch time in milliseconds, or a relative datetime such as `1 year ago`. | str \| int | N | | `description` | Description of the model. Automatically registered in the SQL engine's table COMMENT field or equivalent (if supported by the engine). | str | N | | `column_descriptions` | A key-value mapping of column names to column comments that will be registered in the SQL engine's table COMMENT field (if supported by the engine). Specified as key-value pairs (`column_name = 'column comment'`). If present, [inline column comments](../concepts/models/overview.md#inline-column-comments) will not be registered in the SQL engine. | dict | N | diff --git a/sqlmesh/core/analytics/collector.py b/sqlmesh/core/analytics/collector.py index cfdb60aadd..c13c96f659 100644 --- a/sqlmesh/core/analytics/collector.py +++ b/sqlmesh/core/analytics/collector.py @@ -171,7 +171,7 @@ def on_plan_apply_start( "no_gaps": plan.no_gaps, "forward_only": plan.forward_only, "ensure_finalized_snapshots": plan.ensure_finalized_snapshots, - "has_restatements": bool(plan.restatements), + "has_restatements": plan.is_restatement, "directly_modified_count": len(plan.directly_modified_snapshots), "indirectly_modified_count": len( { diff --git a/sqlmesh/core/console.py b/sqlmesh/core/console.py index 8ae3be9083..36e3d32644 100644 --- a/sqlmesh/core/console.py +++ b/sqlmesh/core/console.py @@ -2064,12 +2064,31 @@ def _prompt_categorize( # We are computing dev previews, do not confuse the user by printing out something to do # with restatements. Dev previews are already highlighted in the backfill step pass - else: + + if not plan.restatements or ( + plan.start_extension_restatements and not plan.selected_models_to_restate + ): self.show_environment_difference_summary( plan.context_diff, no_diff=no_diff, ) + if plan.start_extension_restatements: + model_fqn_to_snapshot = {s.name: s for s in plan.snapshots.values()} + tree = Tree( + "[bold]Models restated because the start of an upstream model moved earlier:[/bold]\n" + "Their existing intervals for the added date range were computed without the new upstream data" + ) + for s_id in sorted(plan.start_extension_restatements): + tree.add( + model_fqn_to_snapshot[s_id.name].display_name( + plan.environment_naming_info, + default_catalog if self.verbosity < Verbosity.VERY_VERBOSE else None, + dialect=self.dialect, + ) + ) + self._print(tree) + if plan.context_diff.has_changes: self.show_model_difference_summary( plan.context_diff, diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index fd7f8ad071..9f9131d42f 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -158,7 +158,7 @@ ) from sqlmesh.core.snapshot import Node - from sqlmesh.core.snapshot.definition import Intervals + from sqlmesh.core.snapshot.definition import Intervals, Interval ModelOrSnapshot = t.Union[str, Model, Snapshot] NodeOrSnapshot = t.Union[str, Model, StandaloneAudit, Snapshot] @@ -285,6 +285,7 @@ def __init__( parent_intervals: t.Optional[Intervals] = None, variables: t.Optional[t.Dict[str, t.Any]] = None, blueprint_variables: t.Optional[t.Dict[str, t.Any]] = None, + restated_interval: t.Optional[Interval] = None, ): self.snapshots = snapshots self.deployability_index = deployability_index @@ -295,6 +296,7 @@ def __init__( self._blueprint_variables = blueprint_variables or {} self._is_restatement = is_restatement self._parent_intervals = parent_intervals + self._restated_interval = restated_interval @property def default_dialect(self) -> t.Optional[str]: @@ -327,6 +329,10 @@ def is_restatement(self) -> t.Optional[bool]: def parent_intervals(self) -> t.Optional[Intervals]: return self._parent_intervals + @property + def restated_interval(self) -> t.Optional[Interval]: + return self._restated_interval + def var(self, var_name: str, default: t.Optional[t.Any] = None) -> t.Optional[t.Any]: """Returns a variable value.""" return self._variables.get(var_name.lower(), default) @@ -348,6 +354,8 @@ def with_variables( self._default_dialect, self._default_catalog, self._is_restatement, + parent_intervals=self._parent_intervals, + restated_interval=self._restated_interval, variables=variables, blueprint_variables=blueprint_variables, ) diff --git a/sqlmesh/core/plan/builder.py b/sqlmesh/core/plan/builder.py index a6307a9ffd..ec53419503 100644 --- a/sqlmesh/core/plan/builder.py +++ b/sqlmesh/core/plan/builder.py @@ -8,6 +8,7 @@ from datetime import datetime +from sqlmesh.core import constants as c from sqlmesh.core.console import PlanBuilderConsole, get_console from sqlmesh.core.config import ( AutoCategorizationMode, @@ -31,11 +32,12 @@ ) from sqlmesh.core.snapshot import ( DeployabilityIndex, + Intervals, Snapshot, SnapshotChangeCategory, ) from sqlmesh.core.snapshot.categorizer import categorize_change -from sqlmesh.core.snapshot.definition import Interval, SnapshotId +from sqlmesh.core.snapshot.definition import Interval, SnapshotId, start_date from sqlmesh.utils import columns_to_types_all_known, random_id from sqlmesh.utils.dag import DAG from sqlmesh.utils.date import ( @@ -43,6 +45,8 @@ now, to_datetime, yesterday_ds, + to_ds, + to_ts, to_timestamp, time_like_to_str, is_relative, @@ -51,6 +55,8 @@ logger = logging.getLogger(__name__) +MS_PER_DAY = 24 * 60 * 60 * 1000 + class PlanBuilder: """Plan Builder constructs a Plan based on user choices for how they want to backfill, preview, etc. their changes. @@ -324,9 +330,10 @@ def build(self) -> Plan: else DeployabilityIndex.all_deployable() ) - restatements = self._build_restatements( + restatements, start_extension_restatements = self._build_restatements( dag, earliest_interval_start(self._context_diff.snapshots.values(), self.execution_time), + deployability_index, ) models_to_backfill = self._build_models_to_backfill(dag, restatements) @@ -363,7 +370,9 @@ def build(self) -> Plan: deployability_index=deployability_index, selected_models_to_restate=self._restate_models, restatements=restatements, - restate_all_snapshots=self._restate_all_snapshots, + start_extension_restatements=start_extension_restatements, + restate_all_snapshots=self._restate_all_snapshots + or self._restates_start_extension_in_prod(start_extension_restatements), start_override_per_model=self._start_override_per_model, end_override_per_model=end_override_per_model, selected_models_to_backfill=self._backfill_models, @@ -386,15 +395,27 @@ def _build_dag(self) -> DAG[SnapshotId]: return dag def _build_restatements( - self, dag: DAG[SnapshotId], earliest_interval_start: TimeLike - ) -> t.Dict[SnapshotId, Interval]: + self, + dag: DAG[SnapshotId], + earliest_interval_start: TimeLike, + deployability_index: DeployabilityIndex, + ) -> t.Tuple[t.Dict[SnapshotId, Interval], t.Set[SnapshotId]]: + """Builds the intervals to restate. + + Besides the models that were requested by a user (and their downstream models), this also covers + downstream models whose existing intervals are stale because the start of an upstream model moved earlier. + + Returns: + The restatements and the subset of the restated snapshots that are only restated because + of an upstream start extension. + """ restate_models = self._restate_models if restate_models == set(): # This is a warning but we print this as error since the Console is lacking API for warnings. self._console.log_error( "Provided restated models do not match any models. No models will be included in plan." ) - return {} + return {}, set() restatements: t.Dict[SnapshotId, Interval] = {} forward_only_preview_needed = self._forward_only_preview_needed @@ -417,8 +438,12 @@ def _build_restatements( } is_preview = True - if not restate_models: - return {} + start_extensions, cut_extensions = self._find_start_extensions() + if cut_extensions and not self._is_dev: + self._warn_about_cut_start_extensions(dag, cut_extensions, deployability_index) + if not restate_models and not start_extensions: + return {}, set() + restate_models = restate_models or set() start = self._start or earliest_interval_start end = self._end or now() @@ -428,24 +453,153 @@ def _build_restatements( if model_fqn not in self._model_fqn_to_snapshot: raise PlanError(f"Cannot restate model '{model_fqn}'. Model does not exist.") + backfill_scope = ( + self._build_models_to_backfill(dag, ()) if self._backfill_models is not None else None + ) + explicit_restatements: t.Set[SnapshotId] = set() + start_extension_restatements: t.Set[SnapshotId] = set() + # The range of an upstream start extension that reached each snapshot (restated or not). + start_extension_flow: t.Dict[SnapshotId, Interval] = {} + # Snapshots with stale intervals that are not restated, grouped by the reason. + outside_backfill_scope: t.List[Snapshot] = [] + shared_with_prod: t.List[Snapshot] = [] + full_history_only: t.List[Snapshot] = [] + stale_ranges: t.Dict[SnapshotId, Interval] = {} + start_cache: t.Dict[str, datetime] = {} + # Get restatement intervals for all restated snapshots and make sure that if an incremental snapshot expands it's # restatement range that it's downstream dependencies all expand their restatement ranges as well. for s_id in dag: snapshot = self._context_diff.snapshots[s_id] - if is_preview and snapshot.is_no_preview: - continue - # Since we are traversing the graph in topological order and the largest interval range is pushed down # the graph we just have to check our immediate parents in the graph and not the whole upstream graph. restating_parents = [ self._context_diff.snapshots[s] for s in snapshot.parents if s in restatements ] + extension_ranges = [ + r + for p in snapshot.parents + for r in (start_extensions.get(p), start_extension_flow.get(p)) + if r is not None + ] - if not restating_parents and snapshot.name not in restate_models: + is_explicit = not (is_preview and snapshot.is_no_preview) and ( + snapshot.name in restate_models + or any(p.snapshot_id in explicit_restatements for p in restating_parents) + ) + if not is_explicit and not extension_ranges: continue - if not forward_only_preview_needed: + extension_start = min((r[0] for r in extension_ranges), default=0) + extension_end = max((r[1] for r in extension_ranges), default=0) + aligned_extension = ( + _align_to_interval_unit(snapshot, extension_start, extension_end) + if extension_ranges + else (0, 0) + ) + floored_snapshot_start = to_timestamp( + snapshot.node.interval_unit.cron_floor( + start_date( + snapshot, + self._context_diff.snapshots, + start_cache, + relative_to=self.execution_time, + ) + ) + ) + is_passthrough = snapshot.is_embedded or ( + snapshot.is_view and not snapshot.is_materialized_view + ) + is_full_refresh = snapshot.is_full or snapshot.is_materialized_view + existing_intervals = _intervals(snapshot, deployability_index) + + if is_passthrough: + if extension_ranges: + start_extension_flow[s_id] = aligned_extension + if not is_explicit: + continue + elif is_full_refresh: + if is_explicit: + if extension_ranges: + start_extension_flow[s_id] = aligned_extension + else: + execution_ts = to_timestamp(self.execution_time) + upper_bound_ts = ( + execution_ts + if self._ignore_cron + else to_timestamp(snapshot.node.cron_floor(execution_ts)) + ) + if not snapshot.allow_partials: + upper_bound_ts = to_timestamp( + snapshot.node.interval_unit.cron_floor(upper_bound_ts) + ) + if self.end: + plan_end_ts = snapshot.inclusive_exclusive( + self.end, self.end, strict=False + )[1] + upper_bound_ts = min(upper_bound_ts, plan_end_ts) + + can_backfill_in_plan = floored_snapshot_start < upper_bound_ts + if not existing_intervals: + if extension_ranges and can_backfill_in_plan: + start_extension_flow[s_id] = aligned_extension + continue + if not can_backfill_in_plan: + outside_backfill_scope.append(snapshot) + stale_ranges[s_id] = aligned_extension + continue + if extension_ranges: + start_extension_flow[s_id] = aligned_extension + if self._is_dev and ( + not snapshot.is_paused or deployability_index.is_deployable(snapshot) + ): + shared_with_prod.append(snapshot) + start_extension_flow.pop(s_id, None) + continue + else: + if extension_ranges and ( + is_explicit or floored_snapshot_start < aligned_extension[1] + ): + # The extended range flows to the downstream models even if this snapshot isn't restated + # itself, as long as it holds (or will hold) data for that range. + start_extension_flow[s_id] = aligned_extension + + if not is_explicit: + if floored_snapshot_start >= aligned_extension[1]: + if _has_intervals_in_range( + snapshot, + aligned_extension[0], + aligned_extension[1], + deployability_index, + ): + outside_backfill_scope.append(snapshot) + stale_ranges[s_id] = aligned_extension + continue + + if self._is_dev and ( + not snapshot.is_paused or deployability_index.is_deployable(snapshot) + ): + # The physical table of this version is the one used in production. + if _has_intervals_in_range( + snapshot, + aligned_extension[0], + aligned_extension[1], + deployability_index, + ): + shared_with_prod.append(snapshot) + start_extension_flow.pop(s_id, None) + continue + # Only existing intervals can be stale. Anything else is backfilled as usual. + if not _has_intervals_in_range( + snapshot, + aligned_extension[0], + aligned_extension[1], + deployability_index, + ): + continue + + if not (forward_only_preview_needed and is_explicit): if self._is_dev and not snapshot.is_paused: self._console.log_warning( f"Cannot restate model '{snapshot.name}' because the current version is used in production. " @@ -457,44 +611,211 @@ def _build_restatements( f"Cannot restate model '{snapshot.name}'. " "Restatement is disabled for this model to prevent possible data loss. " "If you want to restate this model, change the model's `disable_restatement` setting to `false`." + + ( + "" + if is_explicit + else " Models downstream of it are not restated through it, since its data doesn't change." + ) ) + if not is_explicit: + start_extension_flow.pop(s_id, None) continue elif snapshot.is_seed: logger.info("Skipping restatement for model '%s'", snapshot.name) continue + if not is_explicit: + if snapshot.full_history_restatement_only and snapshot.is_incremental: + # Restating these models rebuilds their whole history, which is too expensive + # to do just because the start of an upstream model moved by a few intervals. + full_history_only.append(snapshot) + stale_ranges[s_id] = start_extension_flow.pop(s_id, aligned_extension) + continue + if ( + self._skip_backfill + or self._empty_backfill + or (backfill_scope is not None and snapshot.name not in backfill_scope) + ): + outside_backfill_scope.append(snapshot) + stale_ranges[s_id] = start_extension_flow.get(s_id, aligned_extension) + continue + possible_intervals = { restatements[p.snapshot_id] for p in restating_parents if p.is_incremental } - removal_start = ( - self._forward_only_preview_start(snapshot, start, end) if is_preview else start - ) - possible_intervals.add( - snapshot.get_removal_interval( - removal_start, - end, - self._execution_time, - strict=False, - is_preview=is_preview, + if is_explicit: + removal_start = ( + self._forward_only_preview_start(snapshot, start, end) if is_preview else start ) - ) - snapshot_start = min(i[0] for i in possible_intervals) - snapshot_end = max(i[1] for i in possible_intervals) + possible_intervals.add( + snapshot.get_removal_interval( + removal_start, + end, + self._execution_time, + strict=False, + is_preview=is_preview, + ) + ) + if extension_ranges: + ext_removal = snapshot.get_removal_interval( + extension_start, extension_end, self._execution_time, strict=False + ) + if is_full_refresh and existing_intervals and not (is_explicit and is_preview): + ext_removal = ( + min(ext_removal[0], existing_intervals[0][0]), + max(ext_removal[1], existing_intervals[-1][1]), + ) + possible_intervals.add(ext_removal) + if snapshot.is_incremental: + start_extension_flow[s_id] = _align_to_interval_unit(snapshot, *ext_removal) # We may be tasked with restating a time range smaller than the target snapshot interval unit # For example, restating an hour of Hourly Model A, which has a downstream dependency of Daily Model B # we need to ensure the whole affected day in Model B is restated - floored_snapshot_start = snapshot.node.interval_unit.cron_floor(snapshot_start) - floored_snapshot_end = snapshot.node.interval_unit.cron_floor(snapshot_end) - if to_timestamp(floored_snapshot_end) < snapshot_end: - snapshot_start = to_timestamp(floored_snapshot_start) - snapshot_end = to_timestamp( - snapshot.node.interval_unit.cron_next(floored_snapshot_end) - ) + restatements[s_id] = _align_to_interval_unit( + snapshot, + min(i[0] for i in possible_intervals), + max(i[1] for i in possible_intervals), + ) + if is_explicit: + explicit_restatements.add(s_id) + else: + start_extension_restatements.add(s_id) + + self._warn_about_stale_start_extensions( + outside_backfill_scope, + shared_with_prod, + full_history_only, + stale_ranges, + ) + + return restatements, start_extension_restatements + + def _find_start_extensions( + self, + ) -> t.Tuple[t.Dict[SnapshotId, Interval], t.Dict[SnapshotId, Interval]]: + """Finds the incremental models whose start moved earlier in this plan. + + Returns: + A mapping from the new snapshot ID to the [new start, old start) range that was added to its history, + and a mapping to the part of that range that was cut off because it's before the plan start. + Downstream models that already have intervals within this range computed them from incomplete data. + Whether those intervals are still around after the plan is applied (as opposed to being + rebuilt because of a breaking change) is decided when walking the downstream models. + """ + modified = self._context_diff.modified_snapshots + snapshots = self._context_diff.snapshots + # Unmodified snapshots are the same before and after the plan; this lets inferred starts + # be resolved through the parents that were in place before the plan. + previous_snapshots = {s_id: s for s_id, s in snapshots.items() if s.name not in modified} + previous_snapshots.update({old.snapshot_id: old for _, old in modified.values()}) + new_cache: t.Dict[str, datetime] = {} + old_cache: t.Dict[str, datetime] = {} + # Skip changes that don't move the start by a whole interval, e.g. an adjustment of the time of day. + plan_start = to_timestamp(self.start) if self.start else None + + extensions: t.Dict[SnapshotId, Interval] = {} + cut: t.Dict[SnapshotId, Interval] = {} + resolvable_cache: t.Dict[SnapshotId, bool] = {} + for new, old in modified.values(): + if not (new.is_model and old.is_model and new.is_incremental): + continue + if not _is_start_resolvable(old, previous_snapshots, resolvable_cache): + # The previous start would be guessed wrongly if some of its parents are unknown. + continue + interval_unit = new.node.interval_unit + new_start = start_date(new, snapshots, new_cache, relative_to=self.execution_time) + old_start = start_date( + old, previous_snapshots, old_cache, relative_to=self.execution_time + ) + start_ts = to_timestamp(interval_unit.cron_floor(new_start)) + end_ts = to_timestamp(interval_unit.cron_floor(old_start)) + if plan_start is not None and start_ts < min(plan_start, end_ts): + cut[new.snapshot_id] = (start_ts, min(plan_start, end_ts)) + if plan_start is not None: + start_ts = max(start_ts, plan_start) + if start_ts < end_ts: + extensions[new.snapshot_id] = (start_ts, end_ts) + return extensions, cut + + def _restates_start_extension_in_prod(self, snapshot_ids: t.Set[SnapshotId]) -> bool: + """Whether intervals of other versions of the snapshots also need to be cleared from state. + + This mirrors what is done for user requested production restatements. + """ + return not self._is_dev and any( + self._context_diff.snapshots[s_id].virtual_environment_mode.is_full + for s_id in snapshot_ids + ) - restatements[s_id] = (snapshot_start, snapshot_end) + def _warn_about_cut_start_extensions( + self, + dag: DAG[SnapshotId], + cut_extensions: t.Dict[SnapshotId, Interval], + deployability_index: DeployabilityIndex, + ) -> None: + """Warns about the part of an extension before the plan start, which is never detected again.""" + affected: t.Dict[SnapshotId, Snapshot] = {} + for s_id, (start, end) in cut_extensions.items(): + for downstream_id in dag.downstream(s_id): + downstream = self._context_diff.snapshots[downstream_id] + if _has_intervals_in_range(downstream, start, end, deployability_index): + affected[downstream_id] = downstream + if not affected: + return + start = min(r[0] for r in cut_extensions.values()) + end = max(r[1] for r in cut_extensions.values()) + restate_args = " ".join(f"--restate-model '{s.name}'" for s in affected.values()) + self._console.log_warning( + "The start of an upstream model moved earlier, but the plan start is after the beginning " + "of the added range, so downstream models with intervals in the cut off part were not " + f"restated: {', '.join(s.name for s in affected.values())}. " + f"To refresh them, run: sqlmesh plan {restate_args} {_range_args(start, end)}" + ) - return restatements + def _warn_about_stale_start_extensions( + self, + outside_backfill_scope: t.List[Snapshot], + shared_with_prod: t.List[Snapshot], + full_history_only: t.List[Snapshot], + flow: t.Dict[SnapshotId, Interval], + ) -> None: + intro = ( + "The start of an upstream model moved earlier, so some existing intervals of these " + "downstream models were computed without the added data" + ) + if shared_with_prod: + names = ", ".join(s.name for s in shared_with_prod) + self._console.log_warning( + f"{intro}: {names}. They use tables that production uses or will use, so this development " + "plan doesn't restate them. They are restated, along with their downstream models, when this change is applied to production." + ) + for snapshots, reason in ( + ( + outside_backfill_scope, + "they were not restated since the plan doesn't backfill them", + ), + ( + full_history_only, + "they only support restating their full history, so they were not restated automatically " + "(models downstream of them are not restated through them)", + ), + ): + if not snapshots: + continue + start = min(flow[s.snapshot_id][0] for s in snapshots) + end = max(flow[s.snapshot_id][1] for s in snapshots) + range_args = _range_args(start, end) + restate_args = " ".join(f"--restate-model '{s.name}'" for s in snapshots) + environment = ( + f" {self._context_diff.environment}" + if self._context_diff.environment != c.PROD + else "" + ) + self._console.log_warning( + f"{intro}, but {reason}: {', '.join(s.name for s in snapshots)}. " + f"To refresh them, run: sqlmesh plan{environment} {restate_args} {range_args}" + ) def _forward_only_preview_start( self, snapshot: Snapshot, default_start: TimeLike, end: TimeLike @@ -959,3 +1280,61 @@ def _modified_and_added_snapshots(self) -> t.List[Snapshot]: if snapshot.name in self._context_diff.modified_snapshots or snapshot.snapshot_id in self._context_diff.added ] + + +def _range_args(start: int, end: int) -> str: + """Formats the [start, end) range as command line arguments, where dates are inclusive.""" + if start % MS_PER_DAY == 0 and end % MS_PER_DAY == 0: + return f"--start {to_ds(start)} --end {to_ds(end - 1)}" + return f"--start '{to_ts(start)}' --end '{to_ts(end)}'" + + +def _intervals(snapshot: Snapshot, deployability_index: DeployabilityIndex) -> Intervals: + return ( + snapshot.intervals + if deployability_index.is_deployable(snapshot) + else snapshot.dev_intervals + ) + + +def _has_intervals_in_range( + snapshot: Snapshot, start: int, end: int, deployability_index: DeployabilityIndex +) -> bool: + intervals = _intervals(snapshot, deployability_index) + return any(i_start < end and start < i_end for i_start, i_end in intervals) + + +def _is_start_resolvable( + snapshot: Snapshot, + snapshots: t.Dict[SnapshotId, Snapshot], + cache: t.Optional[t.Dict[SnapshotId, bool]] = None, +) -> bool: + """Whether the effective start can be derived from the given snapshots (it may be inferred from parents).""" + if cache is not None and snapshot.snapshot_id in cache: + return cache[snapshot.snapshot_id] + + if snapshot.node.start or not snapshot.parents: + res = True + else: + res = all( + p in snapshots and _is_start_resolvable(snapshots[p], snapshots, cache) + for p in snapshot.parents + ) + + if cache is not None: + cache[snapshot.snapshot_id] = res + + return res + + +def _align_to_interval_unit(snapshot: Snapshot, start: int, end: int) -> Interval: + """Widens the range if it starts or ends in the middle of one of the snapshot's intervals.""" + interval_unit = snapshot.node.interval_unit + floored_start = to_timestamp(interval_unit.cron_floor(start)) + floored_end = interval_unit.cron_floor(end) + aligned_end = ( + to_timestamp(interval_unit.cron_next(floored_end)) + if to_timestamp(floored_end) < end + else end + ) + return floored_start, aligned_end diff --git a/sqlmesh/core/plan/common.py b/sqlmesh/core/plan/common.py index bece17639c..31b598bcdd 100644 --- a/sqlmesh/core/plan/common.py +++ b/sqlmesh/core/plan/common.py @@ -137,6 +137,11 @@ def identify_restatement_intervals_across_snapshot_versions( snapshot=affected_snapshot.id_and_version, interval=interval ) snapshot_intervals_to_clear[affected_snapshot.snapshot_id] = clear_request + else: + clear_request.interval = ( + min(clear_request.interval[0], interval[0]), + max(clear_request.interval[1], interval[1]), + ) clear_request.environment_names |= set([env.name]) diff --git a/sqlmesh/core/plan/definition.py b/sqlmesh/core/plan/definition.py index 866299eff8..e4370d77b3 100644 --- a/sqlmesh/core/plan/definition.py +++ b/sqlmesh/core/plan/definition.py @@ -67,6 +67,11 @@ class Plan(PydanticModel, frozen=True): Note that dev previews are also considered restatements, so :selected_models_to_restate can be empty while :restatements is still populated with dev previews """ + start_extension_restatements: t.Set[SnapshotId] = set() + """ + Models that are part of :restatements only because the start of one of their upstream models moved + earlier, i.e. the restatement wasn't requested by a user. + """ restate_all_snapshots: bool """Whether or not to clear intervals from state for other versions of the models listed in :restatements""" @@ -263,15 +268,25 @@ def is_selected_for_backfill(self, model_fqn: str) -> bool: return self.models_to_backfill is None or model_fqn in self.models_to_backfill def to_evaluatable(self) -> EvaluatablePlan: + if not self.context_diff.is_new_environment: + previous_environment_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = { + s.name: s.table_info for s in self.context_diff.environment_snapshots + } + elif self.previous_plan_id is None: + previous_environment_snapshots = {} + else: + previous_environment_snapshots = None return EvaluatablePlan( start=self.start, end=self.end, new_snapshots=self.new_snapshots, environment=self.environment, + previous_environment_snapshots=previous_environment_snapshots, no_gaps=self.no_gaps, skip_backfill=self.skip_backfill, empty_backfill=self.empty_backfill, restatements={s.name: i for s, i in self.restatements.items()}, + start_extension_restatements={s_id.name for s_id in self.start_extension_restatements}, restate_all_snapshots=self.restate_all_snapshots, is_dev=self.is_dev, allow_destructive_models=self.allow_destructive_models, @@ -313,10 +328,13 @@ class EvaluatablePlan(PydanticModel): end: TimeLike new_snapshots: t.List[Snapshot] environment: Environment + previous_environment_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = None no_gaps: bool skip_backfill: bool empty_backfill: bool restatements: t.Dict[str, Interval] + start_extension_restatements: t.Set[str] = set() + """Names of the models in :restatements that are only there because an upstream start moved earlier.""" restate_all_snapshots: bool is_dev: bool allow_destructive_models: t.Set[str] @@ -339,6 +357,11 @@ class EvaluatablePlan(PydanticModel): user_provided_flags: t.Optional[t.Dict[str, UserProvidedFlags]] = None selected_models: t.Optional[t.Set[str]] = None + @property + def is_restatement(self) -> bool: + """Whether the plan restates data on request (or previews), as opposed to just refreshing extended ranges.""" + return bool(set(self.restatements) - self.start_extension_restatements) + def is_selected_for_backfill(self, model_fqn: str) -> bool: return self.models_to_backfill is None or model_fqn in self.models_to_backfill diff --git a/sqlmesh/core/plan/evaluator.py b/sqlmesh/core/plan/evaluator.py index 71e5285d63..1a3346eb67 100644 --- a/sqlmesh/core/plan/evaluator.py +++ b/sqlmesh/core/plan/evaluator.py @@ -261,7 +261,8 @@ def visit_backfill_stage(self, stage: stages.BackfillStage, plan: EvaluatablePla allow_additive_snapshots=plan.allow_additive_models, selected_snapshot_ids=stage.selected_snapshot_ids, selected_models=plan.selected_models, - is_restatement=bool(plan.restatements), + is_restatement=plan.is_restatement, + restatements=plan.restatements, ) if errors: raise PlanError("Plan application failed.") @@ -321,19 +322,65 @@ def visit_restatement_stage( str, t.Tuple[SnapshotTableInfo, SnapshotTableInfo] ] = {} # tuple of (restated_snapshot, current_prod_snapshot) - if deployed_env := self.state_sync.get_environment(plan.environment.name): - promoted_snapshots_by_name = {s.name: s for s in deployed_env.snapshots} - - for name in plan.restatements: - snapshot = stage.all_snapshots[name] - version = snapshot.table_info.version - if ( - prod_snapshot := promoted_snapshots_by_name.get(name) - ) and prod_snapshot.version != version: - deployed_during_restatement[name] = ( - snapshot.table_info, - prod_snapshot.table_info, - ) + current_plan_id = getattr(plan, "plan_id", None) + if not isinstance(current_plan_id, str): + current_plan_id = plan.environment.plan_id + + if ( + (deployed_env := self.state_sync.get_environment(plan.environment.name)) + and not deployed_env.expired + and deployed_env.plan_id != current_plan_id + ): + ensure_finalized = getattr(plan, "ensure_finalized_snapshots", False) is True + if ensure_finalized: + if deployed_env.plan_id == plan.environment.previous_plan_id: + snapshot_maps = [ + {s.name: s for s in deployed_env.finalized_or_current_snapshots} + ] + else: + snapshot_maps = [ + {s.name: s for s in deployed_env.snapshots}, + {s.name: s for s in deployed_env.finalized_or_current_snapshots}, + ] + else: + snapshot_maps = [{s.name: s for s in deployed_env.snapshots}] + + if stage.expected_previous_snapshots is not None: + for name in plan.restatements: + snapshot = stage.all_snapshots[name] + expected_prev = stage.expected_previous_snapshots.get(name) + + for promoted_snapshots_by_name in snapshot_maps: + prod_snapshot = promoted_snapshots_by_name.get(name) + if expected_prev is not None: + if ( + prod_snapshot is None + or prod_snapshot.version != expected_prev.version + ): + deployed_during_restatement[name] = ( + snapshot.table_info, + prod_snapshot.table_info if prod_snapshot else expected_prev, + ) + break + elif prod_snapshot is not None: + deployed_during_restatement[name] = ( + snapshot.table_info, + prod_snapshot.table_info, + ) + break + else: + for name in plan.restatements: + snapshot = stage.all_snapshots[name] + version = snapshot.table_info.version + for promoted_snapshots_by_name in snapshot_maps: + if ( + prod_snapshot := promoted_snapshots_by_name.get(name) + ) and prod_snapshot.version != version: + deployed_during_restatement[name] = ( + snapshot.table_info, + prod_snapshot.table_info, + ) + break # we need to *not* clear the intervals on the snapshots where new versions were deployed while the restatement was running in order to prevent # subsequent plans from having unexpected intervals to backfill. diff --git a/sqlmesh/core/plan/explainer.py b/sqlmesh/core/plan/explainer.py index 6e3fb8f9c3..ebfc4dfcac 100644 --- a/sqlmesh/core/plan/explainer.py +++ b/sqlmesh/core/plan/explainer.py @@ -3,7 +3,7 @@ import abc import typing as t import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from collections import defaultdict from rich.console import Console as RichConsole @@ -82,7 +82,9 @@ class ExplainableRestatementStage(stages.RestatementStage): of what might happen when they ask for the plan to be explained """ - snapshot_intervals_to_clear: t.Dict[str, t.List[SnapshotIntervalClearRequest]] + snapshot_intervals_to_clear: t.Dict[str, t.List[SnapshotIntervalClearRequest]] = field( + default_factory=dict + ) """Which snapshots from other environments would have intervals cleared as part of restatement, grouped by name.""" @classmethod @@ -107,6 +109,7 @@ def from_restatement_stage( return cls( snapshot_intervals_to_clear=snapshot_intervals_to_clear, all_snapshots=stage.all_snapshots, + expected_previous_snapshots=stage.expected_previous_snapshots, ) diff --git a/sqlmesh/core/plan/stages.py b/sqlmesh/core/plan/stages.py index 729e1705b4..8e5fd17fa7 100644 --- a/sqlmesh/core/plan/stages.py +++ b/sqlmesh/core/plan/stages.py @@ -110,9 +110,11 @@ class RestatementStage: Args: all_snapshots: All snapshots in the plan by name. Note that this does not include the snapshots from other environments that will get their intervals cleared, it's included here as an optimization to prevent having to re-fetch the current plan's snapshots + expected_previous_snapshots: The baseline snapshots by name in the target environment when the plan was created. """ all_snapshots: t.Dict[str, Snapshot] + expected_previous_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = None @dataclass @@ -364,7 +366,9 @@ def build(self, plan: EvaluatablePlan) -> t.List[PlanStage]: # in addition, when we update intervals in state, we only clear intervals from dev snapshots to force dev models to be backfilled based on the new prod data. # we can leave prod intervals alone because by the time this plan finishes, the intervals in state have not actually changed, since restatement replaces # data for existing intervals and does not produce new ones - restatement_stage = self._get_restatement_stage(plan, snapshots_by_name) + restatement_stage = self._get_restatement_stage( + plan, snapshots_by_name, existing_environment + ) if restatement_stage: stages.append(restatement_stage) @@ -460,7 +464,10 @@ def _get_after_all_stage( ) def _get_restatement_stage( - self, plan: EvaluatablePlan, snapshots_by_name: t.Dict[str, Snapshot] + self, + plan: EvaluatablePlan, + snapshots_by_name: t.Dict[str, Snapshot], + existing_environment: t.Optional[Environment] = None, ) -> t.Optional[RestatementStage]: if plan.restate_all_snapshots: if plan.is_dev: @@ -469,8 +476,40 @@ def _get_restatement_stage( ) if plan.restatements: + expected_previous_snapshots: t.Optional[t.Dict[str, SnapshotTableInfo]] = None + previous_env_snapshots = getattr(plan, "previous_environment_snapshots", None) + if isinstance(previous_env_snapshots, dict): + expected_previous_snapshots = previous_env_snapshots + else: + ensure_finalized = getattr(plan, "ensure_finalized_snapshots", False) is True + if existing_environment is None: + existing_environment = self.state_reader.get_environment( + plan.environment.name + ) + if isinstance(existing_environment, Environment) and ( + plan.environment.previous_finalized_snapshots is None + or existing_environment.plan_id == plan.environment.previous_plan_id + ): + if existing_environment.expired: + expected_previous_snapshots = {} + else: + baseline = ( + existing_environment.finalized_or_current_snapshots + if ensure_finalized + else existing_environment.snapshots + ) + expected_previous_snapshots = {s.name: s.table_info for s in baseline} + elif plan.environment.previous_finalized_snapshots is not None: + expected_previous_snapshots = { + s.name: s.table_info + for s in plan.environment.previous_finalized_snapshots + } + elif existing_environment is None: + expected_previous_snapshots = {} + return RestatementStage( all_snapshots=snapshots_by_name, + expected_previous_snapshots=expected_previous_snapshots, ) return None diff --git a/sqlmesh/core/scheduler.py b/sqlmesh/core/scheduler.py index 7345d92888..bef85e63fb 100644 --- a/sqlmesh/core/scheduler.py +++ b/sqlmesh/core/scheduler.py @@ -343,6 +343,9 @@ def batch_intervals( environment_naming_info: EnvironmentNamingInfo, dag: t.Optional[DAG[SnapshotId]] = None, is_restatement: bool = False, + restatements: t.Optional[ + t.Union[t.Dict[SnapshotId, Interval], t.Dict[str, Interval]] + ] = None, ) -> t.Dict[Snapshot, Intervals]: dag = dag or snapshots_to_dag(merged_intervals) @@ -377,6 +380,10 @@ def batch_intervals( parent_intervals.extend(snapshot_batches[parent_snapshot]) + restated_interval = None + if restatements: + restated_interval = restatements.get(snapshot_id) or restatements.get(snapshot.name) # type: ignore[call-overload] + context = ExecutionContext( adapter, self.snapshots_by_name, @@ -385,6 +392,7 @@ def batch_intervals( default_catalog=self.default_catalog, is_restatement=is_restatement, parent_intervals=parent_intervals, + restated_interval=restated_interval, ) intervals = self._check_ready_intervals( @@ -441,6 +449,9 @@ def run_merged_intervals( audit_only: bool = False, auto_restatement_triggers: t.Dict[SnapshotId, t.List[SnapshotId]] = {}, is_restatement: bool = False, + restatements: t.Optional[ + t.Union[t.Dict[SnapshotId, Interval], t.Dict[str, Interval]] + ] = None, ) -> t.Tuple[t.List[NodeExecutionFailedError[SchedulingUnit]], t.List[SchedulingUnit]]: """Runs precomputed batches of missing intervals. @@ -480,6 +491,7 @@ def run_merged_intervals( environment_naming_info, dag=snapshot_dag, is_restatement=is_restatement, + restatements=restatements, ) self.console.start_evaluation_progress( batched_intervals, diff --git a/sqlmesh/core/signal.py b/sqlmesh/core/signal.py index 554dd60a39..5310b7a6c5 100644 --- a/sqlmesh/core/signal.py +++ b/sqlmesh/core/signal.py @@ -2,13 +2,13 @@ import typing as t from sqlmesh.utils import UniqueKeyDict, registry_decorator +from sqlmesh.utils.date import to_timestamp from sqlmesh.utils.errors import MissingSourceError if t.TYPE_CHECKING: from sqlmesh.core.context import ExecutionContext from sqlmesh.core.snapshot.definition import Snapshot from sqlmesh.utils.date import DatetimeRanges - from sqlmesh.core.snapshot.definition import DeployabilityIndex class signal(registry_decorator): @@ -47,7 +47,7 @@ def freshness( batch: DatetimeRanges, snapshot: Snapshot, context: ExecutionContext, -) -> bool: +) -> t.Union[bool, DatetimeRanges]: """ Implements model freshness as a signal, i.e it considers this model to be fresh if: - Any upstream SQLMesh model has available intervals to compute i.e is fresh @@ -57,6 +57,8 @@ def freshness( if context.is_restatement or not adapter.SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS: return True + from sqlmesh.core.snapshot.definition import DeployabilityIndex + deployability_index = context.deployability_index or DeployabilityIndex.all_deployable() last_altered_ts = ( @@ -65,34 +67,46 @@ def freshness( else snapshot.dev_last_altered_ts ) + ready_intervals: t.Union[bool, DatetimeRanges] if not last_altered_ts: - return True - - parent_snapshots = {context.snapshots[p.name] for p in snapshot.parents} - - upstream_parent_snapshots = {p for p in parent_snapshots if not p.is_external} - external_parents = snapshot.node.depends_on - {p.name for p in upstream_parent_snapshots} - - if context.parent_intervals: - # At least one upstream sqlmesh model has intervals to compute (i.e is fresh), - # so the current model is considered fresh too - return True - - if external_parents: - external_last_altered_timestamps = adapter.get_table_last_modified_ts( - list(external_parents) - ) - - if len(external_last_altered_timestamps) != len(external_parents): - raise MissingSourceError( - f"Expected {len(external_parents)} sources to be present, but got {len(external_last_altered_timestamps)}." + ready_intervals = True + else: + parent_snapshots = {context.snapshots[p.name] for p in snapshot.parents} + upstream_parent_snapshots = {p for p in parent_snapshots if not p.is_external} + external_parents = snapshot.node.depends_on - {p.name for p in upstream_parent_snapshots} + + if context.parent_intervals: + # At least one upstream sqlmesh model has intervals to compute (i.e is fresh), + # so the current model is considered fresh too + ready_intervals = True + elif external_parents: + external_last_altered_timestamps = adapter.get_table_last_modified_ts( + list(external_parents) ) - # Finding new data means that the upstream depedencies have been altered - # since the last time the model was evaluated - return any( - external_last_altered_ts > last_altered_ts - for external_last_altered_ts in external_last_altered_timestamps - ) + if len(external_last_altered_timestamps) != len(external_parents): + raise MissingSourceError( + f"Expected {len(external_parents)} sources to be present, but got {len(external_last_altered_timestamps)}." + ) - return False + # Finding new data means that the upstream depedencies have been altered + # since the last time the model was evaluated + ready_intervals = any( + external_last_altered_ts > last_altered_ts + for external_last_altered_ts in external_last_altered_timestamps + ) + else: + ready_intervals = False + + if context.restated_interval and ready_intervals is not True: + r_start, r_end = context.restated_interval + restated_batch = [ + i for i in batch if to_timestamp(i[0]) < r_end and r_start < to_timestamp(i[1]) + ] + if restated_batch: + if isinstance(ready_intervals, list): + combined_set = set(ready_intervals) | set(restated_batch) + return [i for i in batch if i in combined_set] + return restated_batch + + return ready_intervals diff --git a/sqlmesh/integrations/github/cicd/controller.py b/sqlmesh/integrations/github/cicd/controller.py index 58b0664c2d..9f9e288c97 100644 --- a/sqlmesh/integrations/github/cicd/controller.py +++ b/sqlmesh/integrations/github/cicd/controller.py @@ -528,7 +528,9 @@ def get_plan_summary(self, plan: Plan) -> str: try: # Clear out any output that might exist from prior steps self._console.consume_captured_output() - if plan.restatements: + if plan.restatements and not ( + plan.start_extension_restatements and not plan.selected_models_to_restate + ): self._console._print("\n**Restating models**\n") else: self._console.show_environment_difference_summary( diff --git a/tests/core/analytics/test_collector.py b/tests/core/analytics/test_collector.py index e87704e694..a574357dec 100644 --- a/tests/core/analytics/test_collector.py +++ b/tests/core/analytics/test_collector.py @@ -375,3 +375,25 @@ def test_on_migration(collector: AnalyticsCollector, mocker: MockerFixture): ), ] ) + + +def test_on_plan_apply_start_ignores_start_extension_restatements( + collector: AnalyticsCollector, init_and_plan_context: t.Callable +): + _, plan = init_and_plan_context("examples/sushi") + evaluatable = plan.to_evaluatable() + + def has_restatements(restatements: t.Dict, extensions: t.Set[str]) -> str: + updated = evaluatable.copy( + update={"restatements": restatements, "start_extension_restatements": extensions} + ) + collector._dispatcher.add_event.reset_mock() # type: ignore + collector.on_plan_apply_start( + plan=updated, engine_type="duckdb", state_sync_type="duckdb", scheduler_type="builtin" + ) + collector.flush() + event = collector._dispatcher.add_event.call_args.args[0]["event"] # type: ignore + return json.loads(event)["has_restatements"] + + assert has_restatements({"a": (0, 1)}, set()) is True + assert has_restatements({"a": (0, 1)}, {"a"}) is False diff --git a/tests/core/integration/test_restatement.py b/tests/core/integration/test_restatement.py index 245aa9d670..566907ec99 100644 --- a/tests/core/integration/test_restatement.py +++ b/tests/core/integration/test_restatement.py @@ -23,6 +23,7 @@ GatewayConfig, ModelDefaultsConfig, DuckDBConnectionConfig, + PlanConfig, ) from sqlmesh.core.context import Context from sqlmesh.core.model import ( @@ -39,7 +40,8 @@ from sqlmesh.utils.errors import ( ConflictingPlanError, ) -from tests.core.integration.utils import add_projection_to_model +from sqlmesh.core.snapshot import SnapshotChangeCategory +from tests.core.integration.utils import add_projection_to_model, plan_choice pytestmark = pytest.mark.slow @@ -1970,3 +1972,506 @@ def test_restatement_plan_outside_parent_date_range(init_and_plan_context: t.Cal ] context.apply(plan) + + +# --- Start extension (#4837): moving a model's `start` earlier must refresh downstream intervals --- + + +_EXTENSION_NOW = "2023-01-04 12:00:00 UTC" + + +def _extension_model( + name: str, + source: str, + start: str, + *, + kind: str = "INCREMENTAL_BY_TIME_RANGE (time_column ds)", + cron: str = "@daily", + extra_columns: str = "", + query: t.Optional[str] = None, +) -> str: + query = query or f"SELECT ds::DATE AS ds{extra_columns} FROM {source}" + if "FULL" not in kind: + query += " WHERE ds BETWEEN @start_ds AND @end_ds" + return f"MODEL (name s.{name}, kind {kind}, start '{start}', cron '{cron}');\n{query}" + + +class _ExtensionProject: + """A scratch project of models reading from `s.src` that can be rewritten between plans.""" + + def __init__(self, tmp_path: Path, models: t.Dict[str, str]) -> None: + self.models_dir = tmp_path / "models" + self.models_dir.mkdir() + self.write(models) + config = Config( + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + default_connection=DuckDBConnectionConfig(), + ) + self.context = Context(paths=[tmp_path], config=config) + adapter = self.context.engine_adapter + adapter.create_schema("s") + adapter.execute( + "CREATE TABLE s.src AS SELECT range::DATE AS ds FROM " + "range(DATE '2022-12-25', DATE '2023-01-06', INTERVAL 1 DAY)" + ) + + def write(self, models: t.Dict[str, str]) -> None: + for name, definition in models.items(): + (self.models_dir / f"{name}.sql").write_text(definition) + + def update(self, models: t.Dict[str, str]) -> None: + self.write(models) + self.context.load() + + def ds(self, table: str) -> t.List[str]: + rows = self.context.engine_adapter.fetchall(f"SELECT ds FROM {table} ORDER BY ds") + return [str(r[0]) for r in rows] + + +@pytest.fixture +def capture_console() -> t.Iterator[CaptureTerminalConsole]: + original = get_console() + console = CaptureTerminalConsole() + set_console(console) + yield console + set_console(original) + + +def _chain(a_start: str, b_start: str = "2020-01-01", c_start: str = "2023-01-02", **b_kwargs): + return { + "a": _extension_model("a", "s.src", a_start), + "b": _extension_model("b", "s.a", b_start, **b_kwargs), + "c": _extension_model("c", "s.b", c_start), + } + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +@pytest.mark.parametrize( + "c_start_before, c_start_after", [("2023-01-02", "2023-01-01"), ("2020-01-01", "2020-01-01")] +) +def test_start_extension_refreshes_downstream_intervals( + tmp_path: Path, c_start_before: str, c_start_after: str +): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start=c_start_before)) + project.context.plan(auto_apply=True, no_prompts=True) + for table in ("a", "b", "c"): + assert project.ds(f"s.{table}") == ["2023-01-02", "2023-01-03"] + + project.update(_chain("2023-01-01", c_start=c_start_after)) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + # b (and c, if its history already covers the range) are restated for the added range only. + restated = {s_id.name: interval for s_id, interval in plan.restatements.items()} + day = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + assert restated['"memory"."s"."b"'] == day + for table in ("a", "b", "c"): + assert project.ds(f"s.{table}") == ["2023-01-01", "2023-01-02", "2023-01-03"], table + + # Nothing is left to do. + assert not project.context.plan_builder().build().requires_backfill + + +@time_machine.travel("2023-01-02 12:00:00 UTC", tick=False) +def test_start_extension_restates_whole_coarser_downstream_interval(tmp_path: Path): + hourly_a = ( + "MODEL (name s.a, kind INCREMENTAL_BY_TIME_RANGE (time_column ts), start '{start}', " + "cron '@hourly');\nSELECT ts FROM s.hours WHERE ts BETWEEN @start_ts AND @end_ts" + ) + daily_b = ( + "MODEL (name s.b, kind INCREMENTAL_BY_TIME_RANGE (time_column ds), start '2023-01-01', " + "cron '@daily');\nSELECT ts::DATE AS ds, count(*) AS n FROM s.a " + "WHERE ts BETWEEN @start_ts AND @end_ts GROUP BY 1" + ) + project = _ExtensionProject( + tmp_path, {"a": hourly_a.format(start="2023-01-01 06:00:00"), "b": daily_b} + ) + project.context.engine_adapter.execute( + "CREATE TABLE s.hours AS SELECT range AS ts FROM " + "range(TIMESTAMP '2023-01-01 00:00:00', TIMESTAMP '2023-01-02 12:00:00', INTERVAL 1 HOUR)" + ) + project.context.plan(auto_apply=True, no_prompts=True) + count = "SELECT n FROM s.b WHERE ds = '2023-01-01'" + assert project.context.engine_adapter.fetchone(count) == (18,) + + project.update({"a": hourly_a.format(start="2023-01-01 00:00:00")}) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + assert list(plan.restatements.values()) == [ + (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + ] + assert project.context.engine_adapter.fetchone(count) == (24,) + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_rebuilds_full_downstream_model(tmp_path: Path): + models = { + "a": _extension_model("a", "s.src", "2023-01-02"), + "b": _extension_model("b", "s.a", "2023-01-01", kind="FULL"), + } + project = _ExtensionProject(tmp_path, models) + project.context.plan(auto_apply=True, no_prompts=True) + assert project.ds("s.b") == ["2023-01-02", "2023-01-03"] + + project.update({"a": _extension_model("a", "s.src", "2023-01-01")}) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + assert len(plan.restatements) == 1 + assert project.ds("s.b") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_skips_downstream_with_disabled_restatement( + tmp_path: Path, capture_console: CaptureTerminalConsole +): + kind = "INCREMENTAL_BY_TIME_RANGE (time_column ds, disable_restatement true)" + project = _ExtensionProject(tmp_path, _chain("2023-01-02", kind=kind)) + project.context.plan(auto_apply=True, no_prompts=True) + + project.update(_chain("2023-01-01", kind=kind)) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + assert not plan.restatements + assert "Restatement is disabled for this model" in capture_console.captured_warnings + assert project.ds("s.a") == ["2023-01-01", "2023-01-02", "2023-01-03"] + assert project.ds("s.b") == ["2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_with_non_breaking_change_restates_extension_only(tmp_path: Path): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + + # Add a column to `a` (non-breaking) and move its start earlier. + project.update({"a": _extension_model("a", "s.src", "2023-01-01", extra_columns=", 1 AS one")}) + plan_builder = project.context.plan_builder(no_auto_categorization=True) + plan_choice(plan_builder, SnapshotChangeCategory.NON_BREAKING) + plan = plan_builder.build() + project.context.apply(plan) + + day = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + assert {s_id.name: i for s_id, i in plan.restatements.items()} == { + '"memory"."s"."b"': day, + '"memory"."s"."c"': day, + } + for table in ("a", "b", "c"): + assert project.ds(f"s.{table}") == ["2023-01-01", "2023-01-02", "2023-01-03"], table + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_with_breaking_change_needs_no_restatement(tmp_path: Path): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + + project.update({"a": _extension_model("a", "s.src", "2023-01-01", extra_columns=", 1 AS one")}) + plan_builder = project.context.plan_builder(no_auto_categorization=True) + plan_choice(plan_builder, SnapshotChangeCategory.BREAKING) + plan = plan_builder.build() + project.context.apply(plan) + + assert not plan.restatements + for table in ("a", "b", "c"): + assert project.ds(f"s.{table}") == ["2023-01-01", "2023-01-02", "2023-01-03"], table + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_moved_later_does_not_restate(tmp_path: Path): + project = _ExtensionProject(tmp_path, _chain("2023-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + + project.update(_chain("2023-01-02")) + plan = project.context.plan(auto_apply=True, no_prompts=True) + assert not plan.restatements + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_with_select_model_warns_about_stale_models( + tmp_path: Path, capture_console: CaptureTerminalConsole +): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + + project.update(_chain("2023-01-01", c_start="2020-01-01")) + plan = project.context.plan(select_models=["s.a"], auto_apply=True, no_prompts=True) + + assert not plan.restatements + warnings = capture_console.captured_warnings + assert '"memory"."s"."b"' in warnings and "--restate-model" in warnings + assert project.ds("s.b") == ["2023-01-02", "2023-01-03"] + + # Following the advice from the warning fixes the downstream data. + project.context.plan( + restate_models=["s.b"], + start="2023-01-01", + end="2023-01-01", + auto_apply=True, + no_prompts=True, + ) + assert project.ds("s.b") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_clears_intervals_of_derived_dev_versions(tmp_path: Path): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + context = project.context + context.plan(auto_apply=True, no_prompts=True) + + # dev gets its own version of b, which has the (soon to be stale) intervals of its own. + project.update(_chain("2023-01-02", c_start="2020-01-01", extra_columns=", 1 AS one")) + context.plan("dev", auto_apply=True, no_prompts=True) + dev_b = context.get_snapshot("s.b") + assert dev_b is not None + assert dev_b.intervals + + # Back to prod's version of b, with the extended start of a. + project.update(_chain("2023-01-01", c_start="2020-01-01")) + context.plan("prod", auto_apply=True, no_prompts=True) + + day = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + state_b = context.state_sync.get_snapshots([dev_b.snapshot_id])[dev_b.snapshot_id] + assert not any(start <= day[0] and day[1] <= end for start, end in state_b.intervals) + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_is_explained_in_plan_output( + tmp_path: Path, capture_console: CaptureTerminalConsole +): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + capture_console.consume_captured_output() + + project.update(_chain("2023-01-01", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True, skip_tests=True) + output = capture_console.consume_captured_output() + + assert "Differences from the `prod` environment" in output + assert "restated because the start of an upstream model moved earlier" in output + assert "s.b" in output and "s.c" in output + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_in_dev_does_not_rewrite_prod_tables(tmp_path: Path): + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + context = project.context + context.plan(auto_apply=True, no_prompts=True) + + project.update(_chain("2023-01-01", c_start="2020-01-01")) + dev_plan = context.plan("dev", auto_apply=True, no_prompts=True, skip_tests=True) + + # b and c share their tables with prod, so the dev plan leaves them alone. + assert not dev_plan.restatements + assert project.ds("s.a") == ["2023-01-01", "2023-01-02", "2023-01-03"] + assert project.ds("s.b") == ["2023-01-02", "2023-01-03"] + assert project.ds("s.c") == ["2023-01-02", "2023-01-03"] + + # They are restated when the change is deployed to prod. + prod_plan = context.plan("prod", auto_apply=True, no_prompts=True, skip_tests=True) + assert {s_id.name for s_id in prod_plan.restatements} == { + '"memory"."s"."b"', + '"memory"."s"."c"', + } + assert project.ds("s.b") == ["2023-01-01", "2023-01-02", "2023-01-03"] + assert project.ds("s.c") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_flows_through_view(tmp_path: Path): + models = { + "a": _extension_model("a", "s.src", "2023-01-02"), + "b": "MODEL (name s.b, kind VIEW);\nSELECT ds FROM s.a", + "c": _extension_model("c", "s.b", "2020-01-01"), + } + project = _ExtensionProject(tmp_path, models) + project.context.plan(auto_apply=True, no_prompts=True) + assert project.ds("s.c") == ["2023-01-02", "2023-01-03"] + + project.update({"a": _extension_model("a", "s.src", "2023-01-01")}) + project.context.plan(auto_apply=True, no_prompts=True) + assert project.ds("s.c") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_does_not_run_plan_as_restatement(tmp_path: Path, mocker): + from sqlmesh.core.scheduler import Scheduler + + spy = mocker.spy(Scheduler, "run_merged_intervals") + project = _ExtensionProject(tmp_path, _chain("2023-01-02", c_start="2020-01-01")) + project.context.plan(auto_apply=True, no_prompts=True) + + project.update(_chain("2023-01-01", c_start="2020-01-01")) + plan = project.context.plan(auto_apply=True, no_prompts=True) + assert plan.restatements + assert spy.call_args.kwargs["is_restatement"] is False + + project.context.plan( + restate_models=["s.b"], + start="2023-01-01", + end="2023-01-01", + auto_apply=True, + no_prompts=True, + ) + assert spy.call_args.kwargs["is_restatement"] is True + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +@pytest.mark.parametrize("b_kind", ["VIEW", "EMBEDDED"]) +def test_start_extension_flows_through_passthrough_starting_after_range( + tmp_path: Path, b_kind: str +): + models = { + "a": _extension_model("a", "s.src", "2023-01-02"), + "b": f"MODEL (name s.b, kind {b_kind}, start '2023-01-03');\nSELECT ds FROM s.a", + "c": _extension_model("c", "s.b", "2020-01-01"), + } + project = _ExtensionProject(tmp_path, models) + project.context.plan(auto_apply=True, no_prompts=True) + assert project.ds("s.c") == ["2023-01-02", "2023-01-03"] + + project.update({"a": _extension_model("a", "s.src", "2023-01-01")}) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + restated_names = {s_id.name for s_id in plan.restatements} + assert restated_names == {'"memory"."s"."c"'} + assert project.ds("s.c") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_rebuilds_full_intermediate_starting_after_range_and_downstream( + tmp_path: Path, +): + models = { + "a": _extension_model("a", "s.src", "2023-01-02"), + "b": _extension_model("b", "s.a", "2023-01-03", kind="FULL"), + "c": _extension_model("c", "s.b", "2020-01-01"), + } + project = _ExtensionProject(tmp_path, models) + project.context.plan(auto_apply=True, no_prompts=True) + assert project.ds("s.b") == ["2023-01-02", "2023-01-03"] + assert project.ds("s.c") == ["2023-01-02", "2023-01-03"] + + project.update({"a": _extension_model("a", "s.src", "2023-01-01")}) + plan = project.context.plan(auto_apply=True, no_prompts=True) + + restated = {s_id.name: interval for s_id, interval in plan.restatements.items()} + assert set(restated) == {'"memory"."s"."b"', '"memory"."s"."c"'} + assert restated['"memory"."s"."c"'] == (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + assert project.ds("s.b") == ["2023-01-01", "2023-01-02", "2023-01-03"] + assert project.ds("s.c") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +def test_start_extension_full_intermediate_clears_early_interval_on_derived_dev_model( + tmp_path: Path, +): + models = { + "a": _extension_model("a", "s.src", "2023-01-02"), + "b": _extension_model("b", "s.a", "2023-01-03", kind="FULL"), + "c": _extension_model("c", "s.b", "2020-01-01"), + } + project = _ExtensionProject(tmp_path, models) + context = project.context + context.plan("prod", auto_apply=True, no_prompts=True) + + # Create a dev environment with a modified version of c downstream of FULL b. + project.update({"c": _extension_model("c", "s.b", "2020-01-01", extra_columns=", 1 AS one")}) + context.plan("dev", auto_apply=True, no_prompts=True) + dev_c = context.get_snapshot("s.c") + assert dev_c is not None and dev_c.intervals + assert project.ds("s__dev.c") == ["2023-01-02", "2023-01-03"] + + # Extend a's start in prod; both b ([Jan 3, Jan 5)) and c ([Jan 1, Jan 2)) are restated in prod, + # and dev_c must have its [Jan 1, Jan 2) interval cleared too. + project.update( + { + "a": _extension_model("a", "s.src", "2023-01-01"), + "c": _extension_model("c", "s.b", "2020-01-01"), + } + ) + context.plan("prod", auto_apply=True, no_prompts=True) + + day = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + state_dev_c = context.state_sync.get_snapshots([dev_c.snapshot_id])[dev_c.snapshot_id] + assert not any(start <= day[0] and day[1] <= end for start, end in state_dev_c.intervals) + + # Re-applying dev_c in dev backfills the cleared 2023-01-01 interval from rebuilt prod b. + project.update({"c": _extension_model("c", "s.b", "2020-01-01", extra_columns=", 1 AS one")}) + context.plan("dev", auto_apply=True, no_prompts=True) + assert project.ds("s__dev.c") == ["2023-01-01", "2023-01-02", "2023-01-03"] + + +@time_machine.travel(_EXTENSION_NOW, tick=False) +@pytest.mark.parametrize("use_finalized_state", [False, True]) +def test_start_extension_recovers_unfinalized_prod_without_conflict( + tmp_path: Path, use_finalized_state: bool +): + from unittest.mock import patch + + models = _chain("2023-01-02", c_start="2020-01-01") + models_dir = tmp_path / "models" + models_dir.mkdir() + for name, definition in models.items(): + (models_dir / f"{name}.sql").write_text(definition) + + config = Config( + model_defaults=ModelDefaultsConfig(dialect="duckdb"), + default_connection=DuckDBConnectionConfig(), + plan=PlanConfig(use_finalized_state=use_finalized_state), + ) + context = Context(paths=[tmp_path], config=config) + context.engine_adapter.create_schema("s") + context.engine_adapter.execute( + "CREATE TABLE s.src AS SELECT range::DATE AS ds FROM " + "range(DATE '2022-12-25', DATE '2023-01-06', INTERVAL 1 DAY)" + ) + + # 1. Finalized initial prod deployment (v1). + context.plan("prod", auto_apply=True, no_prompts=True) + + # 2. Separate dev version of b so RestatementStage has cross-environment intervals to clear. + for name, definition in _chain( + "2023-01-02", c_start="2020-01-01", extra_columns=", 99 AS dev_col" + ).items(): + (models_dir / f"{name}.sql").write_text(definition) + context.load() + context.plan("dev", auto_apply=True, no_prompts=True) + dev_b = context.get_snapshot("s.b") + assert dev_b is not None and dev_b.intervals + + # 3. Promote v2 to prod, failing right before FinalizeEnvironmentStage so prod stays unfinalized + # with snapshots=[v2] and previous_finalized_snapshots=[v1]. + for name, definition in _chain( + "2023-01-02", c_start="2020-01-01", extra_columns=", 2 AS v2_col" + ).items(): + (models_dir / f"{name}.sql").write_text(definition) + context.load() + + with patch.object(context.state_sync, "finalize", side_effect=RuntimeError("boom")): + with pytest.raises(RuntimeError, match="boom"): + context.plan("prod", auto_apply=True, no_prompts=True) + + unfinalized_prod = context.state_sync.get_environment("prod") + assert unfinalized_prod is not None + assert unfinalized_prod.finalized_ts is None + assert unfinalized_prod.previous_finalized_snapshots is not None + + # 4. Extend a's start to 2023-01-01 on top of the active baseline and apply to prod. + baseline_extra = "" if use_finalized_state else ", 2 AS v2_col" + for name, definition in _chain( + "2023-01-01", c_start="2020-01-01", extra_columns=baseline_extra + ).items(): + (models_dir / f"{name}.sql").write_text(definition) + context.load() + + plan = context.plan("prod", auto_apply=True, no_prompts=True) + assert plan.restatements + + finalized_prod = context.state_sync.get_environment("prod") + assert finalized_prod is not None + assert finalized_prod.finalized_ts is not None + + for table in ("a", "b", "c"): + rows = context.engine_adapter.fetchall(f"SELECT ds FROM s.{table} ORDER BY ds") + assert [str(r[0]) for r in rows] == ["2023-01-01", "2023-01-02", "2023-01-03"], table + + day = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + state_dev_b = context.state_sync.get_snapshots([dev_b.snapshot_id])[dev_b.snapshot_id] + assert not any(start <= day[0] and day[1] <= end for start, end in state_dev_b.intervals) diff --git a/tests/core/plan/test_evaluator.py b/tests/core/plan/test_evaluator.py new file mode 100644 index 0000000000..149e6adf9c --- /dev/null +++ b/tests/core/plan/test_evaluator.py @@ -0,0 +1,1235 @@ +import json +import typing as t +import pytest +from unittest.mock import MagicMock +from sqlglot import parse_one +from sqlmesh.core.context_diff import ContextDiff +from sqlmesh.core.environment import Environment +from sqlmesh.core.model import FullKind, SqlModel, IncrementalByTimeRangeKind +from sqlmesh.core.plan import EvaluatablePlan, PlanBuilder, stages as plan_stages +from sqlmesh.core.plan.common import identify_restatement_intervals_across_snapshot_versions +from sqlmesh.core.plan.evaluator import BuiltInPlanEvaluator +from sqlmesh.core.snapshot import SnapshotChangeCategory +from sqlmesh.utils.date import to_timestamp +from sqlmesh.utils.errors import ConflictingPlanError + + +def _make_evaluatable_plan( + environment_name: str = "prod", + restatements: t.Optional[dict] = None, + restate_all_snapshots: bool = True, + previous_finalized_snapshots: t.Optional[list] = None, + ensure_finalized_snapshots: bool = False, + previous_plan_id: t.Optional[str] = None, + previous_environment_snapshots: t.Optional[dict] = None, +) -> EvaluatablePlan: + env = Environment( + name=environment_name, + snapshots=[], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="plan_2", + previous_plan_id=previous_plan_id, + previous_finalized_snapshots=previous_finalized_snapshots, + ) + plan = MagicMock(spec=EvaluatablePlan) + plan.environment = env + plan.is_dev = False + plan.is_prod = True + plan.ensure_finalized_snapshots = ensure_finalized_snapshots + plan.previous_environment_snapshots = previous_environment_snapshots + plan.restate_all_snapshots = restate_all_snapshots + plan.restatements = restatements or {} + plan.disabled_restatement_models = set() + plan.execution_time = to_timestamp("2023-01-02") + return plan + + +def test_restatement_stage_expected_previous_snapshots_allows_promoted_model(make_snapshot): + # Model b was modified and promoted (b_v1 -> b_v2) in the current plan while a extended start + model_b_old = SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_new = SqlModel( + name="b", query="SELECT 2 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_dev = SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + + snap_b_old = make_snapshot(model_b_old) + snap_b_old.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_new = make_snapshot(model_b_new) + snap_b_new.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_new.previous_versions = snap_b_old.all_versions + snap_b_dev = make_snapshot(model_b_dev) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + assert snap_b_old.version != snap_b_new.version + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_new.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[snap_b_old.table_info], + ) + + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_new.name: snap_b_new}, + expected_previous_snapshots={snap_b_old.name: snap_b_old.table_info}, + ) + + # In prod, the active snapshot is still b_v1 prior to promotion + deployed_env = Environment( + name="prod", + snapshots=[snap_b_old.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="prev_plan", + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + # Should not raise ConflictingPlanError because prod version matches expected_previous_snapshots + evaluator.visit_restatement_stage(stage, evaluatable_plan) + + +def test_restatement_stage_expected_previous_snapshots_detects_concurrent_deploy(make_snapshot): + # Model b was promoted (b_v1 -> b_v2), but someone concurrently deployed b_v3 to prod + model_b_v1 = SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_v2 = SqlModel( + name="b", query="SELECT 2 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_v3 = SqlModel( + name="b", query="SELECT 3 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_dev = SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + + snap_b_v1 = make_snapshot(model_b_v1) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2 = make_snapshot(model_b_v2) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v3 = make_snapshot(model_b_v3) + snap_b_v3.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot(model_b_dev) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_v2.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[snap_b_v1.table_info], + ) + + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_v2.name: snap_b_v2}, + expected_previous_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + + # Prod has b_v3 deployed concurrently! + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v3.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="concurrent_plan", + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + + +def test_restatement_stage_expected_previous_snapshots_new_model_not_in_prod(make_snapshot): + # Model b did not exist in prod when plan was created + model_b = SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_dev = SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + + snap_b = make_snapshot(model_b) + snap_b.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot(model_b_dev) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[], + ) + + stage = plan_stages.RestatementStage( + all_snapshots={snap_b.name: snap_b}, + expected_previous_snapshots={}, # empty because b is new + ) + + # Prod does not have b yet + deployed_env = Environment( + name="prod", + snapshots=[], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="prev_plan", + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + # Adding a brand-new model does not raise a conflict + evaluator.visit_restatement_stage(stage, evaluatable_plan) + + +def test_restatement_stage_expected_previous_snapshots_new_model_conflicts_if_deployed( + make_snapshot, +): + # Model b did not exist when plan was created, but someone deployed another version of b to prod + model_b_v1 = SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_v2 = SqlModel( + name="b", query="SELECT 2 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + model_b_dev = SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + + snap_b_v1 = make_snapshot(model_b_v1) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2 = make_snapshot(model_b_v2) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot(model_b_dev) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_v1.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[], + ) + + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_v1.name: snap_b_v1}, + expected_previous_snapshots={}, + ) + + # Prod now has b_v2 deployed + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v2.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="other_plan", + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + + +def test_restatement_stage_populates_expected_previous_snapshots_in_builder(make_snapshot): + model_b = SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + snap_b = make_snapshot(model_b) + snap_b.categorize_as(SnapshotChangeCategory.BREAKING) + + env = Environment( + name="prod", + snapshots=[snap_b.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="plan_1", + previous_finalized_snapshots=[snap_b.table_info], + ) + plan = MagicMock(spec=EvaluatablePlan) + plan.environment = env + plan.is_dev = False + plan.restate_all_snapshots = True + plan.restatements = {snap_b.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))} + + state_reader = MagicMock() + builder = plan_stages.PlanStagesBuilder(state_reader, default_catalog=None) + stage = builder._get_restatement_stage(plan, {snap_b.name: snap_b}) + + assert stage is not None + assert stage.expected_previous_snapshots is not None + assert snap_b.name in stage.expected_previous_snapshots + assert stage.expected_previous_snapshots[snap_b.name].version == snap_b.version + + +@pytest.mark.parametrize("ensure_finalized_snapshots", [False, True]) +def test_restatement_stage_unfinalized_prod_recovery( + make_snapshot, ensure_finalized_snapshots: bool +): + model_a_old = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-02", + ) + model_a_new = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-01", + ) + model_b_v1 = SqlModel( + name="b", + query=parse_one("SELECT 1 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_v2 = SqlModel( + name="b", + query=parse_one("SELECT 2 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_dev = SqlModel( + name="b", + query=parse_one("SELECT 99 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + + snap_a_old = make_snapshot(model_a_old) + snap_a_old.categorize_as(SnapshotChangeCategory.BREAKING) + snap_a_old.add_interval("2023-01-02", "2023-01-03") + + snap_a_new = make_snapshot(model_a_new) + snap_a_new.previous_versions = snap_a_old.all_versions + + snap_b_v1 = make_snapshot(model_b_v1, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v1.add_interval("2020-01-01", "2023-01-03") + + snap_b_v2 = make_snapshot(model_b_v2, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2.add_interval("2020-01-01", "2023-01-03") + + baseline_b = snap_b_v1 if ensure_finalized_snapshots else snap_b_v2 + snap_b_next = make_snapshot(baseline_b.model, nodes={snap_a_new.name: snap_a_new.model}) + snap_b_next.previous_versions = baseline_b.all_versions + + snap_b_dev = make_snapshot(model_b_dev, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev.add_interval("2020-01-01", "2023-01-03") + + # prod is unfinalized: plan_1 promoted b_v2 and failed before FinalizeEnvironmentStage, + # leaving previous_finalized_snapshots = [a_old, b_v1] and finalized_ts = None. + deployed_env = Environment( + name="prod", + snapshots=[snap_a_old.table_info, snap_b_v2.table_info], + previous_finalized_snapshots=[snap_a_old.table_info, snap_b_v1.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="plan_1", + finalized_ts=None, + ) + dev_env = Environment( + name="dev", + snapshots=[snap_a_old.table_info, snap_b_dev.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="dev_plan", + finalized_ts=100, + ) + + context_diff = ContextDiff( + environment="prod", + is_new_environment=False, + is_unfinalized_environment=True, + normalize_environment_name=True, + create_from="prod", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots={ + snap_a_new.name: (snap_a_new, snap_a_old), + snap_b_next.name: (snap_b_next, baseline_b), + }, + snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_next.snapshot_id: snap_b_next, + }, + new_snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_next.snapshot_id: snap_b_next, + }, + previous_plan_id="plan_1", + previously_promoted_snapshot_ids={snap_a_old.snapshot_id, snap_b_v2.snapshot_id}, + previous_finalized_snapshots=[snap_a_old.table_info, snap_b_v1.table_info], + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + + plan = PlanBuilder( + context_diff, + execution_time="2023-01-04 12:00:00 UTC", + ensure_finalized_snapshots=ensure_finalized_snapshots, + ).build() + + # Environment.previous_finalized_snapshots must keep its exact state-sync semantics ([b_v1]). + assert plan.environment.previous_finalized_snapshots is not None + prev_finalized_by_name = {s.name: s for s in plan.environment.previous_finalized_snapshots} + assert prev_finalized_by_name[snap_b_v1.name].version == snap_b_v1.version + + evaluatable_plan = plan.to_evaluatable() + assert evaluatable_plan.previous_environment_snapshots is not None + assert ( + evaluatable_plan.previous_environment_snapshots[baseline_b.name].version + == baseline_b.version + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [deployed_env.summary, dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + state_sync.get_snapshots.return_value = {} + + builder = plan_stages.PlanStagesBuilder(state_sync, default_catalog=None) + stage = builder._get_restatement_stage( + evaluatable_plan, + {snap_a_new.name: snap_a_new, snap_b_next.name: snap_b_next}, + deployed_env, + ) + assert stage is not None + assert stage.expected_previous_snapshots is not None + assert stage.expected_previous_snapshots[baseline_b.name].version == baseline_b.version + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + # Recovery must not self-deadlock with a false ConflictingPlanError. + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_called_once() + + +def test_restatement_stage_in_flight_plan_finalizes_concurrently_under_ensure_finalized( + make_snapshot, +): + # At t=0, prod was unfinalized under plan_1 (snapshots=[b_v2], previous_finalized_snapshots=[b_v1]). + # plan_2 was created with ensure_finalized_snapshots=True (baseline b_v1, previous_plan_id="plan_1"). + # Before visit_restatement_stage runs at t=2, plan_1 finishes FinalizeEnvironmentStage (finalized_ts set). + snap_b_v1 = make_snapshot( + SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2 = make_snapshot( + SqlModel( + name="b", query="SELECT 2 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot( + SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_v1.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[snap_b_v1.table_info], + ensure_finalized_snapshots=True, + previous_plan_id="plan_1", + previous_environment_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_v1.name: snap_b_v1}, + expected_previous_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v2.table_info], + previous_finalized_snapshots=[snap_b_v1.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="plan_1", + finalized_ts=200, # plan_1 finalized concurrently! + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_not_called() + + +@pytest.mark.parametrize("ensure_finalized_snapshots", [False, True]) +@pytest.mark.parametrize("plan_3_finalized_ts", [None, 300]) +def test_restatement_stage_concurrent_plan_3_detected_even_when_unfinalized( + make_snapshot, + ensure_finalized_snapshots: bool, + plan_3_finalized_ts: t.Optional[int], +): + snap_b_v1 = make_snapshot( + SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v3 = make_snapshot( + SqlModel( + name="b", query="SELECT 3 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v3.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot( + SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_v1.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_finalized_snapshots=[snap_b_v1.table_info], + ensure_finalized_snapshots=ensure_finalized_snapshots, + previous_plan_id="plan_1", + previous_environment_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_v1.name: snap_b_v1}, + expected_previous_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + + # Concurrent plan_3 promoted b_v3 (either finalized or crashed unfinalized with previous_finalized_snapshots=[b_v1]). + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v3.table_info], + previous_finalized_snapshots=[snap_b_v1.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="plan_3", + finalized_ts=plan_3_finalized_ts, + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_not_called() + + +def test_restatement_stage_new_environment_with_create_from_dev(make_snapshot): + # First-ever prod plan initialized with create_from="dev" (where dev already had a and b). + model_a_old = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-02", + ) + model_a_new = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-01", + ) + model_b = SqlModel( + name="b", + query=parse_one("SELECT 1 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_dev_other = SqlModel( + name="b", + query=parse_one("SELECT 99 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + + snap_a_old = make_snapshot(model_a_old) + snap_a_old.categorize_as(SnapshotChangeCategory.BREAKING) + snap_a_old.add_interval("2023-01-02", "2023-01-03") + snap_a_new = make_snapshot(model_a_new) + snap_a_new.previous_versions = snap_a_old.all_versions + + snap_b_old = make_snapshot(model_b, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_old.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_old.add_interval("2020-01-01", "2023-01-03") + snap_b_new = make_snapshot(model_b, nodes={snap_a_new.name: snap_a_new.model}) + snap_b_new.previous_versions = snap_b_old.all_versions + + snap_b_dev_other = make_snapshot(model_b_dev_other, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_dev_other.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev_other.add_interval("2020-01-01", "2023-01-03") + + context_diff = ContextDiff( + environment="prod", + is_new_environment=True, + is_unfinalized_environment=False, + normalize_environment_name=True, + create_from="dev", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots={ + snap_a_new.name: (snap_a_new, snap_a_old), + snap_b_new.name: (snap_b_new, snap_b_old), + }, + snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_new.snapshot_id: snap_b_new, + }, + new_snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_new.snapshot_id: snap_b_new, + }, + previous_plan_id=None, + previously_promoted_snapshot_ids=set(), + previous_finalized_snapshots=[snap_a_old.table_info, snap_b_old.table_info], + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + + plan = PlanBuilder(context_diff, execution_time="2023-01-04 12:00:00 UTC").build() + evaluatable_plan = plan.to_evaluatable() + # Target environment (prod) did not exist at t=0, even though create_from="dev" had snapshots. + assert evaluatable_plan.previous_environment_snapshots == {} + + dev_env = Environment( + name="dev_other", + snapshots=[snap_b_dev_other.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="dev_plan", + ) + + # 1. Succeeds when prod is still absent (None) or expired at t=2. + expired_prod = Environment( + name="prod", + snapshots=[snap_b_dev_other.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="old_expired_plan", + expiration_ts=1, + ) + for live_prod in (None, expired_prod): + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name, p=live_prod: ( + p if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + builder = plan_stages.PlanStagesBuilder(state_sync, default_catalog=None) + stage = builder._get_restatement_stage( + evaluatable_plan, + {snap_a_new.name: snap_a_new, snap_b_new.name: snap_b_new}, + live_prod, + ) + assert stage is not None + assert stage.expected_previous_snapshots == {} + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_called_once() + + # 2. Conflicts if another plan concurrently created active prod with b (same or different version). + for concurrent_b in (snap_b_new, snap_b_dev_other): + concurrent_prod = Environment( + name="prod", + snapshots=[concurrent_b.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="concurrent_init_plan", + ) + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name, p=concurrent_prod: ( + p if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + stage = plan_stages.RestatementStage( + all_snapshots={snap_a_new.name: snap_a_new, snap_b_new.name: snap_b_new}, + expected_previous_snapshots={}, + ) + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_not_called() + + +def test_restatement_stage_concurrent_model_deletion_detected(make_snapshot): + snap_b = make_snapshot( + SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot( + SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_environment_snapshots={snap_b.name: snap_b.table_info}, + ) + stage = plan_stages.RestatementStage( + all_snapshots={snap_b.name: snap_b}, + expected_previous_snapshots={snap_b.name: snap_b.table_info}, + ) + # Concurrent plan removed b from prod. + deployed_env = Environment( + name="prod", + snapshots=[], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="delete_plan", + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + with pytest.raises(ConflictingPlanError, match="Another plan.*deployed new versions"): + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_not_called() + + +def test_restatement_stage_legacy_evaluatable_plan_without_previous_environment_snapshots( + make_snapshot, +): + model_b_v1 = SqlModel( + name="b", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_v2 = SqlModel( + name="b", + query=parse_one("SELECT 2 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_dev = SqlModel( + name="b", + query=parse_one("SELECT 99 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + + snap_b_v1 = make_snapshot(model_b_v1) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2 = make_snapshot(model_b_v2) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2.add_interval("2020-01-01", "2023-01-03") + snap_b_dev = make_snapshot(model_b_dev) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + context_diff = ContextDiff( + environment="prod", + is_new_environment=False, + is_unfinalized_environment=True, + normalize_environment_name=True, + create_from="prod", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots={}, + snapshots={snap_b_v2.snapshot_id: snap_b_v2}, + new_snapshots={}, + previous_plan_id="plan_1", + previously_promoted_snapshot_ids={snap_b_v2.snapshot_id}, + previous_finalized_snapshots=[snap_b_v1.table_info], + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + plan = PlanBuilder( + context_diff, + restate_models={snap_b_v2.name}, + restate_all_snapshots=True, + start="2023-01-01", + end="2023-01-01", + execution_time="2023-01-04 12:00:00 UTC", + ).build() + + payload = json.loads(plan.to_evaluatable().json()) + payload.pop("previous_environment_snapshots", None) + legacy_plan = EvaluatablePlan.parse_obj(payload) + assert legacy_plan.previous_environment_snapshots is None + + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v2.table_info], + previous_finalized_snapshots=[snap_b_v1.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="plan_1", + finalized_ts=None, + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + builder = plan_stages.PlanStagesBuilder(state_sync, default_catalog=None) + stage = builder._get_restatement_stage(legacy_plan, {snap_b_v2.name: snap_b_v2}) + assert stage is not None + assert stage.expected_previous_snapshots == {snap_b_v2.name: snap_b_v2.table_info} + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + evaluator.visit_restatement_stage(stage, legacy_plan) + state_sync.remove_intervals.assert_called_once() + + +def test_identify_restatement_intervals_widens_overlapping_downstream_dev_snapshot(make_snapshot): + # b is FULL starting at 2023-01-03 (restated for [Jan 3, Jan 5)), while downstream c is + # INCREMENTAL starting at 2020-01-01 (restated for [Jan 1, Jan 2)). + # In dev, b shares prod's version while c has a modified c_dev version; c_dev's clear request + # must widen across both intervals to cover [Jan 1, Jan 5), not stop at [Jan 3, Jan 5). + model_a = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-01", + ) + model_b = SqlModel( + name="b", + query=parse_one("SELECT id, ds FROM a"), + kind=FullKind(), + start="2023-01-03", + ) + model_c_prod = SqlModel( + name="c", + query=parse_one("SELECT id, ds FROM b"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_c_dev = SqlModel( + name="c", + query=parse_one("SELECT id, ds, 99 as extra FROM b"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + + snap_a = make_snapshot(model_a) + snap_a.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b = make_snapshot(model_b, nodes={snap_a.name: snap_a.model}) + snap_b.categorize_as(SnapshotChangeCategory.BREAKING) + snap_c_prod = make_snapshot( + model_c_prod, nodes={snap_a.name: snap_a.model, snap_b.name: snap_b.model} + ) + snap_c_prod.categorize_as(SnapshotChangeCategory.BREAKING) + snap_c_dev = make_snapshot( + model_c_dev, nodes={snap_a.name: snap_a.model, snap_b.name: snap_b.model} + ) + snap_c_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + dev_env = Environment( + name="dev", + snapshots=[snap_a.table_info, snap_b.table_info, snap_c_dev.table_info], + start_at="2020-01-01", + end_at="2023-01-04", + plan_id="dev_plan", + ) + + state_reader = MagicMock() + state_reader.get_environments_summary.return_value = [dev_env.summary] + state_reader.get_environment.return_value = dev_env + state_reader.get_snapshots_by_names.return_value = [] + + prod_restatements = { + snap_b.name: (to_timestamp("2023-01-03"), to_timestamp("2023-01-05")), + snap_c_prod.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")), + } + cleared = identify_restatement_intervals_across_snapshot_versions( + state_reader=state_reader, + prod_restatements=prod_restatements, + disable_restatement_models=set(), + loaded_snapshots={ + snap_a.snapshot_id: snap_a, + snap_b.snapshot_id: snap_b, + snap_c_prod.snapshot_id: snap_c_prod, + }, + current_ts=to_timestamp("2023-01-04"), + ) + + assert set(cleared) == {snap_c_dev.snapshot_id} + assert cleared[snap_c_dev.snapshot_id].interval == ( + to_timestamp("2023-01-01"), + to_timestamp("2023-01-05"), + ) + + +@pytest.mark.parametrize("ensure_finalized_snapshots", [False, True]) +def test_restatement_stage_recreate_existing_environment_with_create_from_dev( + make_snapshot, ensure_finalized_snapshots: bool +): + # When always_recreate_environment=True and create_from="dev" targets an existing non-expired + # prod environment, ContextDiff sets is_new_environment=True AND previous_plan_id=prod.plan_id, + # while context_diff.environment_snapshots holds dev's snapshots. + model_a_old = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-02", + ) + model_a_new = SqlModel( + name="a", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-01", + ) + model_b_prod = SqlModel( + name="b", + query=parse_one("SELECT 1 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_dev = SqlModel( + name="b", + query=parse_one("SELECT 2 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + model_b_other = SqlModel( + name="b", + query=parse_one("SELECT 99 as id, ds FROM a"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2020-01-01", + ) + + snap_a_old = make_snapshot(model_a_old) + snap_a_old.categorize_as(SnapshotChangeCategory.BREAKING) + snap_a_old.add_interval("2023-01-02", "2023-01-03") + snap_a_new = make_snapshot(model_a_new) + snap_a_new.previous_versions = snap_a_old.all_versions + + snap_b_prod = make_snapshot(model_b_prod, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_prod.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_prod.add_interval("2020-01-01", "2023-01-03") + + snap_b_dev = make_snapshot(model_b_dev, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev.add_interval("2020-01-01", "2023-01-03") + + snap_b_new = make_snapshot(model_b_dev, nodes={snap_a_new.name: snap_a_new.model}) + snap_b_new.previous_versions = snap_b_dev.all_versions + + snap_b_other = make_snapshot(model_b_other, nodes={snap_a_old.name: snap_a_old.model}) + snap_b_other.categorize_as(SnapshotChangeCategory.BREAKING) + + context_diff = ContextDiff( + environment="prod", + is_new_environment=True, + is_unfinalized_environment=False, + normalize_environment_name=True, + create_from="dev", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots={ + snap_a_new.name: (snap_a_new, snap_a_old), + snap_b_new.name: (snap_b_new, snap_b_dev), + }, + snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_new.snapshot_id: snap_b_new, + }, + new_snapshots={ + snap_a_new.snapshot_id: snap_a_new, + snap_b_new.snapshot_id: snap_b_new, + }, + previous_plan_id="prod_plan_1", + previously_promoted_snapshot_ids=set(), + previous_finalized_snapshots=[snap_a_old.table_info, snap_b_dev.table_info], + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + + plan = PlanBuilder( + context_diff, + execution_time="2023-01-04 12:00:00 UTC", + ensure_finalized_snapshots=ensure_finalized_snapshots, + ).build() + evaluatable_plan = plan.to_evaluatable() + assert evaluatable_plan.previous_environment_snapshots is None + + deployed_prod = Environment( + name="prod", + snapshots=[snap_a_old.table_info, snap_b_prod.table_info], + previous_finalized_snapshots=[snap_a_old.table_info, snap_b_prod.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="prod_plan_1", + finalized_ts=100, + ) + other_dev = Environment( + name="other_dev", + snapshots=[snap_b_other.table_info], + start_at="2020-01-01", + end_at="2023-01-03", + plan_id="other_dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_prod if name == "prod" else other_dev + ) + state_sync.get_environments_summary.return_value = [other_dev.summary] + state_sync.get_snapshots_by_names.return_value = [] + + builder = plan_stages.PlanStagesBuilder(state_sync, default_catalog=None) + stage = builder._get_restatement_stage( + evaluatable_plan, + {snap_a_new.name: snap_a_new, snap_b_new.name: snap_b_new}, + deployed_prod, + ) + assert stage is not None + assert stage.expected_previous_snapshots is not None + assert stage.expected_previous_snapshots[snap_b_prod.name].version == snap_b_prod.version + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_called_once() + + +def test_restatement_stage_same_plan_id_retry_is_idempotent(make_snapshot): + snap_b_v1 = make_snapshot( + SqlModel( + name="b", query="SELECT 1 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v1.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_v2 = make_snapshot( + SqlModel( + name="b", query="SELECT 2 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_v2.categorize_as(SnapshotChangeCategory.BREAKING) + snap_b_dev = make_snapshot( + SqlModel( + name="b", query="SELECT 99 as id, ds", kind=IncrementalByTimeRangeKind(time_column="ds") + ) + ) + snap_b_dev.categorize_as(SnapshotChangeCategory.BREAKING) + + evaluatable_plan = _make_evaluatable_plan( + restatements={snap_b_v2.name: (to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))}, + previous_environment_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + evaluatable_plan.plan_id = evaluatable_plan.environment.plan_id + stage = plan_stages.RestatementStage( + all_snapshots={snap_b_v2.name: snap_b_v2}, + expected_previous_snapshots={snap_b_v1.name: snap_b_v1.table_info}, + ) + + # Same plan_id ("plan_2") already promoted b_v2; retrying visit_restatement_stage must not conflict with itself. + deployed_env = Environment( + name="prod", + snapshots=[snap_b_v2.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id=evaluatable_plan.plan_id, + ) + dev_env = Environment( + name="dev", + snapshots=[snap_b_dev.table_info], + start_at="2023-01-01", + end_at="2023-01-02", + plan_id="dev_plan", + ) + + state_sync = MagicMock() + state_sync.get_environment.side_effect = lambda name: ( + deployed_env if name == "prod" else dev_env + ) + state_sync.get_environments_summary.return_value = [dev_env.summary] + state_sync.get_snapshots_by_names.return_value = [] + + evaluator = BuiltInPlanEvaluator( + state_sync=state_sync, + snapshot_evaluator=MagicMock(), + create_scheduler=MagicMock(), + default_catalog=None, + console=MagicMock(), + ) + evaluator.visit_restatement_stage(stage, evaluatable_plan) + state_sync.remove_intervals.assert_called_once() diff --git a/tests/core/test_plan.py b/tests/core/test_plan.py index add85d2eec..de387e913c 100644 --- a/tests/core/test_plan.py +++ b/tests/core/test_plan.py @@ -4510,3 +4510,843 @@ def test_forward_only_indirect_change_to_materialized_view(make_snapshot): # Forward-only indirect changes to MVs should not always be classified as indirect breaking. # Instead, we want to preserve the standard categorization. assert snapshot_b_new.change_category == SnapshotChangeCategory.INDIRECT_NON_BREAKING + + +# --- Start extension (#4837): extending a model's `start` earlier must refresh downstream intervals --- + +_EXECUTION_TIME = "2023-01-04 12:00:00 UTC" + + +def _ts(value: str) -> int: + return to_timestamp(value) + + +def _start_extension_context_diff( + make_snapshot: t.Callable, + specs: t.List[t.Dict[str, t.Any]], + environment: str = "prod", +) -> t.Tuple[ContextDiff, t.Dict[str, Snapshot], t.Dict[str, Snapshot]]: + """Builds a context diff in which every listed model has an old and a new version. + + Each spec is a dict with `name`, an optional `parent`, `old` / `new` model kwargs overrides and + optional `intervals` (a list of (start, end) dates added to the old snapshot). + """ + old_nodes: t.Dict[str, t.Any] = {} + new_nodes: t.Dict[str, t.Any] = {} + olds: t.Dict[str, Snapshot] = {} + news: t.Dict[str, Snapshot] = {} + + for spec in specs: + name = spec["name"] + parent = spec.get("parent") + if spec.get("parents"): + query = parse_one(" union all ".join(f"select ds from {p}" for p in spec["parents"])) + elif parent and spec.get("self_referencing"): + query = parse_one(f"select ds from {parent} union all select ds from {name}") + else: + query = parse_one(f"select ds from {parent}" if parent else "select 1 as ds") + common: t.Dict[str, t.Any] = { + "name": name, + "query": query, + "kind": spec.get("kind", IncrementalByTimeRangeKind(time_column="ds")), + "start": spec.get("start", "2020-01-01"), + "cron": spec.get("cron", "@daily"), + } + old = make_snapshot(SqlModel(**{**common, **spec.get("old", {})}), nodes=dict(old_nodes)) + new = make_snapshot(SqlModel(**{**common, **spec.get("new", {})}), nodes=dict(new_nodes)) + old.categorize_as(SnapshotChangeCategory.BREAKING) + new.previous_versions = old.all_versions + for start, end in spec.get("intervals", []): + old.add_interval(start, end) + if spec.get("unpaused"): + new.unpaused_ts = 1 + + old_nodes[old.name] = old.model + new_nodes[new.name] = new.model + olds[name] = old + news[name] = new + + context_diff = ContextDiff( + environment=environment, + is_new_environment=False, + is_unfinalized_environment=False, + normalize_environment_name=True, + create_from="prod", + create_from_env_exists=True, + added=set(), + removed_snapshots={}, + modified_snapshots={new.name: (new, olds[n]) for n, new in news.items()}, + snapshots={s.snapshot_id: s for s in news.values()}, + new_snapshots={s.snapshot_id: s for s in news.values()}, + previous_plan_id=None, + previously_promoted_snapshot_ids=set(), + previous_finalized_snapshots=None, + previous_gateway_managed_virtual_layer=False, + gateway_managed_virtual_layer=False, + environment_statements=[], + ) + return context_diff, news, olds + + +def _chain_specs( + a_old: str = "2023-01-02", + a_new: str = "2023-01-01", + **child_overrides: t.Any, +) -> t.List[t.Dict[str, t.Any]]: + """a (start moves) -> b -> c where b and c have long-standing intervals.""" + intervals = [("2020-01-01", "2023-01-03")] + return [ + { + "name": "a", + "old": {"start": a_old}, + "new": {"start": a_new}, + "intervals": [(a_old, "2023-01-03")], + }, + {"name": "b", "parent": "a", "intervals": intervals, **child_overrides}, + {"name": "c", "parent": "b", "intervals": intervals}, + ] + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_restates_downstream(make_snapshot): + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + plan = PlanBuilder(context_diff).build() + + expected = (_ts("2023-01-01"), _ts("2023-01-02")) + assert plan.restatements == { + snaps["b"].snapshot_id: expected, + snaps["c"].snapshot_id: expected, + } + # The model whose start moved is backfilled as usual and is not restated itself. + assert snaps["a"].snapshot_id not in plan.restatements + # This is not a user requested restatement. + assert not plan.selected_models_to_restate + assert plan.models_to_backfill is None + assert plan.requires_backfill + assert plan.start_extension_restatements == {snaps["b"].snapshot_id, snaps["c"].snapshot_id} + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_moved_later_does_not_restate(make_snapshot): + context_diff, _, _ = _start_extension_context_diff( + make_snapshot, _chain_specs(a_old="2023-01-01", a_new="2023-01-02") + ) + plan = PlanBuilder(context_diff).build() + assert not plan.restatements + assert not plan.start_extension_restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_without_downstream_is_a_noop(make_snapshot): + context_diff, _, _ = _start_extension_context_diff(make_snapshot, _chain_specs()[:1]) + plan = PlanBuilder(context_diff).build() + assert not plan.restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_downstream_starting_after_range_is_not_restated(make_snapshot): + specs = _chain_specs() + specs[1].update(start="2023-01-03", intervals=[("2023-01-03", "2023-01-03")]) + specs[2].update(start="2023-01-03", intervals=[("2023-01-03", "2023-01-03")]) + context_diff, _, _ = _start_extension_context_diff(make_snapshot, specs) + assert not PlanBuilder(context_diff).build().restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_unchanged_relative_start_is_not_an_extension(make_snapshot): + specs = _chain_specs() + specs[0].update( + old={"start": "2 days ago", "description": "old"}, + new={"start": "2 days ago", "description": "new"}, + ) + context_diff, _, _ = _start_extension_context_diff(make_snapshot, specs) + assert not PlanBuilder(context_diff).build().restatements + + # The same relative expression resolves against the plan's execution time, not the clock + # at the time the snapshots were created. + assert not PlanBuilder(context_diff, execution_time="2023-02-01").build().restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_relative_start_moved_earlier(make_snapshot): + specs = _chain_specs() + specs[0].update(old={"start": "2 days ago"}, new={"start": "3 days ago"}) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + expected = (_ts("3 days ago"), _ts("2 days ago")) + assert plan.restatements[snaps["b"].snapshot_id] == expected + assert plan.restatements[snaps["c"].snapshot_id] == expected + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_widens_range_for_coarser_downstream(make_snapshot): + specs = _chain_specs(a_old="2023-01-01 06:00:00", a_new="2023-01-01 00:00:00") + specs[0]["cron"] = "@hourly" + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + # The six hours that were added belong to a daily interval, which gets restated as a whole. + whole_day = (_ts("2023-01-01"), _ts("2023-01-02")) + assert plan.restatements[snaps["b"].snapshot_id] == whole_day + assert plan.restatements[snaps["c"].snapshot_id] == whole_day + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_full_downstream_is_restated(make_snapshot): + context_diff, snaps, _ = _start_extension_context_diff( + make_snapshot, _chain_specs(kind=FullKind()) + ) + plan = PlanBuilder(context_diff).build() + assert snaps["b"].snapshot_id in plan.restatements + # Descendants of the FULL model still only get the affected range. + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_respects_disable_restatement(make_snapshot, mocker: MockerFixture): + context_diff, snaps, _ = _start_extension_context_diff( + make_snapshot, + _chain_specs(kind=IncrementalByTimeRangeKind(time_column="ds", disable_restatement=True)), + ) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + # b keeps its data in this plan, so c (derived from it) is consistent with it. + assert not plan.restatements + console.log_warning.assert_called_once() + message = console.log_warning.call_args[0][0] + assert "Restatement is disabled for this model" in message + assert "Models downstream of it are not restated through it" in message + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_no_spurious_warning_without_existing_intervals( + make_snapshot, mocker: MockerFixture +): + # SCD Type 2 has `disable_restatement` enabled by default, but there is nothing stale to + # restate if the model has no intervals in the affected range, so there must be no warning. + specs = _chain_specs( + kind=dict(name=ModelKindName.SCD_TYPE_2_BY_TIME, unique_key=["ds"]), + start="2023-01-02", + intervals=[("2023-01-02", "2023-01-03")], + ) + context_diff, _, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + assert not PlanBuilder(context_diff, console=console).build().restatements + console.log_warning.assert_not_called() + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_scd_type_2_downstream_warns(make_snapshot, mocker: MockerFixture): + specs = _chain_specs(kind=dict(name=ModelKindName.SCD_TYPE_2_BY_TIME, unique_key=["ds"])) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + assert ( + snaps["b"].snapshot_id + not in PlanBuilder(context_diff, console=console).build().restatements + ) + assert "Restatement is disabled" in console.log_warning.call_args[0][0] + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_full_history_only_downstream_warns_once( + make_snapshot, mocker: MockerFixture +): + specs = _chain_specs(kind=dict(name=ModelKindName.INCREMENTAL_BY_UNIQUE_KEY, unique_key=["ds"])) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + # A one day start move must not rebuild the full history of b automatically. + assert snaps["b"].snapshot_id not in plan.restatements + console.log_warning.assert_called_once() + message = console.log_warning.call_args[0][0] + assert snaps["b"].name in message and "full history" in message + assert f"--restate-model '{snaps['b'].name}'" in message + assert "--start 2023-01-01 --end 2023-01-01" in message + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_dev_does_not_restate_versions_shared_with_prod( + make_snapshot, mocker: MockerFixture +): + # The new (paused) versions of b and c are metadata only changes: they share their physical + # tables with prod, so a dev plan must not rewrite those. + context_diff, snaps, _ = _start_extension_context_diff( + make_snapshot, _chain_specs(), environment="dev" + ) + console = mocker.Mock() + plan = PlanBuilder(context_diff, is_dev=True, console=console).build() + + assert not plan.restatements + assert not plan.start_extension_restatements + console.log_warning.assert_called_once() + message = console.log_warning.call_args[0][0] + assert snaps["b"].name in message + assert "tables that production uses or will use" in message + assert "along with their downstream models" in message + assert "Cannot restate model" not in message + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_dev_restates_versions_with_their_own_table( + make_snapshot, mocker: MockerFixture +): + # Forward-only changes are previewed in dev tables (initialized from the production intervals), + # so restating them can't affect prod. + specs = _chain_specs() + specs[0]["new"]["query"] = parse_one("select 1 as ds, 2 as extra") + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs, environment="dev") + console = mocker.Mock() + plan = PlanBuilder( + context_diff, is_dev=True, forward_only=True, start="2023-01-01", console=console + ).build() + + assert not plan.deployability_index.is_deployable(snaps["b"]) + assert {s_id.name for s_id in plan.start_extension_restatements} == { + snaps["b"].name, + snaps["c"].name, + } + assert snaps["b"].dev_intervals + console.log_warning.assert_not_called() + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flows_through_view_without_intervals(make_snapshot): + specs = _chain_specs(kind=ViewKind(), intervals=[]) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + # c is derived from the view, which reflects the new data of a right away. + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flows_through_embedded_model(make_snapshot): + specs = _chain_specs(kind=dict(name=ModelKindName.EMBEDDED), intervals=[]) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flow_stops_at_models_that_do_not_cover_the_range(make_snapshot): + # Incremental b starts after the added range, so its interval-sliced query does not cover + # the extended range and neither does anything derived from it. + specs = _chain_specs( + kind=IncrementalByTimeRangeKind(time_column="ds"), + start="2023-01-03", + intervals=[("2023-01-03", "2023-01-04")], + ) + context_diff, _, _ = _start_extension_context_diff(make_snapshot, specs) + assert not PlanBuilder(context_diff).build().restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flows_through_view_starting_after_range(make_snapshot): + specs = _chain_specs( + kind=ViewKind(), + start="2023-01-03", + intervals=[("2023-01-03", "2023-01-04")], + ) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].snapshot_id not in plan.restatements + assert plan.restatements == {snaps["c"].snapshot_id: (_ts("2023-01-01"), _ts("2023-01-02"))} + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flows_through_embedded_starting_after_range(make_snapshot): + specs = _chain_specs( + kind=dict(name=ModelKindName.EMBEDDED), + start="2023-01-03", + intervals=[], + ) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].snapshot_id not in plan.restatements + assert plan.restatements == {snaps["c"].snapshot_id: (_ts("2023-01-01"), _ts("2023-01-02"))} + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_refreshes_full_model_starting_after_range_and_flows_downstream( + make_snapshot, +): + specs = _chain_specs( + kind=FullKind(), + start="2023-01-03", + intervals=[("2023-01-03", "2023-01-04")], + ) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].snapshot_id in plan.restatements + assert snaps["c"].snapshot_id in plan.restatements + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + missing_by_id = {si.snapshot_id: si.intervals for si in plan.missing_intervals} + assert snaps["b"].snapshot_id in missing_by_id + assert snaps["c"].snapshot_id in missing_by_id + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_refreshes_materialized_view_starting_after_range_and_flows_downstream( + make_snapshot, +): + specs = _chain_specs( + kind=ViewKind(materialized=True), + start="2023-01-03", + intervals=[("2023-01-03", "2023-01-04")], + ) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].snapshot_id in plan.restatements + assert snaps["c"].snapshot_id in plan.restatements + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + missing_by_id = {si.snapshot_id: si.intervals for si in plan.missing_intervals} + assert snaps["b"].snapshot_id in missing_by_id + assert snaps["c"].snapshot_id in missing_by_id + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_stops_at_future_unpopulated_full_model( + make_snapshot, mocker: MockerFixture +): + specs = _chain_specs( + kind=FullKind(), + start="2023-01-10", + intervals=[], + ) + context_diff, _, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + assert not plan.restatements + console.log_warning.assert_not_called() + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flow_stops_at_child_skipped_by_a_guard( + make_snapshot, mocker: MockerFixture +): + specs = _chain_specs(kind=dict(name=ModelKindName.INCREMENTAL_BY_UNIQUE_KEY, unique_key=["ds"])) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + # b is warn-only and unchanged by this plan, so c is consistent with it: no second warning either. + assert not plan.restatements + console.log_warning.assert_called_once() + assert ( + "models downstream of them are not restated through them" + in console.log_warning.call_args[0][0] + ) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_flows_through_child_that_is_backfilled_in_this_plan(make_snapshot): + # b has no intervals at all so it is backfilled for the added range, and c was computed from the old b. + specs = _chain_specs(intervals=[]) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + assert set(plan.restatements) == {snaps["c"].snapshot_id} + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_ignores_unresolvable_inferred_start(make_snapshot): + # b has no start and one of its old parents is gone, so its old start can't be known. + specs = _chain_specs() + specs[1].update(start=None) + specs[2].update(start=None) + context_diff, snaps, olds = _start_extension_context_diff(make_snapshot, specs) + # a is no longer reported as modified, so the old version of a that b was derived from can't be found. + context_diff.modified_snapshots = { + name: pair + for name, pair in context_diff.modified_snapshots.items() + if name != snaps["a"].name + } + assert olds["a"].snapshot_id not in context_diff.snapshots + + plan = PlanBuilder(context_diff).build() + assert not plan.restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_before_plan_start_is_ignored(make_snapshot): + context_diff, _, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + assert not PlanBuilder(context_diff, is_dev=True, start="2023-01-02").build().restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_within_the_same_interval_unit_is_ignored(make_snapshot): + context_diff, _, _ = _start_extension_context_diff( + make_snapshot, _chain_specs(a_old="2023-01-01 12:00:00", a_new="2023-01-01 00:00:00") + ) + assert not PlanBuilder(context_diff).build().restatements + + +def test_evaluatable_plan_is_restatement_ignores_start_extensions(make_snapshot): + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + with time_machine.travel(_EXECUTION_TIME): + plan = PlanBuilder(context_diff).build() + evaluatable = plan.to_evaluatable() + assert evaluatable.restatements + assert evaluatable.start_extension_restatements == {snaps["b"].name, snaps["c"].name} + assert not evaluatable.is_restatement + + explicit = PlanBuilder(context_diff, restate_models={snaps["a"].name}).build() + assert explicit.to_evaluatable().is_restatement + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_with_breaking_change_does_not_restate(make_snapshot): + specs = _chain_specs() + specs[0]["new"]["query"] = parse_one("select 1 as ds, 2 as extra") + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + builder = PlanBuilder(context_diff, auto_categorization_enabled=False) + builder.set_choice(snaps["a"], SnapshotChangeCategory.BREAKING) + plan = builder.build() + + # Downstream models get new versions and are rebuilt from scratch: no extra work is needed. + assert snaps["b"].change_category == SnapshotChangeCategory.INDIRECT_BREAKING + assert not plan.restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_with_non_breaking_change_restates_extension_only(make_snapshot): + specs = _chain_specs() + specs[0]["new"]["query"] = parse_one("select 1 as ds, 2 as extra") + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + builder = PlanBuilder(context_diff, auto_categorization_enabled=False) + builder.set_choice(snaps["a"], SnapshotChangeCategory.NON_BREAKING) + plan = builder.build() + + assert snaps["b"].change_category == SnapshotChangeCategory.INDIRECT_NON_BREAKING + expected = (_ts("2023-01-01"), _ts("2023-01-02")) + assert plan.restatements == { + snaps["b"].snapshot_id: expected, + snaps["c"].snapshot_id: expected, + } + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_inferred_start_is_resolved_through_parents(make_snapshot): + # b has no explicit start so its start moves together with a's. It has no intervals in the + # affected range itself, but c (explicit start) does and is derived from b. + specs = _chain_specs() + specs[1].update(start=None, intervals=[("2023-01-02", "2023-01-03")]) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert set(plan.restatements) == {snaps["c"].snapshot_id} + assert plan.restatements[snaps["c"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-02")) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_depends_on_past_downstream_restates_through_execution_time( + make_snapshot, +): + specs = _chain_specs() + specs[1]["self_referencing"] = True + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].depends_on_past + start, end = plan.restatements[snaps["b"].snapshot_id] + assert start == _ts("2023-01-01") + assert end >= _ts(_EXECUTION_TIME) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_forward_only_downstream_follows_regular_rules(make_snapshot): + specs = _chain_specs(kind=IncrementalByTimeRangeKind(time_column="ds", forward_only=True)) + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + assert snaps["b"].snapshot_id in plan.restatements + assert snaps["c"].snapshot_id in plan.restatements + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_merges_with_explicit_restatement(make_snapshot): + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + plan = PlanBuilder( + context_diff, restate_models={snaps["a"].name}, start="2023-01-01", end="2023-01-03" + ).build() + + # The extension range is covered by the explicit range: there is a single, merged range per model. + assert plan.restatements[snaps["a"].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-04")) + for name in ("b", "c"): + assert plan.restatements[snaps[name].snapshot_id] == (_ts("2023-01-01"), _ts("2023-01-04")) + # Only explicitly requested models are listed as such. + assert plan.selected_models_to_restate == {snaps["a"].name} + assert plan.start_extension_restatements == set() + + +@time_machine.travel(_EXECUTION_TIME) +@pytest.mark.parametrize("flag", ["select_model", "skip_backfill", "empty_backfill"]) +def test_start_extension_warns_instead_of_restating_when_backfill_is_limited( + make_snapshot, mocker: MockerFixture, flag: str +): + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + kwargs: t.Dict[str, t.Any] = ( + {"backfill_models": {snaps["a"].name}} if flag == "select_model" else {flag: True} + ) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console, **kwargs).build() + + assert not plan.restatements + message = console.log_warning.call_args[0][0] + assert snaps["b"].name in message and snaps["c"].name in message + assert "--restate-model" in message + assert "--start 2023-01-01 --end 2023-01-01" in message + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_with_select_model_restates_selected_downstream( + make_snapshot, mocker: MockerFixture +): + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, _chain_specs()) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console, backfill_models={snaps["b"].name}).build() + + # b is selected (c isn't), so only b is restated and the plan warns about c. + assert set(plan.restatements) == {snaps["b"].snapshot_id} + message = console.log_warning.call_args[0][0] + assert snaps["c"].name in message and snaps["b"].name not in message + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_in_diamond_flows_through_unguarded_branch( + make_snapshot, mocker: MockerFixture +): + # a -> b (restatement disabled) -> d and a -> c -> d: d is restated through c only. + intervals = [("2020-01-01", "2023-01-03")] + specs: t.List[t.Dict[str, t.Any]] = [ + { + "name": "a", + "old": {"start": "2023-01-02"}, + "new": {"start": "2023-01-01"}, + "intervals": [("2023-01-02", "2023-01-03")], + }, + { + "name": "b", + "parent": "a", + "kind": IncrementalByTimeRangeKind(time_column="ds", disable_restatement=True), + "intervals": intervals, + }, + {"name": "c", "parent": "a", "intervals": intervals}, + {"name": "d", "parents": ["b", "c"], "intervals": intervals}, + ] + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + day = (_ts("2023-01-01"), _ts("2023-01-02")) + assert plan.restatements == {snaps["c"].snapshot_id: day, snaps["d"].snapshot_id: day} + console.log_warning.assert_called_once() + assert snaps["b"].name in console.log_warning.call_args[0][0] + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_cut_by_plan_start_warns_in_prod_only(make_snapshot, mocker: MockerFixture): + # The range [01-01, 01-03) is cut to [01-02, 01-03) by the plan start. The remainder won't be + # detected again once this plan is applied. + def build(is_dev: bool, console) -> Plan: + context_diff, snaps, _ = _start_extension_context_diff( + make_snapshot, + _chain_specs(a_old="2023-01-03", a_new="2023-01-01"), + environment="dev" if is_dev else "prod", + ) + return PlanBuilder( + context_diff, + is_dev=is_dev, + start="2023-01-02", + restate_models=None if is_dev else {snaps["a"].name}, + console=console, + ).build() + + console = mocker.Mock() + build(False, console) + console.log_warning.assert_called_once() + message = console.log_warning.call_args[0][0] + assert "--restate-model '\"b\"' --restate-model '\"c\"'" in message + assert "--start 2023-01-01 --end 2023-01-01" in message + + console = mocker.Mock() + build(True, console) + assert not any("cut off" in call[0][0] for call in console.log_warning.call_args_list) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_depends_on_past_propagates_expanded_range_to_descendants( + make_snapshot, +): + specs = [ + { + "name": "a", + "old": {"start": "2023-01-02"}, + "new": {"start": "2023-01-01"}, + "intervals": [("2023-01-02", "2023-01-03")], + }, + { + "name": "b", + "parent": "a", + "self_referencing": True, + "intervals": [("2020-01-01", "2023-01-03")], + }, + { + "name": "c", + "parent": "b", + "start": "2023-01-03", + "intervals": [("2023-01-03", "2023-01-05")], + }, + { + "name": "v", + "parent": "b", + "kind": ViewKind(), + }, + { + "name": "d", + "parent": "v", + "start": "2023-01-03", + "intervals": [("2023-01-03", "2023-01-05")], + }, + ] + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + plan = PlanBuilder(context_diff).build() + + assert snaps["b"].snapshot_id in plan.restatements + assert snaps["c"].snapshot_id in plan.restatements + assert snaps["d"].snapshot_id in plan.restatements + + c_start, c_end = plan.restatements[snaps["c"].snapshot_id] + assert c_start == _ts("2023-01-01") + assert c_end >= _ts(_EXECUTION_TIME) + + d_start, d_end = plan.restatements[snaps["d"].snapshot_id] + assert d_start == _ts("2023-01-01") + assert d_end >= _ts(_EXECUTION_TIME) + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_full_model_does_not_over_expand_downstream( + make_snapshot, mocker: MockerFixture +): + specs: t.List[t.Dict[str, t.Any]] = [ + { + "name": "a", + "old": {"start": "2023-01-02"}, + "new": {"start": "2023-01-01"}, + "intervals": [("2023-01-02", "2023-01-03")], + }, + { + "name": "b", + "parent": "a", + "kind": FullKind(), + "intervals": [("2020-01-01", "2023-01-03")], + }, + { + "name": "c", + "parent": "b", + "start": "2020-01-01", + "intervals": [("2020-01-01", "2023-01-03")], + }, + ] + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + assert snaps["b"].snapshot_id in plan.restatements + assert snaps["c"].snapshot_id in plan.restatements + c_start, c_end = plan.restatements[snaps["c"].snapshot_id] + assert c_start == _ts("2023-01-01") + assert c_end == _ts("2023-01-02") + + +def test_align_to_interval_unit_independent_start_and_end(make_snapshot): + from sqlmesh.core.plan.builder import _align_to_interval_unit + + model = SqlModel(name="daily_model", query=parse_one("SELECT 1 as id, ds"), cron="@daily") + snap = make_snapshot(model) + + start = _ts("2023-01-01 12:00:00") + end = _ts("2023-01-02 00:00:00") + aligned_start, aligned_end = _align_to_interval_unit(snap, start, end) + assert aligned_start == _ts("2023-01-01 00:00:00") + assert aligned_end == _ts("2023-01-02 00:00:00") + + end_midday = _ts("2023-01-02 12:00:00") + aligned_start, aligned_end = _align_to_interval_unit(snap, start, end_midday) + assert aligned_start == _ts("2023-01-01 00:00:00") + assert aligned_end == _ts("2023-01-03 00:00:00") + + +@time_machine.travel(_EXECUTION_TIME) +def test_start_extension_retained_pre_start_intervals_no_key_error( + make_snapshot, mocker: MockerFixture +): + specs: t.List[t.Dict[str, t.Any]] = [ + { + "name": "a", + "old": {"start": "2023-01-02"}, + "new": {"start": "2023-01-01"}, + "intervals": [("2023-01-02", "2023-01-03")], + }, + { + "name": "b", + "parent": "a", + "start": "2023-01-05", + "intervals": [("2023-01-01", "2023-01-03"), ("2023-01-05", "2023-01-10")], + }, + ] + context_diff, snaps, _ = _start_extension_context_diff(make_snapshot, specs) + console = mocker.Mock() + plan = PlanBuilder(context_diff, console=console).build() + + assert snaps["b"].snapshot_id not in plan.restatements + console.log_warning.assert_called_once() + assert snaps["b"].name in console.log_warning.call_args[0][0] + + +def test_is_start_resolvable_memoization_diamond_dag(make_snapshot): + import time + from sqlmesh.core.plan.builder import _is_start_resolvable + + snapshots = {} + all_nodes = {} + root = make_snapshot( + SqlModel(name="root", query=parse_one("SELECT 1 as ds"), start="2023-01-01") + ) + snapshots[root.snapshot_id] = root + all_nodes[root.name] = root.model + prev_layer = [root] + + for level in range(25): + next_layer = [] + for i in range(2): + node = make_snapshot( + SqlModel( + name=f"level_{level}_node_{i}", + query=parse_one( + " UNION ALL ".join(f"SELECT ds FROM {p.name}" for p in prev_layer) + ), + ), + nodes=all_nodes, + ) + snapshots[node.snapshot_id] = node + next_layer.append(node) + for n in next_layer: + all_nodes[n.name] = n.model + prev_layer = next_layer + + leaf = prev_layer[0] + cache = {} + t0 = time.perf_counter() + res = _is_start_resolvable(leaf, snapshots, cache=cache) + duration = time.perf_counter() - t0 + + assert res is True + assert duration < 0.2 + assert leaf.snapshot_id in cache diff --git a/tests/core/test_signal.py b/tests/core/test_signal.py new file mode 100644 index 0000000000..b6143b43e5 --- /dev/null +++ b/tests/core/test_signal.py @@ -0,0 +1,123 @@ +import pytest +from unittest.mock import MagicMock +from sqlglot import parse_one +from sqlmesh.core.context import ExecutionContext +from sqlmesh.core.model import SqlModel, IncrementalByTimeRangeKind +from sqlmesh.core.signal import freshness +from sqlmesh.core.snapshot import SnapshotChangeCategory +from sqlmesh.utils.date import to_datetime, to_timestamp + + +@pytest.fixture +def test_snapshot(make_snapshot): + model = SqlModel( + name="test_model", + query=parse_one("SELECT 1 as id, ds"), + kind=IncrementalByTimeRangeKind(time_column="ds"), + cron="@daily", + start="2023-01-01", + ) + snap = make_snapshot(model) + snap.categorize_as(SnapshotChangeCategory.BREAKING) + snap.last_altered_ts = to_timestamp("2023-01-03") + return snap + + +def test_freshness_signal_evaluates_restated_interval(test_snapshot): + adapter = MagicMock() + adapter.SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True + + context = ExecutionContext( + engine_adapter=adapter, + snapshots={"test_model": test_snapshot}, + is_restatement=False, + parent_intervals=[], + restated_interval=(to_timestamp("2023-01-01"), to_timestamp("2023-01-02")), + ) + + batch = [ + (to_datetime("2023-01-01"), to_datetime("2023-01-02")), + ] + + result = freshness(batch, test_snapshot, context) + assert result == batch + + +def test_freshness_signal_mixed_batch_filters_unready_intervals(test_snapshot): + adapter = MagicMock() + adapter.SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True + + context = ExecutionContext( + engine_adapter=adapter, + snapshots={"test_model": test_snapshot}, + is_restatement=False, + parent_intervals=[], + restated_interval=(to_timestamp("2023-01-01"), to_timestamp("2023-01-02")), + ) + + batch = [ + (to_datetime("2023-01-01"), to_datetime("2023-01-02")), + (to_datetime("2023-01-05"), to_datetime("2023-01-06")), + ] + + result = freshness(batch, test_snapshot, context) + # Only the interval overlapping restated_interval should be ready + assert result == [(to_datetime("2023-01-01"), to_datetime("2023-01-02"))] + + +def test_freshness_signal_returns_false_when_no_restatement_and_unaltered(test_snapshot): + adapter = MagicMock() + adapter.SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True + + context = ExecutionContext( + engine_adapter=adapter, + snapshots={"test_model": test_snapshot}, + is_restatement=False, + parent_intervals=[], + restated_interval=None, + ) + + batch = [ + (to_datetime("2023-01-01"), to_datetime("2023-01-02")), + ] + + result = freshness(batch, test_snapshot, context) + assert result is False + + +def test_freshness_signal_all_ready_when_is_restatement_is_true(test_snapshot): + adapter = MagicMock() + adapter.SUPPORTS_METADATA_TABLE_LAST_MODIFIED_TS = True + + context = ExecutionContext( + engine_adapter=adapter, + snapshots={"test_model": test_snapshot}, + is_restatement=True, + ) + + batch = [ + (to_datetime("2023-01-01"), to_datetime("2023-01-02")), + ] + + result = freshness(batch, test_snapshot, context) + assert result is True + + +def test_execution_context_with_variables_preserves_signal_context(test_snapshot): + adapter = MagicMock() + restated_interval = (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + parent_intervals = [(to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))] + + context = ExecutionContext( + engine_adapter=adapter, + snapshots={"test_model": test_snapshot}, + is_restatement=False, + parent_intervals=parent_intervals, + restated_interval=restated_interval, + ) + + new_context = context.with_variables({"my_var": "val"}) + + assert new_context.parent_intervals == parent_intervals + assert new_context.restated_interval == restated_interval + assert new_context.var("my_var") == "val" diff --git a/tests/integrations/github/cicd/test_github_controller.py b/tests/integrations/github/cicd/test_github_controller.py index dbb09f4336..432cde3273 100644 --- a/tests/integrations/github/cicd/test_github_controller.py +++ b/tests/integrations/github/cicd/test_github_controller.py @@ -961,3 +961,27 @@ def test_chunk_up_api_message_preserves_multibyte_chars( assert "".join(chunks) == message # Each chunk still respects the GitHub API byte limit. assert all(len(chunk.encode("utf-8")) <= max_bytes for chunk in chunks) + + +@time_machine.travel("2025-07-07 00:00:00 UTC", tick=False) +def test_get_plan_summary_for_start_extension_restatements_shows_differences( + github_client, make_controller: t.Callable[..., GithubController] +): + controller = make_controller( + "tests/fixtures/github/pull_request_synchronized.json", + github_client, + mock_out_context=False, + ) + plan = controller.prod_plan + s_id = next(iter(plan.snapshots)) + + explicit = plan.copy(update={"restatements": {s_id: (0, 1)}}) + assert "Restating models" in controller.get_plan_summary(explicit) + + # Restatements that only exist because a start moved earlier don't replace the diff summary. + extension = plan.copy( + update={"restatements": {s_id: (0, 1)}, "start_extension_restatements": {s_id}} + ) + summary = controller.get_plan_summary(extension) + assert "Restating models" not in summary + assert "**Added Models:**" in summary