From c5d94b790f84d8a5837025deb95b17528cb43af8 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Mon, 16 Nov 2026 14:07:34 +0000 Subject: [PATCH 1/3] Prepare no sandbox for expired input A never-allocated Environment whose initial input expired, with no live input, is no longer placement demand; it waits again from its live input's creation once new input arrives. Provisioning rechecks the same query before creating a workspace for a new or released allocation, and ReserveAllocation rechecks it under the deployment lock; a failed recheck releases that attempt's node reservation like any failed preparation. --- contracts/agents-api/sandbox-deployment.md | 2 +- contracts/agents-api/zh/sandbox-deployment.md | 4 +- .../db/queries/runtime_lifecycle_nodes.sql | 28 ++----- services/core/internal/db/sqlc/models.go | 9 +++ .../db/sqlc/runtime_lifecycle_nodes.sql.go | 36 ++++----- .../core/internal/deployment/allocations.go | 7 ++ .../internal/deployment/allocations_test.go | 9 +++ .../core/internal/deployment/fakes_test.go | 3 + services/core/internal/deployment/storage.go | 4 + .../execution/deployment_fixture_test.go | 9 +++ .../execution/runtime_expired_input_test.go | 69 +++++++++++++++++ .../internal/execution/runtime_lifecycle.go | 10 +++ .../postgres/deploymentpg/allocations.go | 14 ++++ .../deploymentpg/placement_demand_test.go | 77 +++++++++++++++++++ .../postgres/deploymentpg/replacement_test.go | 8 ++ .../000016_placement_demand_view.sql | 26 +++++++ 16 files changed, 268 insertions(+), 47 deletions(-) create mode 100644 services/core/internal/execution/runtime_expired_input_test.go create mode 100644 services/core/migrations/000016_placement_demand_view.sql diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index ea02e5307..601f9be8c 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -170,7 +170,7 @@ A node is [online](./node-generation-protocol.md#connection) when it is connecte Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed or offline nodes alone do not keep polling active. -Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. +Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. An Environment that was never placed stops waiting for capacity when its initial input expires, until new input arrives, and a reservation whose input expired before preparation prepares nothing and is released. ## Reset diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 15c8bb616..68beaa603 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: b405b74e9869ae6e6d9dc125a2eceb6a8eb1fe87a873e19e0d13884d4754ec7a +source_hash: b86c842977d47bd394ff65d0759fddc8c455da7868bf88093401ee317eac62e1 --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商和每个沙箱的资源。PostgreSQL 为每个安装维护一个当前有效选择;Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径,且不能覆盖其资源。Runtime 不属于该选择:每个节点运行其所属安装验证过的 Runtime。该选择独立于 Harness。部署可以保持未配置状态,没有节点;此时它拒绝托管准入。 @@ -173,7 +173,7 @@ flowchart TD 每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`,其中 `ready_generation` 是可为 null 的持久服务固定状态,`diagnostic` 是目标代次的固定代码;分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时,才每五秒轮询一次;旧 Session 以及失败或离线的节点本身均不会使轮询保持活动状态。 -节点模式创建先验证目标部署,提交 Session、pending Environment 及初始输入,但不预留计算资源。节点全部已满、离线或准备中时,已接受的工作继续等待;未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描,分页按需求记录时间和 Environment ID 排序,每轮使用固定时间上界,避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性,然后优先选择最新的合格固定代次。预留后的 placement 保持不可变,直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求,并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer),通过同一容量锁预留符合条件的检查点容量;这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations),等待容量的时间也计入其中。 +节点模式创建先验证目标部署,提交 Session、pending Environment 及初始输入,但不预留计算资源。节点全部已满、离线或准备中时,已接受的工作继续等待;未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描,分页按需求记录时间和 Environment ID 排序,每轮使用固定时间上界,避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性,然后优先选择最新的合格固定代次。预留后的 placement 保持不可变,直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求,并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer),通过同一容量锁预留符合条件的检查点容量;这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations),等待容量的时间也计入其中。从未放置的 Environment 在其初始输入过期后不再等待容量,直到新输入到达;输入在准备之前已过期的预留不做任何准备并被释放。 如果准备工作在新 allocation 预留前失败,Core 仅释放该次尝试的节点预留,并保留已有文件系统身份。其下一次放置需求按不早于释放的时间排序,排在已经等待的工作之后;成功重试使用新选中代的 specification。计算结果不确定的 allocation 继续保留其预留。 diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index e91cf6147..584eea156 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -41,27 +41,11 @@ LEFT JOIN runtime_deployment_generations g ON g.generation=COALESCE(a.deployment WHERE e.tenant_id=$1 AND e.id=$2; -- name: ListPlacementDemand :many -WITH demand AS ( -SELECT e.id, e.tenant_id, COALESCE(s.engine, '')::text AS engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, - GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN e.created_at - ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at -FROM environments e JOIN environment_liveness l ON l.id = e.id AND l.active -LEFT JOIN sessions s ON s.environment_id = e.id AND s.deleted_at IS NULL -LEFT JOIN runtime_placements p ON p.environment_id = e.id -LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true -WHERE e.configuration->>'type' = 'openai_hosted' - AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) - AND NOT EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') - AND ( - (e.initialization IN ('pending','complete') AND latest.id IS NULL) - OR (e.initialization = 'complete' - AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') - AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') - AND (latest.compute_wake_requested OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()))) - ) -) -SELECT id, tenant_id, engine, retained, demanded_at, COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM demand -WHERE (demanded_at, id) > (sqlc.arg(after_time)::timestamptz, sqlc.arg(after_id)::uuid) +SELECT id, tenant_id, engine, retained, demanded_at, COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM runtime_placement_demand +WHERE NOT reserved AND (demanded_at, id) > (sqlc.arg(after_time)::timestamptz, sqlc.arg(after_id)::uuid) AND demanded_at <= COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp()) ORDER BY demanded_at, id LIMIT 32; + +-- name: IsPlacementDemanded :one +-- A plain id predicate keeps an index plan under a cached generic plan; the scan's statement could not. +SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1)::boolean; diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index e0f29b68f..41a3c861c 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -385,6 +385,15 @@ type RuntimePlacement struct { DeploymentGeneration pgtype.Int8 `json:"deployment_generation"` } +type RuntimePlacementDemand struct { + ID pgtype.UUID `json:"id"` + TenantID pgtype.UUID `json:"tenant_id"` + Engine string `json:"engine"` + Retained bool `json:"retained"` + DemandedAt pgtype.Timestamptz `json:"demanded_at"` + Reserved pgtype.Bool `json:"reserved"` +} + type RuntimeResetRetainedEnvironment struct { EnvironmentID pgtype.UUID `json:"environment_id"` SessionID pgtype.UUID `json:"session_id"` diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index ee5255673..64a7434b3 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -56,29 +56,21 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti return i, err } +const isPlacementDemanded = `-- name: IsPlacementDemanded :one +SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1)::boolean +` + +// A plain id predicate keeps an index plan under a cached generic plan; the scan's statement could not. +func (q *Queries) IsPlacementDemanded(ctx context.Context, id pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, isPlacementDemanded, id) + var column_1 bool + err := row.Scan(&column_1) + return column_1, err +} + const listPlacementDemand = `-- name: ListPlacementDemand :many -WITH demand AS ( -SELECT e.id, e.tenant_id, COALESCE(s.engine, '')::text AS engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, - GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN e.created_at - ELSE LEAST((SELECT min(r.created_at) FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()), CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at -FROM environments e JOIN environment_liveness l ON l.id = e.id AND l.active -LEFT JOIN sessions s ON s.environment_id = e.id AND s.deleted_at IS NULL -LEFT JOIN runtime_placements p ON p.environment_id = e.id -LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true -WHERE e.configuration->>'type' = 'openai_hosted' - AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) - AND NOT EXISTS (SELECT 1 FROM runtime_placements p WHERE p.environment_id = e.id AND p.released_at IS NULL) - AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') - AND ( - (e.initialization IN ('pending','complete') AND latest.id IS NULL) - OR (e.initialization = 'complete' - AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') - AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') - AND (latest.compute_wake_requested OR EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()))) - ) -) -SELECT id, tenant_id, engine, retained, demanded_at, COALESCE($1::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM demand -WHERE (demanded_at, id) > ($2::timestamptz, $3::uuid) +SELECT id, tenant_id, engine, retained, demanded_at, COALESCE($1::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM runtime_placement_demand +WHERE NOT reserved AND (demanded_at, id) > ($2::timestamptz, $3::uuid) AND demanded_at <= COALESCE($1::timestamptz, statement_timestamp()) ORDER BY demanded_at, id LIMIT 32 ` diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 5e737ec12..2448753b6 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -84,6 +84,13 @@ func (e *ExecutionOperations) ReserveAllocation(ctx context.Context, key Allocat if err := placement.CheckReserved(reserved); err != nil { return err } + // Input may have expired since the placement was reserved; nothing exists to start for it. + if demanded, err := tx.LoadPlacementDemanded(); err != nil || !demanded { + if err == nil { + err = ErrAllocationConflict + } + return err + } allocation.NodeID, allocation.Generation = reserved.NodeID, reserved.Generation } device := sessions.ExecutionDevice{ID: allocation.DeviceID, Name: registration.Name, EnvironmentID: environment.ID} diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 9dee7a540..883c51420 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -28,6 +28,7 @@ type fakeReservationTx struct { findAllocation func() (Allocation, bool, error) lockDeployment func() (placement.Deployment, error) loadReserved func() (placement.Reserved, error) + placementDemanded func() (bool, error) insertAllocation func(NewAllocation) (Allocation, error) } @@ -73,6 +74,14 @@ func (f *fakeReservationTx) LoadReserved() (placement.Reserved, error) { return f.loadReserved() } +// LoadPlacementDemanded answers true unless a test sets placementDemanded. +func (f *fakeReservationTx) LoadPlacementDemanded() (bool, error) { + if f.placementDemanded == nil { + return true, nil + } + return f.placementDemanded() +} + func (f *fakeReservationTx) CanReplaceAllocation() (bool, error) { unexpected(f.t, "CanReplaceAllocation") return false, nil diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index 514e3ac7e..f4008b540 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -638,6 +638,9 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st return sessions.ManagedArchive{}, nil } +func (f *fakeReader) PlacementDemanded(context.Context, AllocationKey) (bool, error) { + panic("unexpected PlacementDemanded") +} func (f *fakeReader) PlacementDemand(context.Context, PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) { panic("unexpected PlacementDemand") } diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index d85067937..cc3e2a9da 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -104,6 +104,8 @@ type ReservationTx interface { LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) // LoadNodes reads eligible placement facts under the deployment lock. LoadNodes() ([]placement.Node, error) + // LoadPlacementDemanded reports whether the Environment is still placement demand, counting one that already reserved a placement: the runtime_placement_demand view the scan uses. + LoadPlacementDemanded() (bool, error) // ReservePlacement inserts a first reservation or replaces a released reservation without changing historical receipts. ReservePlacement(placement.Placement) error // ReleaseFailedPlacement yields only the exact reservation without active compute. @@ -273,6 +275,8 @@ type Reader interface { RetainedNativeHistory(ctx context.Context, key AllocationKey) (bool, error) // PlacementDemand lists up to 32 unreserved first or retained requests in demand-time and ID order. PlacementDemand(ctx context.Context, after PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) + // PlacementDemanded reports whether the Environment is placement demand with the same definition as PlacementDemand, counting one that already reserved a placement. + PlacementDemanded(ctx context.Context, key AllocationKey) (bool, error) // LifecyclePlacement returns what routing the Environment to its // lifecycle reads, including for a deleted Session. A missing Environment // is sessions.ErrNotFound. diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index e1afdb0dd..7ba42d5b3 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -142,6 +142,7 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { + placementDemanded func(context.Context, deployment.AllocationKey) (bool, error) placementDemand func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) t *testing.T @@ -361,6 +362,14 @@ func (r *strictDeploymentReader) PlacementDemand(ctx context.Context, after depl } return r.placementDemand(ctx, after) } + +// PlacementDemanded answers true unless a test sets placementDemanded: provisioning tests reach it with real demand. +func (r *strictDeploymentReader) PlacementDemanded(ctx context.Context, key deployment.AllocationKey) (bool, error) { + if r.placementDemanded == nil { + return true, nil + } + return r.placementDemanded(ctx, key) +} func (r *strictDeploymentReader) RetainedNativeHistory(ctx context.Context, key deployment.AllocationKey) (bool, error) { if r.retainedNativeHistory == nil { return false, unexpectedDeploymentCall(r.t, "RetainedNativeHistory") diff --git a/services/core/internal/execution/runtime_expired_input_test.go b/services/core/internal/execution/runtime_expired_input_test.go new file mode 100644 index 000000000..4504b6339 --- /dev/null +++ b/services/core/internal/execution/runtime_expired_input_test.go @@ -0,0 +1,69 @@ +package execution + +import ( + "context" + "errors" + "testing" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// demandReader answers the placement demand recheck and may report a released allocation. +type demandReader struct { + deployment.Reader + demanded bool + released bool + checks int +} + +func (r *demandReader) PlacementDemanded(context.Context, deployment.AllocationKey) (bool, error) { + r.checks++ + return r.demanded, nil +} + +func (r *demandReader) EnvironmentAllocation(ctx context.Context, key deployment.AllocationKey) (deployment.Allocation, error) { + if r.released { + return deployment.Allocation{State: "released"}, nil + } + return r.Reader.EnvironmentAllocation(ctx, key) +} + +// Input that expired after the placement was reserved prepares no workspace or sandbox and gives the reserved node back. +func TestInputExpiringAfterPlacementPreparesNothing(t *testing.T) { + for name, tc := range map[string]struct{ released bool }{"never allocated": {}, "released allocation": {released: true}} { + t.Run(name, func(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, s := f.lifecycle, f.session + r.config.Mode = string(sandbox.DeploymentNodes) + reader := &demandReader{Reader: r.reader, released: tc.released} + r.reader = reader + _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal(err) + } + var workspaces, allocations int + if err := f.pool.QueryRow(t.Context(), `SELECT (SELECT count(*) FROM environment_workspaces),(SELECT count(*) FROM runtime_allocations)`).Scan(&workspaces, &allocations); err != nil { + t.Fatal(err) + } + target, err := reader.Reader.LifecyclePlacement(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}) + if err != nil || workspaces != 0 || allocations != 0 || provider.creates != 0 || !target.PlacementReleased { + t.Fatal("prepared for expired input", workspaces, allocations, provider.creates, target, err) + } + }) + } +} + +// A direct deployment has no placement demand to check. +func TestDirectDeploymentSkipsPlacementDemandCheck(t *testing.T) { + f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + r, s := f.lifecycle, f.session + r.config.Mode = string(sandbox.DeploymentDirect) + reader := &demandReader{Reader: r.reader} + r.reader = reader + _, _ = r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if reader.checks != 0 { + t.Fatal("direct deployment checked placement demand", reader.checks) + } +} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 104bca84e..d982792bc 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -238,6 +238,16 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p if lookupErr != nil && !errors.Is(lookupErr, deployment.ErrNotFound) { return deployment.Allocation{}, lookupErr } + // Input that expired since the placement was reserved prepares nothing. A direct deployment has no placement demand. + if r.config.Mode == string(sandbox.DeploymentNodes) && (errors.Is(lookupErr, deployment.ErrNotFound) || existing.State == "released") { + demanded, demandErr := r.reader.PlacementDemanded(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}) + if demandErr != nil { + return deployment.Allocation{}, demandErr + } + if !demanded { + return deployment.Allocation{}, deployment.ErrAllocationConflict + } + } if errors.Is(lookupErr, deployment.ErrNotFound) { workspace, err = r.workspaces.Ensure(ctx, tenant, environment, r.config.WorkspaceRequirements, spec.Resources.EnvironmentDiskMiB) if err == nil && workspace == nil { diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 903739afa..092d07761 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -214,6 +214,10 @@ func (t *reservationTx) LoadReserved() (placement.Reserved, error) { return reserved, err } +func (t *reservationTx) LoadPlacementDemanded() (bool, error) { + return t.q.IsPlacementDemanded(t.ctx, t.environment) +} + func (t *reservationTx) CanReplaceAllocation() (bool, error) { current, found, err := t.FindAllocation() if err != nil || !found { @@ -628,6 +632,16 @@ func (s *Store) PlacementDemand(ctx context.Context, after deployment.PlacementD return loadPlacementDemand(ctx, s.pool.Queries(), after) } +func (s *Store) PlacementDemanded(ctx context.Context, key deployment.AllocationKey) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) + defer cancel() + _, environment, err := allocationKey(key) + if err != nil { + return false, err + } + return s.pool.Queries().IsPlacementDemanded(ctx, environment) +} + func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.AllocationKey) (deployment.LifecyclePlacement, error) { tenant, environment, err := allocationKey(key) if err != nil { diff --git a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go index b3dbf6afe..2c4ff69ad 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go @@ -2,6 +2,7 @@ package deploymentpg_test import ( "errors" + "github.com/google/uuid" "testing" "time" @@ -213,3 +214,79 @@ func TestPlacementDemandWrapsDespiteContinuousNewArrivals(t *testing.T) { t.Fatal(again, err) } } + +// A never-allocated Environment waits for a node from its creation, unless its initial input expired: then only a live input demands one, from that input's creation. +func TestPlacementDemandStopsWhenInitialInputExpires(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + now := time.Now().UTC().Truncate(time.Microsecond) + input := func(key deployment.AllocationKey, initial bool, state string, created time.Time, deadline time.Duration) { + t.Helper() + settled := "NULL" + if state != "pending" { + settled = "clock_timestamp()" + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,is_initial,settings,created_at,deadline,state,settled_at) SELECT $1,id,$2,'[{}]',$3,'{}',$4,clock_timestamp()+$5::interval,$6,`+settled+` FROM sessions WHERE environment_id=$7`, uuid.NewString(), uuid.NewString(), initial, created, deadline.String(), state, key.EnvironmentID); err != nil { + t.Fatal(err) + } + } + idle, expired, expiredUnswept, revived, pending := pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f) + livedAt := now.Add(-time.Minute) + input(expired, true, "expired", now.Add(-time.Hour), -time.Hour) + input(expiredUnswept, true, "pending", now.Add(-time.Hour), -time.Minute) + input(revived, true, "expired", now.Add(-time.Hour), -time.Hour) + input(revived, false, "pending", livedAt, time.Minute) + input(pending, true, "pending", now.Add(-time.Second), time.Minute) + for _, key := range []deployment.AllocationKey{idle, expired, expiredUnswept, revived, pending} { + if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET created_at=$2 WHERE id=$1`, key.EnvironmentID, now.Add(-2*time.Hour)); err != nil { + t.Fatal(err) + } + } + demand, _, err := f.adapter.PlacementDemand(t.Context(), deployment.PlacementDemandCursor{}) + if err != nil { + t.Fatal(err) + } + at := map[string]time.Time{} + for _, row := range demand { + at[row.ID] = row.At + } + want := map[string]time.Time{idle.EnvironmentID: now.Add(-2 * time.Hour), revived.EnvironmentID: livedAt, pending.EnvironmentID: now.Add(-2 * time.Hour)} + if len(at) != len(want) { + t.Fatal("demand", at) + } + for id, when := range want { + if !at[id].Equal(when) { + t.Fatal("demanded at", id, at[id], "want", when) + } + } + // The recheck reads the same rule. + for key, wanted := range map[deployment.AllocationKey]bool{idle: true, expired: false, expiredUnswept: false, revived: true, pending: true} { + if got, err := f.adapter.PlacementDemanded(t.Context(), key); err != nil || got != wanted { + t.Fatal("PlacementDemanded", key.EnvironmentID, got, wanted, err) + } + } +} + +// Input that expires after the placement was reserved creates no allocation. +func TestReserveAllocationRefusesExpiredInitialInput(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 1}) + f.connect(t, node.NodeID) + key := pendingHostedEnvironment(t, f) + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,is_initial,settings,created_at,deadline,state,settled_at) SELECT $1,id,$2,'[{}]',true,'{}',clock_timestamp(),clock_timestamp(),'expired',clock_timestamp() FROM sessions WHERE environment_id=$3`, uuid.NewString(), uuid.NewString(), key.EnvironmentID); err != nil { + t.Fatal(err) + } + if _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()); !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal("expired input reserved an allocation", err) + } + var allocations, devices int + if err := f.pool.QueryRow(t.Context(), `SELECT (SELECT count(*) FROM runtime_allocations),(SELECT count(*) FROM devices)`).Scan(&allocations, &devices); err != nil || allocations != 0 || devices != 0 { + t.Fatal(allocations, devices, err) + } +} diff --git a/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go b/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go index 28a79affb..9a4f22d2b 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/replacement_test.go @@ -66,6 +66,9 @@ func TestReplacementKeepsSessionIdentityAndFencesPriorOwner(t *testing.T) { if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_nodes SET connection_id=NULL WHERE id=$1`, firstNode.NodeID); err != nil { t.Fatal(err) } + if err := f.service.TouchActivity(t.Context(), key.TenantID, key.EnvironmentID); err != nil { + t.Fatal(err) + } reserved, err := changes.EnsurePlacement(t.Context(), key, installation) if err != nil || reserved.NodeID != secondNode.NodeID { t.Fatal(reserved, err) @@ -215,6 +218,11 @@ func TestTenRetainedWorkspacesShareThreeComputeSlots(t *testing.T) { } check(0, 0) owners := make([]deployment.Allocation, 3) + for i := range keys { + if err := f.service.TouchActivity(t.Context(), keys[i].TenantID, keys[i].EnvironmentID); err != nil { + t.Fatal(err) + } + } for i := range owners { if _, err := changes.EnsurePlacement(t.Context(), keys[i], installation); err != nil { t.Fatal(err) diff --git a/services/core/migrations/000016_placement_demand_view.sql b/services/core/migrations/000016_placement_demand_view.sql new file mode 100644 index 000000000..67cae75f5 --- /dev/null +++ b/services/core/migrations/000016_placement_demand_view.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- The one definition of placement demand: an Environment that needs a node reservation, with the time it began waiting. +-- A never-allocated one waits from its creation, unless its initial input expired and no input is live: then only from the earliest live input. +-- reserved marks one that already holds a placement; the scan skips it and the per-Environment check counts it. +CREATE VIEW runtime_placement_demand AS +SELECT e.id, e.tenant_id, COALESCE(s.engine, '')::text AS engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, + GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN CASE WHEN i.initial_expired THEN i.live_at ELSE e.created_at END + ELSE LEAST(i.live_at, CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at, + (p.environment_id IS NOT NULL AND p.released_at IS NULL) AS reserved +FROM environments e JOIN environment_liveness l ON l.id = e.id AND l.active +LEFT JOIN sessions s ON s.environment_id = e.id AND s.deleted_at IS NULL +LEFT JOIN runtime_placements p ON p.environment_id = e.id +LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true +LEFT JOIN LATERAL (SELECT min(r.created_at) FILTER (WHERE r.state = 'pending' AND r.deadline > clock_timestamp()) AS live_at, + COALESCE(bool_or(r.is_initial AND (r.state = 'expired' OR (r.state = 'pending' AND r.deadline <= clock_timestamp()))), false)::boolean AS initial_expired + FROM environment_input_reservations r WHERE r.session_id = s.id) i ON true +WHERE e.configuration->>'type' = 'openai_hosted' + AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) + AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') + AND ( + (e.initialization IN ('pending','complete') AND latest.id IS NULL AND (NOT i.initial_expired OR i.live_at IS NOT NULL)) + OR (e.initialization = 'complete' + AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') + AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') + AND (latest.compute_wake_requested OR i.live_at IS NOT NULL)) + ); From a24a78538a42c87c4ee8a6ff77de8399445de823 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Tue, 17 Nov 2026 05:15:59 +0000 Subject: [PATCH 2/3] Bound placement demand cost and keep preparation errors Read live input through the unique pending row and the input history only for an Environment never allocated, so the demand scan no longer grows with history. The preparation check lists only Environments the view marks as not waiting, so reset, failed and not-ready cases keep their own errors; a lapsed refusal rescans placement at once. Document the pending state of an Environment whose input expired. --- contracts/agents-api/environment-files.md | 2 +- contracts/agents-api/sandbox-deployment.md | 2 +- contracts/agents-api/zh/environment-files.md | 4 +- contracts/agents-api/zh/sandbox-deployment.md | 4 +- .../db/queries/runtime_lifecycle_nodes.sql | 8 +- services/core/internal/db/sqlc/models.go | 1 + .../db/sqlc/runtime_lifecycle_nodes.sql.go | 12 +- .../core/internal/deployment/allocations.go | 4 +- .../internal/deployment/allocations_test.go | 12 +- services/core/internal/deployment/errors.go | 3 + .../core/internal/deployment/fakes_test.go | 4 +- services/core/internal/deployment/storage.go | 8 +- .../execution/deployment_fixture_test.go | 12 +- .../execution/runtime_expired_input_test.go | 113 +++++++++++------- .../internal/execution/runtime_lifecycle.go | 13 +- .../postgres/deploymentpg/allocations.go | 9 +- .../deploymentpg/placement_demand_test.go | 45 +++++-- .../000016_placement_demand_view.sql | 22 ++-- 18 files changed, 173 insertions(+), 105 deletions(-) diff --git a/contracts/agents-api/environment-files.md b/contracts/agents-api/environment-files.md index 88b55b116..5be2a4b18 100644 --- a/contracts/agents-api/environment-files.md +++ b/contracts/agents-api/environment-files.md @@ -12,7 +12,7 @@ The routes follow the SDK pinned in [upstream.json](https://github.com/MiniMax-A - Public paths start at `/workspace`, which stands for the Environment's workspace directory wherever it is on the machine. - The Environment is looked up in the caller's Project first; a missing or foreign ID returns 404. - On a standalone Environment that no Session is attached to, both operations return 409 `conflict_error`, "Attach a session to this environment before using its files." -- On an `openai_hosted` Environment that is still `pending`, both operations return 400 `the hosted environment is still provisioning; wait until it is connected before accessing files`. This check runs after request validation and before any source File lookup. Any other state proceeds, and an unreachable Runtime returns 503. +- On an `openai_hosted` Environment that is still `pending`, both operations return 400 `the hosted environment is still provisioning; wait until it is connected before accessing files`. An Environment whose initial input expired before it had a sandbox stays `pending` until [new input arrives](./sandbox-deployment.md#generation-ownership-and-rollout). This check runs after request validation and before any source File lookup. Any other state proceeds, and an unreachable Runtime returns 503. - File operations wait within their request deadline for Environment initialization to complete; `connected` alone does not imply readiness, and failed initialization or exhausted execution capacity still returns 503. - Listing and writing never start a Turn or send input to the model. diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 601f9be8c..db6cb3970 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -170,7 +170,7 @@ A node is [online](./node-generation-protocol.md#connection) when it is connecte Each node adds `rollout: {state, ready_generation, diagnostic?}`, where `ready_generation` is the nullable durable serving pin and `diagnostic` a fixed code for the target generation; allocation items add `deployment_generation`. Poll every five seconds only while `rollout.state` is `preparing` or `reset` is not null; old Sessions and failed or offline nodes alone do not keep polling active. -Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. An Environment that was never placed stops waiting for capacity when its initial input expires, until new input arrives, and a reservation whose input expired before preparation prepares nothing and is released. +Node-backed creation validates the target deployment and commits the Session, pending Environment and any initial input without reserving compute. A full, offline or preparing fleet leaves that accepted work waiting; an unconfigured deployment, reset or unsupported combination still rejects admission. The common scheduler uses bounded rotating scans of unplaced demand, with pages ordered by recorded time and Environment ID and a fixed time boundary for each scan so continuous arrivals cannot prevent it from revisiting older work. It checks online presence, exact serving-generation readiness, address, shared capacity and the selected generation's Harness/filesystem compatibility, then prefers the newest qualifying pin. Placement is immutable once reserved until confirmed release or a settled checkpoint transfer. A bounded scan skips temporarily unavailable demand and resumes after a restart. If preparation fails before a new allocation is reserved, Core releases only that attempt’s node reservation, retaining any filesystem identity. Its next placement demand is ordered no earlier than release time, behind already-waiting work; successful retries use the newly selected generation’s specification. An allocation with an unknown compute outcome keeps its reservation. Existing suspended allocations reserve eligible checkpoint capacity through the same lock under the [checkpoint transfer contract](../../docs/sandbox-provider.md#checkpoint-transfer); this is not a global fairness guarantee across hot restores and unplaced work. Input retains its [original five-minute deadline](./environments.md#reservations), including time waiting for capacity. An Environment that never had a sandbox stops waiting for capacity when its initial input expires and stays `pending` until new input arrives. When preparation begins for an Environment that no longer waits for capacity, Core releases its node reservation and prepares nothing; an Environment created without input, or one with a live input or a wake request, still prepares. ## Reset diff --git a/contracts/agents-api/zh/environment-files.md b/contracts/agents-api/zh/environment-files.md index 612198d73..1a5386d6d 100644 --- a/contracts/agents-api/zh/environment-files.md +++ b/contracts/agents-api/zh/environment-files.md @@ -1,7 +1,7 @@ --- title: "Environment 文件与 Artifact" source: contracts/agents-api/environment-files.md -source_hash: 95d56a1da2027c85e05130f00e8712247682d65365a483c6563c1d10a4c70363 +source_hash: 74b5a88b73547f028fb005f5878eb00a613b927b357ca16f18c105e0aa8e0eb8 --- Session 工作区保存由 agent 及其工具修改的实时文件。`/agents/environments/{environment_id}/files` 列出一个工作区目录,并在其中创建文件。Turn 完成时,Core 将工作区 `outputs/` 目录中的文件复制为不可变 Artifact,通过 `/agents/sessions/{session_id}/artifacts` 读取。Artifact 的生命周期长于 Environment;工作区文件则不是。 @@ -14,7 +14,7 @@ Session 工作区保存由 agent 及其工具修改的实时文件。`/agents/en - 公开路径从 `/workspace` 开始,代表 Environment 在机器上实际位置的工作区目录。 - 首先在调用方项目查找 Environment;不存在或属于其他范围的 ID 返回 404。 - 未挂载任何 Session 的独立 Environment,两种操作均返回 409 `conflict_error`,“Attach a session to this environment before using its files.” -- 仍为 `pending` 的 `openai_hosted` Environment,两种操作均返回 400 `the hosted environment is still provisioning; wait until it is connected before accessing files`。此检查在请求验证之后、任何源 File 查询之前执行。其他状态继续处理,Runtime 不可达时返回 503。 +- 仍为 `pending` 的 `openai_hosted` Environment,两种操作均返回 400 `the hosted environment is still provisioning; wait until it is connected before accessing files`。初始输入在获得沙箱之前过期的 Environment 保持 `pending`,直到[新输入到达](./sandbox-deployment.md#generation-ownership-and-rollout)。此检查在请求验证之后、任何源 File 查询之前执行。其他状态继续处理,Runtime 不可达时返回 503。 - 文件操作在现有请求期限内等待 Environment 初始化完成;`connected` 本身不表示已就绪,初始化失败或执行容量耗尽仍返回 503。 - 列表和写入不启动 Turn,也不向模型发送输入。 diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 68beaa603..87182b7c0 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: b86c842977d47bd394ff65d0759fddc8c455da7868bf88093401ee317eac62e1 +source_hash: 04cc5bd76506cb4bbbd17b6c0132e66bbecdd94355146646e25a2ca39bea2a6d --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商和每个沙箱的资源。PostgreSQL 为每个安装维护一个当前有效选择;Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径,且不能覆盖其资源。Runtime 不属于该选择:每个节点运行其所属安装验证过的 Runtime。该选择独立于 Harness。部署可以保持未配置状态,没有节点;此时它拒绝托管准入。 @@ -173,7 +173,7 @@ flowchart TD 每个节点会添加 `rollout: {state, ready_generation, diagnostic?}`,其中 `ready_generation` 是可为 null 的持久服务固定状态,`diagnostic` 是目标代次的固定代码;分配项会添加 `deployment_generation`。仅当 `rollout.state` 为 `preparing` 或 `reset` 非 null 时,才每五秒轮询一次;旧 Session 以及失败或离线的节点本身均不会使轮询保持活动状态。 -节点模式创建先验证目标部署,提交 Session、pending Environment 及初始输入,但不预留计算资源。节点全部已满、离线或准备中时,已接受的工作继续等待;未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描,分页按需求记录时间和 Environment ID 排序,每轮使用固定时间上界,避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性,然后优先选择最新的合格固定代次。预留后的 placement 保持不可变,直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求,并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer),通过同一容量锁预留符合条件的检查点容量;这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations),等待容量的时间也计入其中。从未放置的 Environment 在其初始输入过期后不再等待容量,直到新输入到达;输入在准备之前已过期的预留不做任何准备并被释放。 +节点模式创建先验证目标部署,提交 Session、pending Environment 及初始输入,但不预留计算资源。节点全部已满、离线或准备中时,已接受的工作继续等待;未配置部署、reset 或不支持的组合仍拒绝准入。共同调度器对尚未放置的需求进行有界循环扫描,分页按需求记录时间和 Environment ID 排序,每轮使用固定时间上界,避免持续的新需求阻止重新访问较早的工作。它检查在线状态、精确服务代次就绪情况、地址、共享容量和所选代次的 Harness/文件系统兼容性,然后优先选择最新的合格固定代次。预留后的 placement 保持不可变,直到确认释放或已结清的检查点转移。有界扫描跳过暂时不可调度的需求,并在重启后继续。已有 suspended allocation 按[检查点转移契约](../../../docs/zh/sandbox-provider.md#checkpoint-transfer),通过同一容量锁预留符合条件的检查点容量;这不构成热恢复和未放置工作之间的全局公平性保证。输入保留[原始五分钟期限](./environments.md#reservations),等待容量的时间也计入其中。从未有过沙箱的 Environment 在其初始输入过期后不再等待容量,并保持 `pending`,直到新输入到达。准备开始时,若 Environment 已不再等待容量,Core 释放其节点预留且不做任何准备;创建时没有输入的 Environment,以及有未过期输入或唤醒请求的 Environment,仍会准备。 如果准备工作在新 allocation 预留前失败,Core 仅释放该次尝试的节点预留,并保留已有文件系统身份。其下一次放置需求按不早于释放的时间排序,排在已经等待的工作之后;成功重试使用新选中代的 specification。计算结果不确定的 allocation 继续保留其预留。 diff --git a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql index 584eea156..e62ac936f 100644 --- a/services/core/internal/db/queries/runtime_lifecycle_nodes.sql +++ b/services/core/internal/db/queries/runtime_lifecycle_nodes.sql @@ -42,10 +42,10 @@ WHERE e.tenant_id=$1 AND e.id=$2; -- name: ListPlacementDemand :many SELECT id, tenant_id, engine, retained, demanded_at, COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM runtime_placement_demand -WHERE NOT reserved AND (demanded_at, id) > (sqlc.arg(after_time)::timestamptz, sqlc.arg(after_id)::uuid) +WHERE waiting AND NOT reserved AND (demanded_at, id) > (sqlc.arg(after_time)::timestamptz, sqlc.arg(after_id)::uuid) AND demanded_at <= COALESCE(sqlc.narg(until_time)::timestamptz, statement_timestamp()) ORDER BY demanded_at, id LIMIT 32; --- name: IsPlacementDemanded :one --- A plain id predicate keeps an index plan under a cached generic plan; the scan's statement could not. -SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1)::boolean; +-- name: IsPlacementDemandLapsed :one +-- Listed by the view but not waiting. A plain id predicate keeps an index plan under a cached generic plan, which the scan's statement could not. +SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1 AND NOT waiting)::boolean; diff --git a/services/core/internal/db/sqlc/models.go b/services/core/internal/db/sqlc/models.go index 41a3c861c..575cfaffc 100644 --- a/services/core/internal/db/sqlc/models.go +++ b/services/core/internal/db/sqlc/models.go @@ -392,6 +392,7 @@ type RuntimePlacementDemand struct { Retained bool `json:"retained"` DemandedAt pgtype.Timestamptz `json:"demanded_at"` Reserved pgtype.Bool `json:"reserved"` + Waiting interface{} `json:"waiting"` } type RuntimeResetRetainedEnvironment struct { diff --git a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go index 64a7434b3..be736a123 100644 --- a/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_lifecycle_nodes.sql.go @@ -56,13 +56,13 @@ func (q *Queries) GetRuntimeLifecyclePlacement(ctx context.Context, arg GetRunti return i, err } -const isPlacementDemanded = `-- name: IsPlacementDemanded :one -SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1)::boolean +const isPlacementDemandLapsed = `-- name: IsPlacementDemandLapsed :one +SELECT EXISTS (SELECT 1 FROM runtime_placement_demand WHERE id = $1 AND NOT waiting)::boolean ` -// A plain id predicate keeps an index plan under a cached generic plan; the scan's statement could not. -func (q *Queries) IsPlacementDemanded(ctx context.Context, id pgtype.UUID) (bool, error) { - row := q.db.QueryRow(ctx, isPlacementDemanded, id) +// Listed by the view but not waiting. A plain id predicate keeps an index plan under a cached generic plan, which the scan's statement could not. +func (q *Queries) IsPlacementDemandLapsed(ctx context.Context, id pgtype.UUID) (bool, error) { + row := q.db.QueryRow(ctx, isPlacementDemandLapsed, id) var column_1 bool err := row.Scan(&column_1) return column_1, err @@ -70,7 +70,7 @@ func (q *Queries) IsPlacementDemanded(ctx context.Context, id pgtype.UUID) (bool const listPlacementDemand = `-- name: ListPlacementDemand :many SELECT id, tenant_id, engine, retained, demanded_at, COALESCE($1::timestamptz, statement_timestamp())::timestamptz AS scan_until FROM runtime_placement_demand -WHERE NOT reserved AND (demanded_at, id) > ($2::timestamptz, $3::uuid) +WHERE waiting AND NOT reserved AND (demanded_at, id) > ($2::timestamptz, $3::uuid) AND demanded_at <= COALESCE($1::timestamptz, statement_timestamp()) ORDER BY demanded_at, id LIMIT 32 ` diff --git a/services/core/internal/deployment/allocations.go b/services/core/internal/deployment/allocations.go index 2448753b6..dd0b37b36 100644 --- a/services/core/internal/deployment/allocations.go +++ b/services/core/internal/deployment/allocations.go @@ -85,9 +85,9 @@ func (e *ExecutionOperations) ReserveAllocation(ctx context.Context, key Allocat return err } // Input may have expired since the placement was reserved; nothing exists to start for it. - if demanded, err := tx.LoadPlacementDemanded(); err != nil || !demanded { + if lapsed, err := tx.LoadPlacementDemandLapsed(); err != nil || lapsed { if err == nil { - err = ErrAllocationConflict + err = ErrPlacementDemandLapsed } return err } diff --git a/services/core/internal/deployment/allocations_test.go b/services/core/internal/deployment/allocations_test.go index 883c51420..684e48489 100644 --- a/services/core/internal/deployment/allocations_test.go +++ b/services/core/internal/deployment/allocations_test.go @@ -28,7 +28,7 @@ type fakeReservationTx struct { findAllocation func() (Allocation, bool, error) lockDeployment func() (placement.Deployment, error) loadReserved func() (placement.Reserved, error) - placementDemanded func() (bool, error) + placementDemandLapsed func() (bool, error) insertAllocation func(NewAllocation) (Allocation, error) } @@ -74,12 +74,12 @@ func (f *fakeReservationTx) LoadReserved() (placement.Reserved, error) { return f.loadReserved() } -// LoadPlacementDemanded answers true unless a test sets placementDemanded. -func (f *fakeReservationTx) LoadPlacementDemanded() (bool, error) { - if f.placementDemanded == nil { - return true, nil +// LoadPlacementDemandLapsed answers not lapsed unless a test sets placementDemandLapsed. +func (f *fakeReservationTx) LoadPlacementDemandLapsed() (bool, error) { + if f.placementDemandLapsed == nil { + return false, nil } - return f.placementDemanded() + return f.placementDemandLapsed() } func (f *fakeReservationTx) CanReplaceAllocation() (bool, error) { diff --git a/services/core/internal/deployment/errors.go b/services/core/internal/deployment/errors.go index 80e1f5b7f..46b9fdb05 100644 --- a/services/core/internal/deployment/errors.go +++ b/services/core/internal/deployment/errors.go @@ -2,6 +2,7 @@ package deployment import ( "errors" + "fmt" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) @@ -27,6 +28,8 @@ var ( // matches the stored allocation, device binding, state or compute revision, // or a replay for another installation. ErrAllocationConflict = errors.New("the sandbox allocation changed") + // ErrPlacementDemandLapsed means the Environment no longer waits for capacity, so nothing is prepared for it. It is ErrAllocationConflict for the caller. + ErrPlacementDemandLapsed = fmt.Errorf("%w: the Environment no longer waits for capacity", ErrAllocationConflict) // ErrNotIdle means neither idle expiry nor current demand requires suspension. ErrNotIdle = errors.New("sandbox compute is not ready to suspend") // ErrNodeAddressMismatch rejects an enrollment whose Core address is not diff --git a/services/core/internal/deployment/fakes_test.go b/services/core/internal/deployment/fakes_test.go index f4008b540..6f7a94ce8 100644 --- a/services/core/internal/deployment/fakes_test.go +++ b/services/core/internal/deployment/fakes_test.go @@ -638,8 +638,8 @@ func (f *fakeSessionReader) GetManagedSessionArchive(context.Context, string, st return sessions.ManagedArchive{}, nil } -func (f *fakeReader) PlacementDemanded(context.Context, AllocationKey) (bool, error) { - panic("unexpected PlacementDemanded") +func (f *fakeReader) PlacementDemandLapsed(context.Context, AllocationKey) (bool, error) { + panic("unexpected PlacementDemandLapsed") } func (f *fakeReader) PlacementDemand(context.Context, PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) { panic("unexpected PlacementDemand") diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index cc3e2a9da..6fbf513da 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -104,8 +104,8 @@ type ReservationTx interface { LoadGenerationSpecification(generation uint64) (GenerationSpecification, error) // LoadNodes reads eligible placement facts under the deployment lock. LoadNodes() ([]placement.Node, error) - // LoadPlacementDemanded reports whether the Environment is still placement demand, counting one that already reserved a placement: the runtime_placement_demand view the scan uses. - LoadPlacementDemanded() (bool, error) + // LoadPlacementDemandLapsed reports whether the Environment could take a placement but no longer waits for one: the runtime_placement_demand view the scan uses, counting one that already reserved a placement. An Environment the view does not list is not lapsed, so the caller's other checks report their own errors. + LoadPlacementDemandLapsed() (bool, error) // ReservePlacement inserts a first reservation or replaces a released reservation without changing historical receipts. ReservePlacement(placement.Placement) error // ReleaseFailedPlacement yields only the exact reservation without active compute. @@ -275,8 +275,8 @@ type Reader interface { RetainedNativeHistory(ctx context.Context, key AllocationKey) (bool, error) // PlacementDemand lists up to 32 unreserved first or retained requests in demand-time and ID order. PlacementDemand(ctx context.Context, after PlacementDemandCursor) ([]PlacementDemand, PlacementDemandCursor, error) - // PlacementDemanded reports whether the Environment is placement demand with the same definition as PlacementDemand, counting one that already reserved a placement. - PlacementDemanded(ctx context.Context, key AllocationKey) (bool, error) + // PlacementDemandLapsed is LoadPlacementDemandLapsed without the deployment lock. + PlacementDemandLapsed(ctx context.Context, key AllocationKey) (bool, error) // LifecyclePlacement returns what routing the Environment to its // lifecycle reads, including for a deleted Session. A missing Environment // is sessions.ErrNotFound. diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 7ba42d5b3..347c5e63b 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -142,7 +142,7 @@ func (s *strictExecutionStorage) WithDeployment(ctx context.Context, apply func( // strictDeploymentReader runs each set func; any other call fails the test. type strictDeploymentReader struct { - placementDemanded func(context.Context, deployment.AllocationKey) (bool, error) + placementDemandLapsed func(context.Context, deployment.AllocationKey) (bool, error) placementDemand func(context.Context, deployment.PlacementDemandCursor) ([]deployment.PlacementDemand, deployment.PlacementDemandCursor, error) retainedNativeHistory func(context.Context, deployment.AllocationKey) (bool, error) t *testing.T @@ -363,12 +363,12 @@ func (r *strictDeploymentReader) PlacementDemand(ctx context.Context, after depl return r.placementDemand(ctx, after) } -// PlacementDemanded answers true unless a test sets placementDemanded: provisioning tests reach it with real demand. -func (r *strictDeploymentReader) PlacementDemanded(ctx context.Context, key deployment.AllocationKey) (bool, error) { - if r.placementDemanded == nil { - return true, nil +// PlacementDemandLapsed answers not lapsed unless a test sets placementDemandLapsed. +func (r *strictDeploymentReader) PlacementDemandLapsed(ctx context.Context, key deployment.AllocationKey) (bool, error) { + if r.placementDemandLapsed == nil { + return false, nil } - return r.placementDemanded(ctx, key) + return r.placementDemandLapsed(ctx, key) } func (r *strictDeploymentReader) RetainedNativeHistory(ctx context.Context, key deployment.AllocationKey) (bool, error) { if r.retainedNativeHistory == nil { diff --git a/services/core/internal/execution/runtime_expired_input_test.go b/services/core/internal/execution/runtime_expired_input_test.go index 4504b6339..e54cdde78 100644 --- a/services/core/internal/execution/runtime_expired_input_test.go +++ b/services/core/internal/execution/runtime_expired_input_test.go @@ -5,65 +5,94 @@ import ( "errors" "testing" + "github.com/google/uuid" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" ) -// demandReader answers the placement demand recheck and may report a released allocation. -type demandReader struct { +// expireInitialInput gives the fixture Session an initial input that expired. +func expireInitialInput(t *testing.T, f workspaceSettlementFixture) { + t.Helper() + if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,settings,is_initial,created_at,deadline,state,settled_at) VALUES($1,$2,$3,'[{}]','{}',true,clock_timestamp()-interval '10 minutes',clock_timestamp()-interval '5 minutes','expired',clock_timestamp()-interval '5 minutes')`, uuid.NewString(), f.session.ID, uuid.NewString()); err != nil { + t.Fatal(err) + } +} + +// Input that expired after the placement was reserved prepares no workspace or sandbox and gives the reserved node back. +func TestExpiredInitialInputPreparesNothingAndReleasesTheReservation(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, s := f.lifecycle, f.session + r.config.Mode = string(sandbox.DeploymentNodes) + expireInitialInput(t, f) + _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if !errors.Is(err, deployment.ErrPlacementDemandLapsed) || !errors.Is(err, deployment.ErrAllocationConflict) { + t.Fatal(err) + } + var workspaces, allocations int + if err := f.pool.QueryRow(t.Context(), `SELECT (SELECT count(*) FROM environment_workspaces),(SELECT count(*) FROM runtime_allocations)`).Scan(&workspaces, &allocations); err != nil { + t.Fatal(err) + } + target, err := r.reader.LifecyclePlacement(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}) + if err != nil || workspaces != 0 || allocations != 0 || provider.creates != 0 || !target.PlacementReleased { + t.Fatal("prepared for expired input", workspaces, allocations, provider.creates, target, err) + } +} + +// lapsedReader reports a released allocation whose demand lapsed. +type lapsedReader struct { deployment.Reader - demanded bool - released bool - checks int + checks int } -func (r *demandReader) PlacementDemanded(context.Context, deployment.AllocationKey) (bool, error) { +func (r *lapsedReader) PlacementDemandLapsed(context.Context, deployment.AllocationKey) (bool, error) { r.checks++ - return r.demanded, nil + return true, nil } -func (r *demandReader) EnvironmentAllocation(ctx context.Context, key deployment.AllocationKey) (deployment.Allocation, error) { - if r.released { - return deployment.Allocation{State: "released"}, nil - } - return r.Reader.EnvironmentAllocation(ctx, key) +func (r *lapsedReader) EnvironmentAllocation(context.Context, deployment.AllocationKey) (deployment.Allocation, error) { + return deployment.Allocation{State: "released"}, nil } -// Input that expired after the placement was reserved prepares no workspace or sandbox and gives the reserved node back. -func TestInputExpiringAfterPlacementPreparesNothing(t *testing.T) { - for name, tc := range map[string]struct{ released bool }{"never allocated": {}, "released allocation": {released: true}} { - t.Run(name, func(t *testing.T) { - provider := &retentionProvider{} - f := newWorkspaceSettlementFixture(t, provider) - r, s := f.lifecycle, f.session - r.config.Mode = string(sandbox.DeploymentNodes) - reader := &demandReader{Reader: r.reader, released: tc.released} - r.reader = reader - _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) - if !errors.Is(err, deployment.ErrAllocationConflict) { - t.Fatal(err) - } - var workspaces, allocations int - if err := f.pool.QueryRow(t.Context(), `SELECT (SELECT count(*) FROM environment_workspaces),(SELECT count(*) FROM runtime_allocations)`).Scan(&workspaces, &allocations); err != nil { - t.Fatal(err) - } - target, err := reader.Reader.LifecyclePlacement(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}) - if err != nil || workspaces != 0 || allocations != 0 || provider.creates != 0 || !target.PlacementReleased { - t.Fatal("prepared for expired input", workspaces, allocations, provider.creates, target, err) - } - }) +// Released compute whose demand lapsed before preparation prepares nothing and gives the reserved node back. +func TestReleasedAllocationWithLapsedDemandPreparesNothing(t *testing.T) { + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) + r, s := f.lifecycle, f.session + r.config.Mode = string(sandbox.DeploymentNodes) + reader := &lapsedReader{Reader: r.reader} + r.reader = reader + if _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID); !errors.Is(err, deployment.ErrPlacementDemandLapsed) { + t.Fatal(err) + } + target, err := reader.Reader.LifecyclePlacement(t.Context(), deployment.AllocationKey{TenantID: s.TenantID, EnvironmentID: s.Environment.ID}) + if err != nil || provider.creates != 0 || !target.PlacementReleased { + t.Fatal(provider.creates, target, err) } } -// A direct deployment has no placement demand to check. +// A direct deployment has no placement demand to check, so provisioning never asks for it. func TestDirectDeploymentSkipsPlacementDemandCheck(t *testing.T) { - f := newWorkspaceSettlementFixture(t, &retentionProvider{}) + provider := &retentionProvider{} + f := newWorkspaceSettlementFixture(t, provider) r, s := f.lifecycle, f.session r.config.Mode = string(sandbox.DeploymentDirect) - reader := &demandReader{Reader: r.reader} - r.reader = reader - _, _ = r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) - if reader.checks != 0 { - t.Fatal("direct deployment checked placement demand", reader.checks) + // A reader that would refuse; a direct deployment never asks it. + checks := 0 + r.reader = &directReader{Reader: r.reader, checks: &checks} + owner, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) + if checks != 0 || err != nil || owner.ID == "" || provider.creates != 1 { + t.Fatal("direct deployment", checks, owner, err, provider.creates) } } + +type directReader struct { + deployment.Reader + checks *int +} + +func (r *directReader) PlacementDemandLapsed(context.Context, deployment.AllocationKey) (bool, error) { + *r.checks++ + return true, nil +} diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index d982792bc..b98afc9c5 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -213,7 +213,12 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) defer cancel() - resultErr = errors.Join(resultErr, r.deployment.ReleaseFailedPlacement(cleanup, key, expected)) + releaseErr := r.deployment.ReleaseFailedPlacement(cleanup, key, expected) + // Input that arrived since the check was skipped as reserved; rescan now, not at the next tick. + if releaseErr == nil && errors.Is(resultErr, deployment.ErrPlacementDemandLapsed) { + r.freedSlot() + } + resultErr = errors.Join(resultErr, releaseErr) }() } } @@ -240,12 +245,12 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p } // Input that expired since the placement was reserved prepares nothing. A direct deployment has no placement demand. if r.config.Mode == string(sandbox.DeploymentNodes) && (errors.Is(lookupErr, deployment.ErrNotFound) || existing.State == "released") { - demanded, demandErr := r.reader.PlacementDemanded(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}) + lapsed, demandErr := r.reader.PlacementDemandLapsed(ctx, deployment.AllocationKey{TenantID: tenant, EnvironmentID: environment}) if demandErr != nil { return deployment.Allocation{}, demandErr } - if !demanded { - return deployment.Allocation{}, deployment.ErrAllocationConflict + if lapsed { + return deployment.Allocation{}, deployment.ErrPlacementDemandLapsed } } if errors.Is(lookupErr, deployment.ErrNotFound) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/allocations.go b/services/core/internal/persistence/postgres/deploymentpg/allocations.go index 092d07761..ecca330aa 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/allocations.go +++ b/services/core/internal/persistence/postgres/deploymentpg/allocations.go @@ -214,8 +214,8 @@ func (t *reservationTx) LoadReserved() (placement.Reserved, error) { return reserved, err } -func (t *reservationTx) LoadPlacementDemanded() (bool, error) { - return t.q.IsPlacementDemanded(t.ctx, t.environment) +func (t *reservationTx) LoadPlacementDemandLapsed() (bool, error) { + return t.q.IsPlacementDemandLapsed(t.ctx, t.environment) } func (t *reservationTx) CanReplaceAllocation() (bool, error) { @@ -632,14 +632,15 @@ func (s *Store) PlacementDemand(ctx context.Context, after deployment.PlacementD return loadPlacementDemand(ctx, s.pool.Queries(), after) } -func (s *Store) PlacementDemanded(ctx context.Context, key deployment.AllocationKey) (bool, error) { +// PlacementDemandLapsed is the check's own statement: a plain id predicate keeps its index plan under a cached generic plan. +func (s *Store) PlacementDemandLapsed(ctx context.Context, key deployment.AllocationKey) (bool, error) { ctx, cancel := context.WithTimeout(ctx, pgunit.ExecutionTimeout) defer cancel() _, environment, err := allocationKey(key) if err != nil { return false, err } - return s.pool.Queries().IsPlacementDemanded(ctx, environment) + return s.pool.Queries().IsPlacementDemandLapsed(ctx, environment) } func (s *Store) LifecyclePlacement(ctx context.Context, key deployment.AllocationKey) (deployment.LifecyclePlacement, error) { diff --git a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go index 2c4ff69ad..8d30a9d07 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go @@ -231,14 +231,16 @@ func TestPlacementDemandStopsWhenInitialInputExpires(t *testing.T) { t.Fatal(err) } } - idle, expired, expiredUnswept, revived, pending := pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f) + idle, idleSettled, expired, expiredUnswept, revived, pending := pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f), pendingHostedEnvironment(t, f) livedAt := now.Add(-time.Minute) + // Only an expired initial input ends the wait; a later input that expired does not. + input(idleSettled, false, "expired", now.Add(-time.Hour), -time.Hour) input(expired, true, "expired", now.Add(-time.Hour), -time.Hour) input(expiredUnswept, true, "pending", now.Add(-time.Hour), -time.Minute) input(revived, true, "expired", now.Add(-time.Hour), -time.Hour) input(revived, false, "pending", livedAt, time.Minute) input(pending, true, "pending", now.Add(-time.Second), time.Minute) - for _, key := range []deployment.AllocationKey{idle, expired, expiredUnswept, revived, pending} { + for _, key := range []deployment.AllocationKey{idle, idleSettled, expired, expiredUnswept, revived, pending} { if _, err := f.pool.Exec(t.Context(), `UPDATE environments SET created_at=$2 WHERE id=$1`, key.EnvironmentID, now.Add(-2*time.Hour)); err != nil { t.Fatal(err) } @@ -251,7 +253,7 @@ func TestPlacementDemandStopsWhenInitialInputExpires(t *testing.T) { for _, row := range demand { at[row.ID] = row.At } - want := map[string]time.Time{idle.EnvironmentID: now.Add(-2 * time.Hour), revived.EnvironmentID: livedAt, pending.EnvironmentID: now.Add(-2 * time.Hour)} + want := map[string]time.Time{idle.EnvironmentID: now.Add(-2 * time.Hour), idleSettled.EnvironmentID: now.Add(-2 * time.Hour), revived.EnvironmentID: livedAt, pending.EnvironmentID: now.Add(-2 * time.Hour)} if len(at) != len(want) { t.Fatal("demand", at) } @@ -260,10 +262,10 @@ func TestPlacementDemandStopsWhenInitialInputExpires(t *testing.T) { t.Fatal("demanded at", id, at[id], "want", when) } } - // The recheck reads the same rule. - for key, wanted := range map[deployment.AllocationKey]bool{idle: true, expired: false, expiredUnswept: false, revived: true, pending: true} { - if got, err := f.adapter.PlacementDemanded(t.Context(), key); err != nil || got != wanted { - t.Fatal("PlacementDemanded", key.EnvironmentID, got, wanted, err) + // The recheck reads the same rule: only an Environment the view lists as not waiting has lapsed. + for key, wanted := range map[deployment.AllocationKey]bool{idle: false, idleSettled: false, expired: true, expiredUnswept: true, revived: false, pending: false} { + if got, err := f.adapter.PlacementDemandLapsed(t.Context(), key); err != nil || got != wanted { + t.Fatal("PlacementDemandLapsed", key.EnvironmentID, got, wanted, err) } } } @@ -282,7 +284,7 @@ func TestReserveAllocationRefusesExpiredInitialInput(t *testing.T) { if _, err := f.pool.Exec(t.Context(), `INSERT INTO environment_input_reservations(id,session_id,idempotency_key,batch,is_initial,settings,created_at,deadline,state,settled_at) SELECT $1,id,$2,'[{}]',true,'{}',clock_timestamp(),clock_timestamp(),'expired',clock_timestamp() FROM sessions WHERE environment_id=$3`, uuid.NewString(), uuid.NewString(), key.EnvironmentID); err != nil { t.Fatal(err) } - if _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()); !errors.Is(err, deployment.ErrAllocationConflict) { + if _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()); !errors.Is(err, deployment.ErrPlacementDemandLapsed) { t.Fatal("expired input reserved an allocation", err) } var allocations, devices int @@ -290,3 +292,30 @@ func TestReserveAllocationRefusesExpiredInitialInput(t *testing.T) { t.Fatal(allocations, devices, err) } } + +// Released compute with no wake request and no live input has no demand, so reserving an allocation for it is refused. +func TestReserveAllocationRefusesReleasedComputeWithoutDemand(t *testing.T) { + f := newFixture(t) + changes, _ := f.execution(t) + installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) + node := f.enroll(t, view, deployment.Capacity{MaxActive: 1, MaxRetained: 2}) + f.connect(t, node.NodeID) + key := hostedEnvironment(t, f.pool) + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + owner, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if err != nil { + t.Fatal(err) + } + releaseRetainedFixture(t, f, changes, owner) + if _, err := f.pool.Exec(t.Context(), `UPDATE runtime_allocations SET compute_wake_requested=false WHERE id=$1`, owner.ID); err != nil { + t.Fatal(err) + } + if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { + t.Fatal(err) + } + if _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()); !errors.Is(err, deployment.ErrPlacementDemandLapsed) { + t.Fatal("released compute without demand reserved an allocation", err) + } +} diff --git a/services/core/migrations/000016_placement_demand_view.sql b/services/core/migrations/000016_placement_demand_view.sql index 67cae75f5..834140634 100644 --- a/services/core/migrations/000016_placement_demand_view.sql +++ b/services/core/migrations/000016_placement_demand_view.sql @@ -1,26 +1,26 @@ -- +goose Up --- The one definition of placement demand: an Environment that needs a node reservation, with the time it began waiting. --- A never-allocated one waits from its creation, unless its initial input expired and no input is live: then only from the earliest live input. --- reserved marks one that already holds a placement; the scan skips it and the per-Environment check counts it. +-- The one definition of placement demand. A row is an active hosted Environment that could take a node reservation: no live compute, either never allocated or with released compute and a ready workspace. +-- waiting is whether it needs one now. A never-allocated Environment waits from its creation, unless its initial input expired and no input is live: then only from the earliest live input. One with released compute waits on a wake request or a live input. +-- reserved marks one that already holds a placement; the scan skips it, and the preparation check reads it. CREATE VIEW runtime_placement_demand AS SELECT e.id, e.tenant_id, COALESCE(s.engine, '')::text AS engine, (latest.id IS NOT NULL OR EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id=e.id))::boolean AS retained, - GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN CASE WHEN i.initial_expired THEN i.live_at ELSE e.created_at END + GREATEST(p.released_at, CASE WHEN latest.id IS NULL THEN CASE WHEN x.initial_expired THEN i.live_at ELSE e.created_at END ELSE LEAST(i.live_at, CASE WHEN latest.compute_wake_requested THEN latest.compute_activity_at END) END)::timestamptz AS demanded_at, - (p.environment_id IS NOT NULL AND p.released_at IS NULL) AS reserved + (p.environment_id IS NOT NULL AND p.released_at IS NULL) AS reserved, + CASE WHEN latest.id IS NULL THEN NOT x.initial_expired OR i.live_at IS NOT NULL ELSE latest.compute_wake_requested OR i.live_at IS NOT NULL END AS waiting FROM environments e JOIN environment_liveness l ON l.id = e.id AND l.active LEFT JOIN sessions s ON s.environment_id = e.id AND s.deleted_at IS NULL LEFT JOIN runtime_placements p ON p.environment_id = e.id LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true -LEFT JOIN LATERAL (SELECT min(r.created_at) FILTER (WHERE r.state = 'pending' AND r.deadline > clock_timestamp()) AS live_at, - COALESCE(bool_or(r.is_initial AND (r.state = 'expired' OR (r.state = 'pending' AND r.deadline <= clock_timestamp()))), false)::boolean AS initial_expired - FROM environment_input_reservations r WHERE r.session_id = s.id) i ON true +-- Input is read through its unique pending row, and the history only for an Environment never allocated, so the cost does not grow with input history. +LEFT JOIN LATERAL (SELECT r.created_at AS live_at FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()) i ON true +LEFT JOIN LATERAL (SELECT (latest.id IS NULL AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.is_initial AND (r.state = 'expired' OR (r.state = 'pending' AND r.deadline <= clock_timestamp())))) AS initial_expired) x ON true WHERE e.configuration->>'type' = 'openai_hosted' AND (SELECT reset_clear IS NULL AND mode = 'nodes' FROM runtime_deployment) AND NOT EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state <> 'released') AND ( - (e.initialization IN ('pending','complete') AND latest.id IS NULL AND (NOT i.initial_expired OR i.live_at IS NOT NULL)) + (e.initialization IN ('pending','complete') AND latest.id IS NULL) OR (e.initialization = 'complete' AND EXISTS (SELECT 1 FROM environment_workspaces w WHERE w.environment_id = e.id AND w.state = 'ready') - AND EXISTS (SELECT 1 FROM runtime_allocations a WHERE a.environment_id = e.id AND a.state = 'released') - AND (latest.compute_wake_requested OR i.live_at IS NOT NULL)) + AND latest.state = 'released') ); From 840674d0f1783143d820a60c3889edeb3a07fba7 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Tue, 17 Nov 2026 05:48:25 +0000 Subject: [PATCH 3/3] Index the initial input and pin refusal errors in tests The never-allocated demand test reads a Session's initial input through a partial index, so the scan no longer reads its whole input history. Tests pin that reset, failed and not-ready cases keep their own errors and that a lapsed release wakes the placement scan. --- .../execution/deployment_fixture_test.go | 3 +- .../execution/runtime_expired_input_test.go | 6 +++ .../deploymentpg/placement_demand_test.go | 40 ++++++++++++++++++- .../000016_placement_demand_view.sql | 4 +- 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/services/core/internal/execution/deployment_fixture_test.go b/services/core/internal/execution/deployment_fixture_test.go index 347c5e63b..6e9a717c7 100644 --- a/services/core/internal/execution/deployment_fixture_test.go +++ b/services/core/internal/execution/deployment_fixture_test.go @@ -363,10 +363,9 @@ func (r *strictDeploymentReader) PlacementDemand(ctx context.Context, after depl return r.placementDemand(ctx, after) } -// PlacementDemandLapsed answers not lapsed unless a test sets placementDemandLapsed. func (r *strictDeploymentReader) PlacementDemandLapsed(ctx context.Context, key deployment.AllocationKey) (bool, error) { if r.placementDemandLapsed == nil { - return false, nil + return false, unexpectedDeploymentCall(r.t, "PlacementDemandLapsed") } return r.placementDemandLapsed(ctx, key) } diff --git a/services/core/internal/execution/runtime_expired_input_test.go b/services/core/internal/execution/runtime_expired_input_test.go index e54cdde78..30708a005 100644 --- a/services/core/internal/execution/runtime_expired_input_test.go +++ b/services/core/internal/execution/runtime_expired_input_test.go @@ -26,6 +26,8 @@ func TestExpiredInitialInputPreparesNothingAndReleasesTheReservation(t *testing. r, s := f.lifecycle, f.session r.config.Mode = string(sandbox.DeploymentNodes) expireInitialInput(t, f) + wake := make(chan struct{}, 1) + r.placementWake = wake _, err := r.provision(t.Context(), s.TenantID, s.Environment.ID, r.config.InstallationID) if !errors.Is(err, deployment.ErrPlacementDemandLapsed) || !errors.Is(err, deployment.ErrAllocationConflict) { t.Fatal(err) @@ -38,6 +40,10 @@ func TestExpiredInitialInputPreparesNothingAndReleasesTheReservation(t *testing. if err != nil || workspaces != 0 || allocations != 0 || provider.creates != 0 || !target.PlacementReleased { t.Fatal("prepared for expired input", workspaces, allocations, provider.creates, target, err) } + // Input that arrived since the check is placed at once, not at the next tick. + if len(wake) != 1 { + t.Fatal("placement scan not woken after the release") + } } // lapsedReader reports a released allocation whose demand lapsed. diff --git a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go index 8d30a9d07..7f491d5e7 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go +++ b/services/core/internal/persistence/postgres/deploymentpg/placement_demand_test.go @@ -293,8 +293,9 @@ func TestReserveAllocationRefusesExpiredInitialInput(t *testing.T) { } } -// Released compute with no wake request and no live input has no demand, so reserving an allocation for it is refused. -func TestReserveAllocationRefusesReleasedComputeWithoutDemand(t *testing.T) { +// releasedWithoutDemand reserves and releases an allocation, then reserves a placement again: released compute with no wake request and no live input. +func releasedWithoutDemand(t *testing.T) (fixture, *deployment.ExecutionOperations, string, deployment.AllocationKey) { + t.Helper() f := newFixture(t) changes, _ := f.execution(t) installation, view := f.initialize(t, changes, sandbox.Selection{Provider: "microsandbox", DeploymentSpec: testSpecification("microsandbox")}) @@ -315,7 +316,42 @@ func TestReserveAllocationRefusesReleasedComputeWithoutDemand(t *testing.T) { if _, err := changes.EnsurePlacement(t.Context(), key, installation); err != nil { t.Fatal(err) } + return f, changes, installation, key +} + +// Released compute with no wake request and no live input has no demand, so reserving an allocation for it is refused. +func TestReserveAllocationRefusesReleasedComputeWithoutDemand(t *testing.T) { + f, changes, installation, key := releasedWithoutDemand(t) + if lapsed, err := f.adapter.PlacementDemandLapsed(t.Context(), key); err != nil || !lapsed { + t.Fatal("lapsed", lapsed, err) + } if _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()); !errors.Is(err, deployment.ErrPlacementDemandLapsed) { t.Fatal("released compute without demand reserved an allocation", err) } } + +// A reset, a failed Environment or a workspace that is not ready is not a lapsed demand: the Environment is not listed, and the refusal is the one those conditions always gave. +func TestOtherRefusalsAreNotReportedAsLapsedDemand(t *testing.T) { + for name, tc := range map[string]struct { + change string + want error + }{ + "reset": {`UPDATE runtime_deployment SET reset_clear='force', reset_requested_at=clock_timestamp(), reset_forced_at=clock_timestamp(), reset_audit='{}'`, placement.ErrResetAdmission}, + "failed": {`UPDATE environments SET status='failed'`, deployment.ErrAllocationConflict}, + "workspace missing": {`UPDATE environment_workspaces SET state='deleting'`, deployment.ErrAllocationConflict}, + } { + t.Run(name, func(t *testing.T) { + f, changes, installation, key := releasedWithoutDemand(t) + if _, err := f.pool.Exec(t.Context(), tc.change); err != nil { + t.Fatal(err) + } + if lapsed, err := f.adapter.PlacementDemandLapsed(t.Context(), key); err != nil || lapsed { + t.Fatal("lapsed", lapsed, err) + } + _, err := changes.ReserveAllocation(t.Context(), key, installation, credentialHash()) + if !errors.Is(err, tc.want) || errors.Is(err, deployment.ErrPlacementDemandLapsed) { + t.Fatal(err, "want", tc.want) + } + }) + } +} diff --git a/services/core/migrations/000016_placement_demand_view.sql b/services/core/migrations/000016_placement_demand_view.sql index 834140634..ceb23ab3a 100644 --- a/services/core/migrations/000016_placement_demand_view.sql +++ b/services/core/migrations/000016_placement_demand_view.sql @@ -1,4 +1,6 @@ -- +goose Up +CREATE INDEX environment_input_reservations_initial_idx ON environment_input_reservations (session_id) WHERE is_initial; + -- The one definition of placement demand. A row is an active hosted Environment that could take a node reservation: no live compute, either never allocated or with released compute and a ready workspace. -- waiting is whether it needs one now. A never-allocated Environment waits from its creation, unless its initial input expired and no input is live: then only from the earliest live input. One with released compute waits on a wake request or a live input. -- reserved marks one that already holds a placement; the scan skips it, and the preparation check reads it. @@ -12,7 +14,7 @@ FROM environments e JOIN environment_liveness l ON l.id = e.id AND l.active LEFT JOIN sessions s ON s.environment_id = e.id AND s.deleted_at IS NULL LEFT JOIN runtime_placements p ON p.environment_id = e.id LEFT JOIN LATERAL (SELECT a.id, a.state, a.compute_wake_requested, a.compute_activity_at FROM runtime_allocations a WHERE a.environment_id = e.id ORDER BY a.created_at DESC, a.id DESC LIMIT 1) latest ON true --- Input is read through its unique pending row, and the history only for an Environment never allocated, so the cost does not grow with input history. +-- Input is read through its unique pending row, and the initial input through its own partial index, only for an Environment never allocated, so the cost does not grow with input history. LEFT JOIN LATERAL (SELECT r.created_at AS live_at FROM environment_input_reservations r WHERE r.session_id = s.id AND r.state = 'pending' AND r.deadline > clock_timestamp()) i ON true LEFT JOIN LATERAL (SELECT (latest.id IS NULL AND EXISTS (SELECT 1 FROM environment_input_reservations r WHERE r.session_id = s.id AND r.is_initial AND (r.state = 'expired' OR (r.state = 'pending' AND r.deadline <= clock_timestamp())))) AS initial_expired) x ON true WHERE e.configuration->>'type' = 'openai_hosted'