From 952dae6100e43cf15bb4872e7a809671a04d9f18 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 18 Nov 2026 10:36:26 +0000 Subject: [PATCH 1/5] Test that unstarted compute fails a Session's unrun first input --- .../compute_start_deadline_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/services/core/tests/integration/compute_start_deadline_test.go b/services/core/tests/integration/compute_start_deadline_test.go index 498c147da..e3f1b5d5d 100644 --- a/services/core/tests/integration/compute_start_deadline_test.go +++ b/services/core/tests/integration/compute_start_deadline_test.go @@ -2,12 +2,14 @@ package integration import ( "context" + "encoding/json" "strings" "testing" "github.com/google/uuid" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions" ) // failedComputeProvider settles Create but its compute is natively failed, so @@ -93,3 +95,35 @@ func TestRunningAllocationIsNeverFailedByStartDeadline(t *testing.T) { t.Fatalf("running allocation was failed: kills=%d environment=%s", p.kills, body) } } + +func TestComputeThatNeverStartsFailsSessionWithUnrunInitialInputAtDeadline(t *testing.T) { + s, key := configuredStore(t) + tenant := uuid.NewString() + session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(), + Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`), + InitialInputs: []sessions.Input{messageInput("first")}})) + if err != nil { + t.Fatal(err) + } + p := &failedComputeProvider{lifecycleProvider{resources: map[string]sandbox.Info{}}} + w, _ := managedWorker(t, s, key, p) + if _, err := w.ProvisionEnvironment(t.Context(), tenant, session.Environment.ID, key); err == nil { + t.Fatal("compute that never starts provisioned") + } + backdateAllocations(t, s) + p.unavailable = true + for range 4 { + if err := w.ReconcileManagedRuntimes(t.Context()); err != nil { + t.Fatal(err) + } + } + // The input never ran, so the Session ends failed with the reason, never idle. + row, err := sessionAdapter(s).GetSession(t.Context(), tenant, session.ID) + if err != nil || row.Environment.Status != "failed" || row.EnvironmentFailure == nil || row.PendingInput || row.LastTurn != nil || + row.EnvironmentInputActivity == nil || row.EnvironmentInputActivity.Status != "failed" { + t.Fatal("unstarted compute left the initial input without a failure", row, err) + } + if events, err := sessionAdapter(s).ListSessionEvents(t.Context(), tenant, session.ID, 0); err != nil || len(events) == 0 || events[len(events)-1].Event.Type != "agent.session.failed" { + t.Fatal("unstarted compute did not report the failed Session", events, err) + } +} From d78b3db6dda65f590c808b1bb5ab53e4c375797e Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 18 Nov 2026 10:47:19 +0000 Subject: [PATCH 2/5] Keep a generation usable after its collection fails --- .../core/internal/sandbox/node/generations.go | 4 +++- .../internal/sandbox/node/generations_test.go | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 132720dad..14b2b455b 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -225,7 +225,9 @@ func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRe m.mu.Lock() defer m.mu.Unlock() if err != nil { - g.removing = false + // A failed removal leaves the generation as it was: a leftover collecting mark would make + // Core's later Deployment frame for it an ownership error that ends the node. + g.removing, g.collecting = false, false return err } if g.value.Close != nil { diff --git a/services/core/internal/sandbox/node/generations_test.go b/services/core/internal/sandbox/node/generations_test.go index b8256c771..252d8d174 100644 --- a/services/core/internal/sandbox/node/generations_test.go +++ b/services/core/internal/sandbox/node/generations_test.go @@ -311,3 +311,24 @@ func TestPreparationDiagnosticPreservesTypedCause(t *testing.T) { }) } } + +func TestFailedCollectionLeavesTheGenerationUsable(t *testing.T) { + m := generationFixture(3) + m.target.ServingGeneration = nil + removal := errors.New("remove failed") + m.options.Remove = func(context.Context, GenerationProvider) error { return removal } + grant := sandbox.GenerationRetention{GenerationReference: sandbox.GenerationReference{Generation: 2, SpecificationDigest: strings.Repeat("a", 64)}} + if err := m.Drop(t.Context(), grant); !errors.Is(err, removal) { + t.Fatal("removal error lost", err) + } + // Core naming the generation again must not end the node over a leftover collecting mark. + serving := uint64(2) + if err := m.Deployment(sandbox.NodeDeployment{Generation: 3, SpecificationDigest: strings.Repeat("a", 64), ServingGeneration: &serving}); err != nil { + t.Fatal("failed collection left an ownership error", err) + } + if _, _, release, err := m.Acquire(2); err != nil { + t.Fatal("failed collection left the generation unusable", err) + } else { + release() + } +} From fb79ca620e6f84670b39104b1be4e37e5c50db50 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 18 Nov 2026 10:47:50 +0000 Subject: [PATCH 3/5] Stop retrying collection of generations that never prepared --- deploy/node/node_generations.py | 35 +++++++++++++++++++--------- deploy/node/test_node_generations.py | 10 ++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/deploy/node/node_generations.py b/deploy/node/node_generations.py index 10b86ae87..1c2202ead 100644 --- a/deploy/node/node_generations.py +++ b/deploy/node/node_generations.py @@ -223,16 +223,29 @@ def collection_lease(root, generation, installer, identity=None, initialize=Fals os.close(descriptor) +def never_prepared(root, generation, installer): + """Whether the node holds no file of this generation, as when its preparation was cancelled before it wrote any.""" + directory = root / "state/node/generations" + base = installer.private_json(root / "provider.json") + if base and base["generation"] == generation: + return False + paths = [directory / (str(generation) + suffix) for suffix in (".json", ".lease", ".lease-identity", ".dropped")] + return not any(path.exists() or path.is_symlink() for path in paths) + + def collect(args, installer): root, _ = owned_root(args, installer) - with installer.install_lock(root), collection_lease(root, args.generation, installer, marker_identity(args)): - value = retained_configs(root, installer).get(args.generation) - if value is None: - return - if (value["installation_id"] != args.installation_id - or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest): - raise installer.InstallError("Collection grant does not match the local generation") - # The Runtime belongs to the installation; uninstall removes it. Keep the immutable - # configuration and this permanent journal as restart identity. They contain no - # provider credentials and are not readiness. - atomic_json(root / "state/node/generations" / (str(args.generation) + ".dropped"), marker_identity(args)) + with installer.install_lock(root): + if never_prepared(root, args.generation, installer): + return # Nothing to retain or release: a failure here would repeat at every retention heartbeat. + with collection_lease(root, args.generation, installer, marker_identity(args)): + value = retained_configs(root, installer).get(args.generation) + if value is None: + return + if (value["installation_id"] != args.installation_id + or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest): + raise installer.InstallError("Collection grant does not match the local generation") + # The Runtime belongs to the installation; uninstall removes it. Keep the immutable + # configuration and this permanent journal as restart identity. They contain no + # provider credentials and are not readiness. + atomic_json(root / "state/node/generations" / (str(args.generation) + ".dropped"), marker_identity(args)) diff --git a/deploy/node/test_node_generations.py b/deploy/node/test_node_generations.py index d2015a180..22346c2bd 100644 --- a/deploy/node/test_node_generations.py +++ b/deploy/node/test_node_generations.py @@ -83,6 +83,16 @@ def test_busy_helper_refuses_collection_then_same_inode_collects(self): final = lease.stat() self.assertEqual((original.st_dev, original.st_ino), (final.st_dev, final.st_ino)) + def test_collecting_a_generation_that_never_prepared_is_a_no_op(self): + self.next_generation({"cpus": 2, "memory_mib": 2048}) + node_generations.collect(self.args, installer) + self.assertEqual(sorted(path.name for path in self.directory.iterdir()), ["1.lease", "1.lease-identity"]) + # A generation with any file of its own still needs its identity and fails loudly without it. + (self.directory / "2.lease").write_bytes(b"") + os.chmod(self.directory / "2.lease", 0o600) + with self.assertRaises(OSError): + node_generations.collect(self.args, installer) + def test_collection_leaves_the_installations_runtime_and_other_generations(self): other = dict(self.value, generation=2) node_generations.atomic_json(self.directory / "2.json", other) From d1aa9b860848cd403a17ef762a3291fe38affdaf Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 18 Nov 2026 10:48:32 +0000 Subject: [PATCH 4/5] Test that a deployment change leaves a preparing sandbox on its generation --- .../runtime_node_generations_test.go | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/services/core/tests/integration/runtime_node_generations_test.go b/services/core/tests/integration/runtime_node_generations_test.go index 28cdd091c..d755788df 100644 --- a/services/core/tests/integration/runtime_node_generations_test.go +++ b/services/core/tests/integration/runtime_node_generations_test.go @@ -216,3 +216,37 @@ func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testin t.Fatal("ready node did not admit waiting Session", err) } } + +// A deployment change while a Session's Environment is still being prepared leaves +// that sandbox on the generation it was placed with; only Sessions created after the +// target is ready use the new one. +func TestDeploymentChangeDuringPreparationKeepsTheSandboxGeneration(t *testing.T) { + s, w, first, input := webSpecificationFixture(t, "microsandbox") + node := specificationNode(t, s, first) + connection := onlineManagerNode(t, s, node.NodeID) + preparing, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + if _, generation := placedGeneration(t, s, w, session); generation != 1 { + t.Fatal("placed on", generation) + } + second, _ := changeNodeTarget(t, w, first, input) + owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: preparing, EnvironmentID: session.Environment.ID}, first.InstallationID, runtimedevice.HashCredential(uuid.NewString())) + if err != nil || owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID { + t.Fatal("change moved the preparing sandbox", owner, err) + } + setup, err := deploymentService(t, s).AllocationSetup(t.Context(), sandbox.Reference{TenantID: preparing, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID}) + if err != nil || setup.Generation != 1 || setup.Specification.Resources.CPUs != first.Specification.Resources.CPUs { + t.Fatal("preparing sandbox lost its specification", setup, err) + } + generationHeartbeat(t, s, node, connection, second, "ready") + next, nextSession := managedArchiveSession(t, s, managerSessionInput(uuid.NewString())) + if _, generation := placedGeneration(t, s, w, nextSession); generation != 2 { + t.Fatal("new Session placed on", generation) + } + fresh, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: next, EnvironmentID: nextSession.Environment.ID}, first.InstallationID, runtimedevice.HashCredential(uuid.NewString())) + if err != nil || fresh.DeploymentGeneration != 2 { + t.Fatal("new Session did not use the new generation", fresh, err) + } + if setup, err = deploymentService(t, s).AllocationSetup(t.Context(), sandbox.Reference{TenantID: next, EnvironmentID: fresh.EnvironmentID, AllocationID: fresh.ID}); err != nil || setup.Specification.Resources.CPUs != first.Specification.Resources.CPUs+1 { + t.Fatal("new sandbox lost the new specification", setup, err) + } +} From 0a3f531da3e84a88318b3018d6f19c8b6bf69099 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Wed, 18 Nov 2026 10:48:49 +0000 Subject: [PATCH 5/5] Document collecting a generation the node never prepared --- deploy/node/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/node/README.md b/deploy/node/README.md index 9c8c2a716..5201ac298 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -57,7 +57,7 @@ Every generation owns a permanent private lease file, `state/node/generations/