Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion deploy/node/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ Every generation owns a permanent private lease file, `state/node/generations/<g

Collection takes the exclusive nonblocking flock before it inspects references, or publishes the dropped marker, and holds it through that change. Lease files belong to stable node state and are never removed or replaced during collection; symlinks, multiply linked files, foreign ownership, unsafe permissions and replaced lock paths are refused. Before the first helper can start, the installer creates the lease exclusively and fsyncs it, then atomically persists and fsyncs a private `.lease-identity` record and its directory. The record binds installation, generation, specification digest, device and inode. The Python collector and the Go helper opener validate the same record on every open, including after a restart; neither adopts a missing identity, replaces its inode or erases it after collection. An installation interrupted before the identity is durable refuses re-adoption and is kept for inspection. A removed identity or a replaced lease refuses even when its current metadata agree.

A dropped generation can never be prepared or used again. A helper's exit is evidence about local files only, not proof that a remote mutation or an uncertain provider receipt has been released; Core's durable allocation and placement retention stays independent.
Collecting a generation that has no file on the node, such as one whose preparation was cancelled before it wrote any, succeeds without change; a generation with any file of its own is collected only through the lease above. A dropped generation can never be prepared or used again. A helper's exit is evidence about local files only, not proof that a remote mutation or an uncertain provider receipt has been released; Core's durable allocation and placement retention stays independent.

### Installation payload

Expand Down
35 changes: 24 additions & 11 deletions deploy/node/node_generations.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,16 +223,29 @@ def collection_lease(root, generation, installer, identity=None, initialize=Fals
os.close(descriptor)


def never_prepared(root, generation, installer):
"""Whether the node holds no file of this generation, as when its preparation was cancelled before it wrote any."""
directory = root / "state/node/generations"
base = installer.private_json(root / "provider.json")
if base and base["generation"] == generation:
return False
paths = [directory / (str(generation) + suffix) for suffix in (".json", ".lease", ".lease-identity", ".dropped")]
return not any(path.exists() or path.is_symlink() for path in paths)


def collect(args, installer):
root, _ = owned_root(args, installer)
with installer.install_lock(root), collection_lease(root, args.generation, installer, marker_identity(args)):
value = retained_configs(root, installer).get(args.generation)
if value is None:
return
if (value["installation_id"] != args.installation_id
or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest):
raise installer.InstallError("Collection grant does not match the local generation")
# The Runtime belongs to the installation; uninstall removes it. Keep the immutable
# configuration and this permanent journal as restart identity. They contain no
# provider credentials and are not readiness.
atomic_json(root / "state/node/generations" / (str(args.generation) + ".dropped"), marker_identity(args))
with installer.install_lock(root):
if never_prepared(root, args.generation, installer):
return # Nothing to retain or release: a failure here would repeat at every retention heartbeat.
with collection_lease(root, args.generation, installer, marker_identity(args)):
value = retained_configs(root, installer).get(args.generation)
if value is None:
return
if (value["installation_id"] != args.installation_id
or installer.node_spec.digest(value["provider"], value["specification"]) != args.specification_digest):
raise installer.InstallError("Collection grant does not match the local generation")
# The Runtime belongs to the installation; uninstall removes it. Keep the immutable
# configuration and this permanent journal as restart identity. They contain no
# provider credentials and are not readiness.
atomic_json(root / "state/node/generations" / (str(args.generation) + ".dropped"), marker_identity(args))
10 changes: 10 additions & 0 deletions deploy/node/test_node_generations.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ def test_busy_helper_refuses_collection_then_same_inode_collects(self):
final = lease.stat()
self.assertEqual((original.st_dev, original.st_ino), (final.st_dev, final.st_ino))

def test_collecting_a_generation_that_never_prepared_is_a_no_op(self):
self.next_generation({"cpus": 2, "memory_mib": 2048})
node_generations.collect(self.args, installer)
self.assertEqual(sorted(path.name for path in self.directory.iterdir()), ["1.lease", "1.lease-identity"])
# A generation with any file of its own still needs its identity and fails loudly without it.
(self.directory / "2.lease").write_bytes(b"")
os.chmod(self.directory / "2.lease", 0o600)
with self.assertRaises(OSError):
node_generations.collect(self.args, installer)

def test_collection_leaves_the_installations_runtime_and_other_generations(self):
other = dict(self.value, generation=2)
node_generations.atomic_json(self.directory / "2.json", other)
Expand Down
4 changes: 3 additions & 1 deletion services/core/internal/sandbox/node/generations.go
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,9 @@ func (m *GenerationManager) Drop(ctx context.Context, grant sandbox.GenerationRe
m.mu.Lock()
defer m.mu.Unlock()
if err != nil {
g.removing = false
// A failed removal leaves the generation as it was: a leftover collecting mark would make
// Core's later Deployment frame for it an ownership error that ends the node.
g.removing, g.collecting = false, false
return err
}
if g.value.Close != nil {
Expand Down
21 changes: 21 additions & 0 deletions services/core/internal/sandbox/node/generations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,3 +311,24 @@ func TestPreparationDiagnosticPreservesTypedCause(t *testing.T) {
})
}
}

func TestFailedCollectionLeavesTheGenerationUsable(t *testing.T) {
m := generationFixture(3)
m.target.ServingGeneration = nil
removal := errors.New("remove failed")
m.options.Remove = func(context.Context, GenerationProvider) error { return removal }
grant := sandbox.GenerationRetention{GenerationReference: sandbox.GenerationReference{Generation: 2, SpecificationDigest: strings.Repeat("a", 64)}}
if err := m.Drop(t.Context(), grant); !errors.Is(err, removal) {
t.Fatal("removal error lost", err)
}
// Core naming the generation again must not end the node over a leftover collecting mark.
serving := uint64(2)
if err := m.Deployment(sandbox.NodeDeployment{Generation: 3, SpecificationDigest: strings.Repeat("a", 64), ServingGeneration: &serving}); err != nil {
t.Fatal("failed collection left an ownership error", err)
}
if _, _, release, err := m.Acquire(2); err != nil {
t.Fatal("failed collection left the generation unusable", err)
} else {
release()
}
}
34 changes: 34 additions & 0 deletions services/core/tests/integration/compute_start_deadline_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@ package integration

import (
"context"
"encoding/json"
"strings"
"testing"

"github.com/google/uuid"

"github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox"
"github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sessions"
)

// failedComputeProvider settles Create but its compute is natively failed, so
Expand Down Expand Up @@ -93,3 +95,35 @@ func TestRunningAllocationIsNeverFailedByStartDeadline(t *testing.T) {
t.Fatalf("running allocation was failed: kills=%d environment=%s", p.kills, body)
}
}

func TestComputeThatNeverStartsFailsSessionWithUnrunInitialInputAtDeadline(t *testing.T) {
s, key := configuredStore(t)
tenant := uuid.NewString()
session, err := s.CreateSession(t.Context(), tenant, WithFixtureModelProvider(sessions.CreateSession{Creator: FixtureCreator(), Engine: "codex", IdempotencyKey: uuid.NewString(),
Configuration: json.RawMessage(`{"agent":{"model":"test"},"environment":{"type":"openai_hosted","network":{"access":"enabled"}}}`),
InitialInputs: []sessions.Input{messageInput("first")}}))
if err != nil {
t.Fatal(err)
}
p := &failedComputeProvider{lifecycleProvider{resources: map[string]sandbox.Info{}}}
w, _ := managedWorker(t, s, key, p)
if _, err := w.ProvisionEnvironment(t.Context(), tenant, session.Environment.ID, key); err == nil {
t.Fatal("compute that never starts provisioned")
}
backdateAllocations(t, s)
p.unavailable = true
for range 4 {
if err := w.ReconcileManagedRuntimes(t.Context()); err != nil {
t.Fatal(err)
}
}
// The input never ran, so the Session ends failed with the reason, never idle.
row, err := sessionAdapter(s).GetSession(t.Context(), tenant, session.ID)
if err != nil || row.Environment.Status != "failed" || row.EnvironmentFailure == nil || row.PendingInput || row.LastTurn != nil ||
row.EnvironmentInputActivity == nil || row.EnvironmentInputActivity.Status != "failed" {
t.Fatal("unstarted compute left the initial input without a failure", row, err)
}
if events, err := sessionAdapter(s).ListSessionEvents(t.Context(), tenant, session.ID, 0); err != nil || len(events) == 0 || events[len(events)-1].Event.Type != "agent.session.failed" {
t.Fatal("unstarted compute did not report the failed Session", events, err)
}
}
34 changes: 34 additions & 0 deletions services/core/tests/integration/runtime_node_generations_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,3 +216,37 @@ func TestNodeGenerationPreparationRefusalCreatesNoProvisionalOwnership(t *testin
t.Fatal("ready node did not admit waiting Session", err)
}
}

// A deployment change while a Session's Environment is still being prepared leaves
// that sandbox on the generation it was placed with; only Sessions created after the
// target is ready use the new one.
func TestDeploymentChangeDuringPreparationKeepsTheSandboxGeneration(t *testing.T) {
s, w, first, input := webSpecificationFixture(t, "microsandbox")
node := specificationNode(t, s, first)
connection := onlineManagerNode(t, s, node.NodeID)
preparing, session := managedArchiveSession(t, s, managerSessionInput(uuid.NewString()))
if _, generation := placedGeneration(t, s, w, session); generation != 1 {
t.Fatal("placed on", generation)
}
second, _ := changeNodeTarget(t, w, first, input)
owner, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: preparing, EnvironmentID: session.Environment.ID}, first.InstallationID, runtimedevice.HashCredential(uuid.NewString()))
if err != nil || owner.DeploymentGeneration != 1 || owner.NodeID != node.NodeID {
t.Fatal("change moved the preparing sandbox", owner, err)
}
setup, err := deploymentService(t, s).AllocationSetup(t.Context(), sandbox.Reference{TenantID: preparing, EnvironmentID: owner.EnvironmentID, AllocationID: owner.ID})
if err != nil || setup.Generation != 1 || setup.Specification.Resources.CPUs != first.Specification.Resources.CPUs {
t.Fatal("preparing sandbox lost its specification", setup, err)
}
generationHeartbeat(t, s, node, connection, second, "ready")
next, nextSession := managedArchiveSession(t, s, managerSessionInput(uuid.NewString()))
if _, generation := placedGeneration(t, s, w, nextSession); generation != 2 {
t.Fatal("new Session placed on", generation)
}
fresh, err := deploymentExecution(t, w).ReserveAllocation(t.Context(), deployment.AllocationKey{TenantID: next, EnvironmentID: nextSession.Environment.ID}, first.InstallationID, runtimedevice.HashCredential(uuid.NewString()))
if err != nil || fresh.DeploymentGeneration != 2 {
t.Fatal("new Session did not use the new generation", fresh, err)
}
if setup, err = deploymentService(t, s).AllocationSetup(t.Context(), sandbox.Reference{TenantID: next, EnvironmentID: fresh.EnvironmentID, AllocationID: fresh.ID}); err != nil || setup.Specification.Resources.CPUs != first.Specification.Resources.CPUs+1 {
t.Fatal("new sandbox lost the new specification", setup, err)
}
}
Loading