From 8d4c7c228b41f55fdbe0c4c0b2a060e9081ed404 Mon Sep 17 00:00:00 2001 From: SaladDay <92240037+SaladDay@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:22:14 +0800 Subject: [PATCH 1/2] Warn once when a Runtime history exporter starts failing Export errors and exporter panics were discarded without any trace, so an unreachable OTLP collector or a failing history insert looked exactly like a Runtime with no history. The dispatcher now logs one warning at the start of each outage with the exporter type and outcome class (error, timeout, panic); the error text is not logged because it may carry a collector response. Observation results and queue behaviour are unchanged. --- contracts/agents-api/runtime-observability.md | 2 +- .../agents-api/zh/runtime-observability.md | 2 +- services/core/internal/runtimeobs/exporter.go | 29 ++++++- .../core/internal/runtimeobs/exporter_test.go | 76 +++++++++++++++++++ 4 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 services/core/internal/runtimeobs/exporter_test.go diff --git a/contracts/agents-api/runtime-observability.md b/contracts/agents-api/runtime-observability.md index f39873f00..12b4fe93e 100644 --- a/contracts/agents-api/runtime-observability.md +++ b/contracts/agents-api/runtime-observability.md @@ -94,7 +94,7 @@ Periodic collection runs in the execution Worker, under its database lease. The sampler sweeps once at startup and again each sampling interval after the previous sweep ends. A sweep is a keyset scan, in Session ID order, of the Sessions that are not deleted, are `openai_hosted` and have no released allocation. It reads pages of 32 Sessions through the same resolver and sources as current reads, with eight concurrent reads and two seconds per source. The sampler checks the lease before each page and every 100 ms during a sweep, cancels in-flight reads when ownership is lost, and checks it again before handing each record to export. A failed row does not stop the sweep, and an incomplete sweep is repeated at the next interval. -Every observation, current or periodic, is marked with its collection source, `on_read` or `periodic`, and handed to each exporter's bounded queue. A full queue drops the record, which becomes a missing sample, never a zero. The PostgreSQL history store and the optional OTLP exporter have independent queues, so an exporter outage cannot delay local history or execution. The [Runtime history file](../../docs/configuration.md#runtime-history-file) sets the interval, queue capacity, timeout and OTLP destination. +Every observation, current or periodic, is marked with its collection source, `on_read` or `periodic`, and handed to each exporter's bounded queue. A full queue drops the record, which becomes a missing sample, never a zero. The PostgreSQL history store and the optional OTLP exporter have independent queues, so an exporter outage cannot delay local history or execution. A failing exporter logs one warning, naming the exporter and the outcome class but not its error text, each time it starts failing. The [Runtime history file](../../docs/configuration.md#runtime-history-file) sets the interval, queue capacity, timeout and OTLP destination. ### Stored history diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index 063489a25..dce8683e3 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -96,7 +96,7 @@ CPU 静默状态、心跳时龄和连接状态都不是空闲时间。 采样器在启动时扫描一次,此后每次扫描结束后再经过一个采样间隔再次扫描。一次扫描按 Session ID 顺序,对未删除、状态为 `openai_hosted` 且没有已释放分配的 Session 执行 keyset 扫描。它通过与当前读取相同的解析器和源,以 32 个 Session 为一页进行读取,并发数为 8,每个源时限为 2 秒。采样器在每页之前以及扫描期间每 100 ms 检查租约;失去所有权时取消进行中的读取;将每条记录交给导出之前再次检查租约。失败的行不会停止扫描;未完成的扫描会在下一个间隔重复。 -每次观测,无论来自当前读取还是周期采集,都会标记采集源 `on_read` 或 `periodic`,并放入每个导出器的有界队列。队列已满时会丢弃记录;该记录将成为缺失采样,而绝不会成为零。PostgreSQL 历史存储和可选 OTLP 导出器使用彼此独立的队列,因此导出器故障不会延迟本地历史记录或执行。[Runtime 历史文件](../../../docs/zh/configuration.md#runtime-history-file) 用于设置采样间隔、队列容量、超时和 OTLP 目标。 +每次观测,无论来自当前读取还是周期采集,都会标记采集源 `on_read` 或 `periodic`,并放入每个导出器的有界队列。队列已满时会丢弃记录;该记录将成为缺失采样,而绝不会成为零。PostgreSQL 历史存储和可选 OTLP 导出器使用彼此独立的队列,因此导出器故障不会延迟本地历史记录或执行。导出器每次开始失败时记录一条警告,只包含导出器和结果类别,不包含其错误文本。[Runtime 历史文件](../../../docs/zh/configuration.md#runtime-history-file) 用于设置采样间隔、队列容量、超时和 OTLP 目标。 ### 存储的历史记录 {#stored-history} diff --git a/services/core/internal/runtimeobs/exporter.go b/services/core/internal/runtimeobs/exporter.go index 547b6bdd9..c8dc92ffd 100644 --- a/services/core/internal/runtimeobs/exporter.go +++ b/services/core/internal/runtimeobs/exporter.go @@ -3,8 +3,11 @@ package runtimeobs import ( "context" "errors" + "fmt" "sync" "time" + + "github.com/MiniMax-AI/OpenAgentCore/internal/obs/log" ) const ( @@ -121,21 +124,39 @@ func (d *exportDispatcher) enqueue(record ExportRecord) { func (d *exportDispatcher) run(ctx context.Context) { defer close(d.done) + failing := false for record := range d.queue { exportCtx, cancel := context.WithTimeout(ctx, d.timeout) - exportSafely(exportCtx, d.exporter, record) + outcome := exportSafely(exportCtx, d.exporter, record) cancel() if ctx.Err() != nil { return } + // Warn once per outage, not per record. The exporter's error may carry a + // collector response, so only its outcome class is logged. + if outcome != "" && !failing { + log.Warn(ctx, "Runtime history export failing; records are dropped until it recovers", "exporter", fmt.Sprintf("%T", d.exporter), "outcome", outcome) + } + failing = outcome != "" } } -func exportSafely(ctx context.Context, exporter Exporter, record ExportRecord) { +// exportSafely returns "" on success, otherwise "timeout", "error" or "panic". +func exportSafely(ctx context.Context, exporter Exporter, record ExportRecord) (outcome string) { defer func() { - _ = recover() + if recover() != nil { + outcome = "panic" + } }() - _ = exporter.Export(ctx, record) + err := exporter.Export(ctx, record) + switch { + case err == nil: + return "" + case errors.Is(err, context.DeadlineExceeded): + return "timeout" + default: + return "error" + } } func (d *exportDispatcher) close(ctx context.Context) error { diff --git a/services/core/internal/runtimeobs/exporter_test.go b/services/core/internal/runtimeobs/exporter_test.go new file mode 100644 index 000000000..9d6108cde --- /dev/null +++ b/services/core/internal/runtimeobs/exporter_test.go @@ -0,0 +1,76 @@ +package runtimeobs + +import ( + "bytes" + "context" + "errors" + "log/slog" + "strings" + "sync" + "testing" +) + +// scriptedExporter returns each scripted outcome in turn: nil, an error, or +// "panic". +type scriptedExporter struct { + mu sync.Mutex + outcomes []any +} + +func (e *scriptedExporter) Export(context.Context, ExportRecord) error { + e.mu.Lock() + outcome := e.outcomes[0] + e.outcomes = e.outcomes[1:] + e.mu.Unlock() + if outcome == "panic" { + panic("collector client bug") + } + err, _ := outcome.(error) + return err +} + +// syncBuffer lets the dispatcher goroutine write while the test reads after Close. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func TestExportFailureIsReportedOncePerOutage(t *testing.T) { + logs := &syncBuffer{} + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(logs, nil))) + defer slog.SetDefault(previous) + + secret := "collector said: token=leaked" + exporter := &scriptedExporter{outcomes: []any{ + errors.New(secret), errors.New(secret), nil, // one outage, then recovery + "panic", nil, // a panicking exporter is a second outage + nil, + }} + dispatcher := newExportDispatcher(exporter, ExportOptions{QueueCapacity: 8}) + for range 6 { + dispatcher.enqueue(ExportRecord{}) + } + if err := dispatcher.close(t.Context()); err != nil { + t.Fatal(err) + } + output := logs.String() + if got := strings.Count(output, "Runtime history export failing"); got != 2 { + t.Fatalf("want one warning per outage (2), got %d:\n%s", got, output) + } + if strings.Contains(output, "leaked") { + t.Fatalf("exporter error text reached the log:\n%s", output) + } +} From 8f198ccb3588709230d0d537b0b8ce92ae13f46d Mon Sep 17 00:00:00 2001 From: SaladDay <92240037+SaladDay@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:19:45 +0800 Subject: [PATCH 2/2] Refresh the Chinese runtime observability translation hash --- contracts/agents-api/zh/runtime-observability.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/agents-api/zh/runtime-observability.md b/contracts/agents-api/zh/runtime-observability.md index dce8683e3..f610a11c4 100644 --- a/contracts/agents-api/zh/runtime-observability.md +++ b/contracts/agents-api/zh/runtime-observability.md @@ -1,7 +1,7 @@ --- title: "Runtime 可观测性" source: contracts/agents-api/runtime-observability.md -source_hash: 5a79ce08bd5c611abae2a3a488e80b1efeb8316bd2939f1a311c0579cc22792b +source_hash: 074d31e8c05493593eb48a95de2c9b2c28aafd3695e5baf16815e3f7a60e18ad --- 这是面向贡献者的契约,规定 Core 如何观测 Runtime 并保留其历史。路由和响应字段见 [Runtime telemetry API](runtime-observability-api.md)。代码位于 `services/core/internal/runtimeobs`(解析、源、采样器和导出)、`services/core/internal/runtimehistory`(历史查询和 PostgreSQL 存储)以及 `services/core/internal/runtimeobs/otlpexporter`。