add automatic heap snapshots for high-memory cli processes (#20788)
parent
8daeacc989
commit
aa2239d5de
|
|
@ -13,6 +13,7 @@ import { Flag } from "@/flag/flag"
|
||||||
import { setTimeout as sleep } from "node:timers/promises"
|
import { setTimeout as sleep } from "node:timers/promises"
|
||||||
import { writeHeapSnapshot } from "node:v8"
|
import { writeHeapSnapshot } from "node:v8"
|
||||||
import { WorkspaceID } from "@/control-plane/schema"
|
import { WorkspaceID } from "@/control-plane/schema"
|
||||||
|
import { Heap } from "@/cli/heap"
|
||||||
|
|
||||||
await Log.init({
|
await Log.init({
|
||||||
print: process.argv.includes("--print-logs"),
|
print: process.argv.includes("--print-logs"),
|
||||||
|
|
@ -23,6 +24,8 @@ await Log.init({
|
||||||
})(),
|
})(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
Heap.start()
|
||||||
|
|
||||||
process.on("unhandledRejection", (e) => {
|
process.on("unhandledRejection", (e) => {
|
||||||
Log.Default.error("rejection", {
|
Log.Default.error("rejection", {
|
||||||
e: e instanceof Error ? e.message : e,
|
e: e instanceof Error ? e.message : e,
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,59 @@
|
||||||
|
import path from "path"
|
||||||
|
import { writeHeapSnapshot } from "node:v8"
|
||||||
|
import { Flag } from "@/flag/flag"
|
||||||
|
import { Global } from "@/global"
|
||||||
|
import { Log } from "@/util/log"
|
||||||
|
|
||||||
|
const log = Log.create({ service: "heap" })
|
||||||
|
const MINUTE = 60_000
|
||||||
|
const LIMIT = 2 * 1024 * 1024 * 1024
|
||||||
|
|
||||||
|
export namespace Heap {
|
||||||
|
let timer: Timer | undefined
|
||||||
|
let lock = false
|
||||||
|
let armed = true
|
||||||
|
|
||||||
|
export function start() {
|
||||||
|
if (!Flag.OPENCODE_AUTO_HEAP_SNAPSHOT) return
|
||||||
|
if (timer) return
|
||||||
|
|
||||||
|
const run = async () => {
|
||||||
|
if (lock) return
|
||||||
|
|
||||||
|
const stat = process.memoryUsage()
|
||||||
|
if (stat.rss <= LIMIT) {
|
||||||
|
armed = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (!armed) return
|
||||||
|
|
||||||
|
lock = true
|
||||||
|
armed = false
|
||||||
|
const file = path.join(
|
||||||
|
Global.Path.log,
|
||||||
|
`heap-${process.pid}-${new Date().toISOString().replace(/[:.]/g, "")}.heapsnapshot`,
|
||||||
|
)
|
||||||
|
log.warn("heap usage exceeded limit", {
|
||||||
|
rss: stat.rss,
|
||||||
|
heap: stat.heapUsed,
|
||||||
|
file,
|
||||||
|
})
|
||||||
|
|
||||||
|
await Promise.resolve()
|
||||||
|
.then(() => writeHeapSnapshot(file))
|
||||||
|
.catch((err) => {
|
||||||
|
log.error("failed to write heap snapshot", {
|
||||||
|
error: err instanceof Error ? err.message : String(err),
|
||||||
|
file,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
lock = false
|
||||||
|
}
|
||||||
|
|
||||||
|
timer = setInterval(() => {
|
||||||
|
void run()
|
||||||
|
}, MINUTE)
|
||||||
|
timer.unref?.()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -12,6 +12,7 @@ function falsy(key: string) {
|
||||||
|
|
||||||
export namespace Flag {
|
export namespace Flag {
|
||||||
export const OPENCODE_AUTO_SHARE = truthy("OPENCODE_AUTO_SHARE")
|
export const OPENCODE_AUTO_SHARE = truthy("OPENCODE_AUTO_SHARE")
|
||||||
|
export const OPENCODE_AUTO_HEAP_SNAPSHOT = truthy("OPENCODE_AUTO_HEAP_SNAPSHOT")
|
||||||
export const OPENCODE_GIT_BASH_PATH = process.env["OPENCODE_GIT_BASH_PATH"]
|
export const OPENCODE_GIT_BASH_PATH = process.env["OPENCODE_GIT_BASH_PATH"]
|
||||||
export const OPENCODE_CONFIG = process.env["OPENCODE_CONFIG"]
|
export const OPENCODE_CONFIG = process.env["OPENCODE_CONFIG"]
|
||||||
export declare const OPENCODE_PURE: boolean
|
export declare const OPENCODE_PURE: boolean
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ import { JsonMigration } from "./storage/json-migration"
|
||||||
import { Database } from "./storage/db"
|
import { Database } from "./storage/db"
|
||||||
import { errorMessage } from "./util/error"
|
import { errorMessage } from "./util/error"
|
||||||
import { PluginCommand } from "./cli/cmd/plug"
|
import { PluginCommand } from "./cli/cmd/plug"
|
||||||
|
import { Heap } from "./cli/heap"
|
||||||
|
|
||||||
process.on("unhandledRejection", (e) => {
|
process.on("unhandledRejection", (e) => {
|
||||||
Log.Default.error("rejection", {
|
Log.Default.error("rejection", {
|
||||||
|
|
@ -96,6 +97,8 @@ const cli = yargs(args)
|
||||||
})(),
|
})(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
Heap.start()
|
||||||
|
|
||||||
process.env.AGENT = "1"
|
process.env.AGENT = "1"
|
||||||
process.env.OPENCODE = "1"
|
process.env.OPENCODE = "1"
|
||||||
process.env.OPENCODE_PID = String(process.pid)
|
process.env.OPENCODE_PID = String(process.pid)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue