Skip to content

Commit 47c87eb

Browse files
authored
fix(pool): terminate stuck orphaned forks workers (#11939)
1 parent f4fdbca commit 47c87eb

6 files changed

Lines changed: 156 additions & 18 deletions

File tree

‎packages/vitest/rollup.config.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ const entries = {
3838
'workers/threads': 'src/runtime/workers/threads.ts',
3939
'workers/vmThreads': 'src/runtime/workers/vmThreads.ts',
4040
'workers/vmForks': 'src/runtime/workers/vmForks.ts',
41+
'workers/watchdog': 'src/runtime/workers/watchdog.ts',
4142

4243
'workers/runVmTests': 'src/runtime/runVmTests.ts',
4344
}

‎packages/vitest/src/node/pools/pool.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { Logger } from '../logger'
44
import type { StateManager } from '../state'
55
import type { PoolOptions, PoolTask, WorkerResponse } from './types'
66
import { PoolRunner } from './poolRunner'
7+
import { stopWatchdog } from './watchdog'
78
import { ForksPoolWorker } from './workers/forksWorker'
89
import { ThreadsPoolWorker } from './workers/threadsWorker'
910
import { TypecheckPoolWorker } from './workers/typecheckWorker'
@@ -240,6 +241,7 @@ export class Pool {
240241

241242
async close(): Promise<void> {
242243
await this.cancel()
244+
stopWatchdog()
243245
}
244246

245247
private getPoolRunner(task: PoolTask, method: 'run' | 'collect'): PoolRunner {
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
import type { ChildProcess } from 'node:child_process'
2+
import { fork } from 'node:child_process'
3+
import { resolve } from 'node:path'
4+
5+
export type WatchdogMessage = { type: 'watch'; pid: number } | { type: 'unwatch'; pid: number }
6+
7+
let watchdog: ChildProcess | undefined
8+
const watched = new Set<number>()
9+
10+
export function watchProcess(child: ChildProcess, distPath: string): void {
11+
const pid = child.pid
12+
if (pid == null) {
13+
return
14+
}
15+
16+
watchdog ??= startWatchdog(distPath)
17+
watched.add(pid)
18+
send({ type: 'watch', pid })
19+
20+
child.once('exit', () => {
21+
watched.delete(pid)
22+
send({ type: 'unwatch', pid })
23+
})
24+
}
25+
26+
export function stopWatchdog(): void {
27+
if (watchdog && watched.size === 0) {
28+
watchdog.disconnect()
29+
watchdog = undefined
30+
}
31+
}
32+
33+
function send(message: WatchdogMessage) {
34+
if (watchdog?.connected) {
35+
watchdog.send(message)
36+
}
37+
}
38+
39+
function startWatchdog(distPath: string): ChildProcess {
40+
/** Loads {@link file://./../../runtime/workers/watchdog.ts} */
41+
const child = fork(resolve(distPath, 'workers/watchdog.js'), [], {
42+
stdio: ['ignore', 'ignore', 'ignore', 'ipc'],
43+
execArgv: [],
44+
env: { ...process.env, NODE_OPTIONS: undefined },
45+
})
46+
47+
// must never keep the main process alive
48+
child.unref()
49+
child.channel?.unref()
50+
51+
// losing the watchdog only loses the cleanup, the test run is not affected
52+
child.on('error', () => {})
53+
child.once('exit', () => {
54+
if (watchdog === child) {
55+
watchdog = undefined
56+
}
57+
})
58+
59+
return child
60+
}

‎packages/vitest/src/node/pools/workers/forksWorker.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { PoolOptions, PoolWorker, WorkerRequest } from '../types'
44
import { fork } from 'node:child_process'
55
import { EventEmitter } from 'node:events'
66
import { resolve } from 'node:path'
7+
import { watchProcess } from '../watchdog'
78
import { streamFlushed } from './utils'
89

910
const SIGKILL_TIMEOUT = 500 /** jest does 500ms by default, let's follow it */
@@ -19,6 +20,7 @@ export class ForksPoolWorker implements PoolWorker {
1920
protected readonly entrypoint: string
2021
protected execArgv: string[]
2122
protected env: Partial<NodeJS.ProcessEnv>
23+
private distPath: string
2224

2325
private _fork?: ChildProcess
2426
private stdout: NodeJS.WriteStream | Writable
@@ -32,6 +34,7 @@ export class ForksPoolWorker implements PoolWorker {
3234
this.env = options.env
3335
this.stdout = options.project.vitest.logger.outputStream
3436
this.stderr = options.project.vitest.logger.errorStream
37+
this.distPath = options.distPath
3538

3639
/** Loads {@link file://./../../../runtime/workers/forks.ts} */
3740
this.entrypoint = resolve(options.distPath, 'workers/forks.js')
@@ -66,6 +69,7 @@ export class ForksPoolWorker implements PoolWorker {
6669
})
6770

6871
this._fork.on('error', this.emitError)
72+
watchProcess(this._fork, this.distPath)
6973

7074
// `end: false`: the logger streams are shared by every worker, so one
7175
// ending worker stream must not end them for everyone else
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import type { WatchdogMessage } from '../../node/pools/watchdog'
2+
3+
process.title = 'vitest-watchdog'
4+
5+
const pids = new Set<number>()
6+
7+
// Ctrl+C and closing the terminal signal the whole process group, but the
8+
// watchdog must outlive the main process to clean up after it
9+
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP'] as const) {
10+
process.on(signal, () => {})
11+
}
12+
13+
process.on('message', (message: WatchdogMessage) => {
14+
if (message.type === 'watch') {
15+
pids.add(message.pid)
16+
} else if (message.type === 'unwatch') {
17+
pids.delete(message.pid)
18+
}
19+
})
20+
21+
// Workers blocked on synchronous code cannot notice the main process dying
22+
process.on('disconnect', () => {
23+
for (const pid of pids) {
24+
try {
25+
process.kill(pid, 'SIGKILL')
26+
} catch {}
27+
}
28+
process.exit()
29+
})

‎test/e2e/test/pool-worker-exit.test.ts‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { TestRunEndReason } from 'vitest/node'
22
import { spawn } from 'node:child_process'
3-
import { existsSync, readFileSync } from 'node:fs'
3+
import { readFileSync } from 'node:fs'
44
import { sep } from 'node:path'
55
import { resolve } from 'pathe'
66
import { expect, onTestFinished, test } from 'vitest'
@@ -229,28 +229,57 @@ test('run with a queued module after worker death is reported as failed', async
229229
})
230230

231231
test('forks worker exits when the main process dies', async () => {
232-
const fs = useTmpFS({
233-
'idle.test.ts': `
234-
import { writeFileSync } from 'node:fs'
235-
import { test } from 'vitest'
232+
const workerPid = await killMainProcessDuringTest(
233+
'forks',
234+
`
235+
setInterval(() => {}, 1_000)
236+
237+
console.log(\`worker-pid:\${process.pid}\`)
238+
await new Promise(() => {})
239+
`,
240+
)
236241

237-
test('idle', async () => {
238-
// keeps the event loop alive like an open server would
239-
setInterval(() => {}, 1_000)
242+
await expect.poll(() => isAlive(workerPid), { timeout: 5_000 }).toBe(false)
243+
})
240244

241-
// let the throttled task updates reach main process first
242-
await new Promise((resolve) => setTimeout(resolve, 500))
245+
test.for(['forks', 'vmForks'])(
246+
'%s worker blocked by synchronous code is killed when the main process dies',
247+
async (pool) => {
248+
const workerPid = await killMainProcessDuringTest(
249+
pool,
250+
`
251+
console.log(\`worker-pid:\${process.pid}\`)
243252
244-
writeFileSync('./worker.pid', String(process.pid))
245-
await new Promise(() => {})
253+
await new Promise((resolve) => setImmediate(resolve))
254+
while (true) {}
255+
`,
256+
)
257+
258+
await expect.poll(() => isAlive(workerPid), { timeout: 5_000 }).toBe(false)
259+
},
260+
)
261+
262+
async function killMainProcessDuringTest(pool: string, testBody: string) {
263+
const fs = useTmpFS({
264+
'basic.test.ts': `
265+
import { test } from 'vitest'
266+
267+
test('basic', async () => {
268+
${testBody}
246269
})
247270
`,
248271
})
249272

250273
const main = spawn(
251274
process.execPath,
252-
[resolve('node_modules/vitest/vitest.mjs'), '--run', '--pool=forks', '--testTimeout=0'],
253-
{ cwd: fs.root, stdio: 'ignore' },
275+
[
276+
resolve('node_modules/vitest/vitest.mjs'),
277+
'--run',
278+
`--pool=${pool}`,
279+
'--testTimeout=0',
280+
'--reporter=default',
281+
],
282+
{ cwd: fs.root, stdio: ['ignore', 'pipe', 'ignore'] },
254283
)
255284
const mainExit = new Promise((resolve) => main.once('exit', resolve))
256285

@@ -265,15 +294,28 @@ test('forks worker exits when the main process dies', async () => {
265294
await mainExit
266295
})
267296

268-
await expect.poll(() => existsSync(fs.resolveFile('worker.pid')), { timeout: 10_000 }).toBe(true)
269-
workerPid = Number(fs.readFile('worker.pid'))
297+
workerPid = await new Promise<number>((resolve, reject) => {
298+
let stdout = ''
299+
300+
main.stdout.on('data', (chunk) => {
301+
stdout += chunk
302+
303+
const pid = stdout.match(/worker-pid:(\d+)/)?.[1]
304+
305+
if (pid) {
306+
resolve(Number(pid))
307+
}
308+
})
309+
310+
main.once('exit', () => reject(new Error(`Vitest exited before the test started:\n${stdout}`)))
311+
})
270312
expect(isAlive(workerPid)).toBe(true)
271313

272314
main.kill('SIGKILL')
273315
await mainExit
274316

275-
await expect.poll(() => isAlive(workerPid!), { timeout: 5_000 }).toBe(false)
276-
})
317+
return workerPid
318+
}
277319

278320
function isAlive(pid: number) {
279321
try {

0 commit comments

Comments
 (0)