Repository navigation
Fix rs-dds-adapter process leak on Windows - #14845
Conversation
There was a problem hiding this comment.
Pull request overview
Fixes a Windows-specific rs-dds-adapter process leak in the DDS adapter depth unit test, which was contaminating subsequent (non-DDS) tests by continuing to republish USB devices as DDS.
Changes:
- Add a pre-test cleanup step to kill any leftover
rs-dds-adapterprocesses from prior runs. - Replace
SIGTERM-based shutdown withPopen.kill()and an additional system-wide kill-by-name safety net (taskkill/pkill). - Extend the wait timeout after killing the adapter process.
💡 Add Copilot custom instructions for smarter, more guided reviews. Learn how to get started.
| """Kill any leftover rs-dds-adapter processes (e.g. from a previous crashed test run)""" | ||
| try: | ||
| if platform.system() == 'Windows': | ||
| subprocess.run( ['taskkill', '/F', '/IM', 'rs-dds-adapter.exe'], | ||
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) | ||
| else: | ||
| subprocess.run( ['pkill', '-9', '-f', 'rs-dds-adapter'], | ||
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) | ||
| except Exception: | ||
| pass | ||
|
|
||
|
|
There was a problem hiding this comment.
kill_all_dds_adapters() swallows all exceptions and also ignores the subprocess return codes, so if taskkill/pkill is missing or fails (permissions, etc.) the test will continue without any indication and may still leak the adapter into subsequent tests. Consider logging at least a debug/warn on unexpected failures and/or returning a boolean so callers can abort when cleanup is required but couldn’t be performed.
| """Kill any leftover rs-dds-adapter processes (e.g. from a previous crashed test run)""" | |
| try: | |
| if platform.system() == 'Windows': | |
| subprocess.run( ['taskkill', '/F', '/IM', 'rs-dds-adapter.exe'], | |
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) | |
| else: | |
| subprocess.run( ['pkill', '-9', '-f', 'rs-dds-adapter'], | |
| stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL ) | |
| except Exception: | |
| pass | |
| """ | |
| Kill any leftover rs-dds-adapter processes (e.g. from a previous crashed test run). | |
| Returns: | |
| bool: True if the kill command was executed and reported success (exit code 0), | |
| False if the command failed, was not found, or an unexpected error occurred. | |
| """ | |
| try: | |
| if platform.system() == 'Windows': | |
| cmd = ['taskkill', '/F', '/IM', 'rs-dds-adapter.exe'] | |
| else: | |
| cmd = ['pkill', '-9', '-f', 'rs-dds-adapter'] | |
| result = subprocess.run( | |
| cmd, | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL | |
| ) | |
| if result.returncode != 0: | |
| log.w(f"kill_all_dds_adapters: command {cmd} exited with return code {result.returncode}") | |
| return False | |
| return True | |
| except FileNotFoundError as e: | |
| # taskkill/pkill not available on this system | |
| log.w(f"kill_all_dds_adapters: command not found when attempting to kill rs-dds-adapter: {e}") | |
| return False | |
| except Exception as e: | |
| # Unexpected error while trying to kill the adapter | |
| log.w(f"kill_all_dds_adapters: unexpected error when attempting to kill rs-dds-adapter: {e}") | |
| return False |
There was a problem hiding this comment.
This is intentionally best-effort. taskkill/pkill return non-zero when there are no matching processes, which is the normal/expected case - logging a warning every time would be noisy and misleading. The caller cannot do anything actionable if this fails anyway; the next step (starting the adapter) will surface the real problem if ports are still held.
| adapter_process = None | ||
|
|
||
| with test.closure( 'Kill leftover rs-dds-adapter processes' ): | ||
| kill_all_dds_adapters() | ||
| time.sleep( 0.5 ) # let the OS release resources |
There was a problem hiding this comment.
The fixed 0.5s sleep is a magic delay and may be either longer than needed (slowing runs) or insufficient on slower/loaded systems. Prefer waiting until no rs-dds-adapter processes remain (up to a timeout) instead of a hard-coded sleep.
| adapter_process = None | |
| with test.closure( 'Kill leftover rs-dds-adapter processes' ): | |
| kill_all_dds_adapters() | |
| time.sleep( 0.5 ) # let the OS release resources | |
| def _dds_adapter_processes_exist(): | |
| """ | |
| Check if any rs-dds-adapter processes are still running. | |
| Returns True if at least one process is found, False otherwise. | |
| """ | |
| try: | |
| if platform.system() == 'Windows': | |
| # Use tasklist to check for the adapter process by image name | |
| result = subprocess.run( | |
| ['tasklist', '/FI', 'IMAGENAME eq rs-dds-adapter.exe'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.DEVNULL, | |
| text=True | |
| ) | |
| # When no tasks match the filter, tasklist prints "No tasks are running..." | |
| output = result.stdout or '' | |
| return 'rs-dds-adapter.exe' in output | |
| else: | |
| # pgrep returns 0 if one or more processes matched, 1 if none matched | |
| result = subprocess.run( | |
| ['pgrep', '-f', 'rs-dds-adapter'], | |
| stdout=subprocess.DEVNULL, | |
| stderr=subprocess.DEVNULL | |
| ) | |
| if result.returncode == 0: | |
| return True | |
| if result.returncode == 1: | |
| return False | |
| # Any other return code is treated conservatively as "maybe exists" | |
| return True | |
| except Exception: | |
| # On any error, fall back to conservative behavior | |
| return True | |
| def wait_for_no_dds_adapters(timeout=5.0, poll_interval=0.1): | |
| """ | |
| Wait until no rs-dds-adapter processes remain, or until timeout (in seconds) | |
| is reached. Polls at the given poll_interval. | |
| """ | |
| end_time = time.time() + timeout | |
| while time.time() < end_time: | |
| if not _dds_adapter_processes_exist(): | |
| return | |
| time.sleep( poll_interval ) | |
| adapter_process = None | |
| with test.closure( 'Kill leftover rs-dds-adapter processes' ): | |
| kill_all_dds_adapters() | |
| # Wait until the OS has actually cleaned up any rs-dds-adapter processes | |
| wait_for_no_dds_adapters() |
There was a problem hiding this comment.
The 0.5s sleep is a simple one-time pause at test startup to let the OS release sockets/ports after killing processes. Adding 50+ lines of polling infrastructure with tasklist/pgrep checks is over-engineering for this use case - the adapter startup and device discovery that follow have their own timeouts and will surface any real issues.
| try: | ||
| # Try graceful termination first | ||
| adapter_process.send_signal( signal.SIGTERM ) | ||
| adapter_process.wait( timeout=2 ) | ||
| log.d( 'rs-dds-adapter terminated gracefully' ) | ||
| except (subprocess.TimeoutExpired, Exception) as e: | ||
| # Force kill if graceful termination fails | ||
| log.e( f'Error terminating rs-dds-adapter: {e}' ) | ||
| adapter_process.kill() | ||
| adapter_process.wait( timeout=2 ) | ||
| adapter_process.wait( timeout=5 ) | ||
| log.d( 'rs-dds-adapter killed' ) | ||
| except Exception as e: | ||
| log.e( f'Error killing rs-dds-adapter: {e}' ) | ||
| # Safety net: kill any orphaned adapter processes by name | ||
| kill_all_dds_adapters() |
There was a problem hiding this comment.
In the cleanup path, exceptions from kill()/wait() are logged but do not fail the test, and kill_all_dds_adapters() is best-effort (exceptions are suppressed). If the adapter is still alive, the test may still report success while leaking a running process into later tests. Consider explicitly verifying the adapter is no longer running after the cleanup attempts and aborting/failing if it isn’t.
There was a problem hiding this comment.
The cleanup already does terminate + wait + kill + wait + kill_all_dds_adapters(). If all three layers fail, an additional verification step won't help either. The pre-test kill_all_dds_adapters() at the top of the file also acts as a guard for the next test run.
| log.e( f'Error terminating rs-dds-adapter: {e}' ) | ||
| adapter_process.kill() | ||
| adapter_process.wait( timeout=2 ) | ||
| adapter_process.wait( timeout=5 ) |
There was a problem hiding this comment.
We wanted the graceful termination before to release all system resources. I say keep the graceful, swallowing exceptions and finally run kill_all_dds_adapters anyway.
There was a problem hiding this comment.
Good point - restored graceful terminate() first (SIGTERM on Linux lets the adapter release DDS resources cleanly), falling back to kill() if that times out, with kill_all_dds_adapters() as safety net regardless.
There was a problem hiding this comment.
I added more protections by registering a signal handler for getting an external signal to abort from Jenkins, tell me what you think
| try: | ||
| # Try graceful termination first | ||
| adapter_process.send_signal( signal.SIGTERM ) | ||
| adapter_process.terminate() |
There was a problem hiding this comment.
terminate does not kill gracefully and may leak resources. Also the kill in the except feels redundant as we call kill_all_dds_adapter. You can do
try:
# Try graceful termination first
adapter_process.send_signal( signal.SIGTERM )
adapter_process.wait( timeout=2 )
log.d( 'rs-dds-adapter terminated gracefully' )
except (subprocess.TimeoutExpired, Exception) as e:
log.e( f'Error terminating rs-dds-adapter: {e}' )
finally:
# Force kill if graceful termination fails
kill_all_dds_adapters()
There was a problem hiding this comment.
Good call — restored SIGTERM for graceful shutdown (lets the adapter release DDS topics/participants on Linux), removed the redundant kill() in except, and moved kill_all_dds_adapters() to a finally block so it always runs. Done in 4dcfb2f.
Summary
rs-dds-adapterprocess leaking fromtest-dds-adapter-depthon Windows, causing USB devices to also appear as DDS for the rest of the nightly runSIGTERMcleanup withkill()+ system-wide process kill by name (cross-platform:taskkillon Windows,pkillon Linux)Root cause
Discovered via nightly build #110856 on rsdsk254:
test-dds-adapter-depthstartsrs-dds-adapterand cleans up withsend_signal(SIGTERM)— unreliable on Windowsignoring device ... with changed connection type: USB -> DDSmessages appeared across 70+ non-DDS tests for the remainder of the nightlytest-dds-adapter-depthcompleted — confirming the leaktest-live-hw-reset-stressto fail on both D455 and D435 (the only test failure in the run)Test plan
test-dds-adapter-depthpasses on Windows nightly with DDS contextignoring device ... USB -> DDSmessages appear in non-DDS tests after the adapter test runstest-live-hw-reset-stresspasses without DDS interference🤖 Generated with Claude Code