Skip to content

Instantly share code, notes, and snippets.

@awesomebytes
Last active March 14, 2023 22:51
Show Gist options
  • Save awesomebytes/a1e8b613c6c53fd2ff39bf6b130cbc5e to your computer and use it in GitHub Desktop.
Save awesomebytes/a1e8b613c6c53fd2ff39bf6b130cbc5e to your computer and use it in GitHub Desktop.
docker exec -t forwarding signals wrapper
#!/usr/bin/env python3
import sys
import argparse
import os
import subprocess
import signal
import time
import threading
import shlex
import docker
"""
This program is meant to substitute running 'docker exec <FLAGS> CONTAINER_NAME COMMAND'
to overcome the limitation of docker exec not forwarding signals to the
executed process.
This was reported here in Nov 2014: https://github.com/moby/moby/issues/9098#issuecomment-312152980)
Furthermore, here: https://github.com/docker/cli/pull/1841 they say
moby/moby#9098 Kill docker exec command will not terminate the spawned process
This patch does not fix the docker exec case; it looks like there's no API to kill an exec'd process,
so there's no signal-proxy for this yet
Note: -i is not supported, couldn't make it work but got as close as I could (maybe it can't work?).
Author: Sammy Pfeiffer <sam.pfeiffer at hullbot.com>
"""
class DockerExecWithSignalForwarding(object):
def __init__(self, container_name, command,
# Offer the rest of the options from docker exec
detach=False,
# Note detach-keys is not implemented
environment=None,
# Not supported
interactive=False,
privileged=False,
tty=False,
# Leaving these as None makes the inner docker API deal with them correctly
user=None,
workdir=None,
# By default the timeout of Python's docker exec is 60s, change it to 1 year-ish
socket_timeout=60 * 60 * 24 * 365):
"""
Provided a set of flags (same ones of docker exec), a container name and a command,
do 'docker exec' but managing signals to be forwarded to the exec-ed process.
"""
if interactive:
raise RuntimeError("Interactive mode not supported, use docker exec.")
# We inherit the rest of the configuration (including DOCKER_HOST) from the environment
self.client = docker.from_env(timeout=socket_timeout)
# Sanity check on the command, should be a string which we split with shlex or already a list/tuple
if isinstance(command, str):
command = shlex.split(command)
if not (isinstance(command, list) or isinstance(command, tuple)):
raise TypeError("Command is of type {} and it must be str/list/tuple. (command: {})".format(
type(command),
command))
# Translate docker exec style arguments into exec_run arguments
try:
# Get a reference to the container
self.container = self.client.containers.get(container_name)
# Get the Id of the 'docker exec' instance (that is not yet being executed) so we can start it
exec_create_response = self.client.api.exec_create(self.container.id,
command,
stdout=True,
stderr=True,
stdin=self.interactive,
tty=tty,
privileged=privileged,
user=user,
environment=environment,
workdir=workdir)
self.exec_id = exec_create_response['Id']
# The following block of code is to manage the situation of an interactive session
# We would like to support it but the underlying API doesn't allow for it (writing into the socket
# simply does not work as far as I could test) it was a lot of work to figure out the bits
# to get this to this state, so I'm leaving it here
# if interactive:
# # Because we want to support stdin we need to access the lower-level socket
# # instead of being able to use exec_start with stream=True
# self.exec_socket = self.client.api.exec_start(self.exec_id,
# detach=detach,
# tty=tty,
# stream=False,
# socket=True,
# demux=True)
# # Recreate the function that offers the generator for output usually when using stream=True
# def _read_from_socket(socket, stream, tty=True, demux=False):
# """
# Adapted from docker/client.py in order to enable stdin... tricky.
# """
# gen = docker.api.client.frames_iter(socket, tty)
# if demux:
# # The generator will output tuples (stdout, stderr)
# gen = (docker.api.client.demux_adaptor(*frame) for frame in gen)
# else:
# # The generator will output strings
# gen = (data for (_, data) in gen)
# if stream:
# return gen
# else:
# # Wait for all the frames, concatenate them, and return the result
# return docker.api.client.consume_socket_output(gen, demux=demux)
# self.exec_output = _read_from_socket(self.exec_socket, True, tty, True)
# else:
self.exec_output = self.client.api.exec_start(self.exec_id,
detach=detach,
tty=tty,
stream=True,
socket=False,
demux=True)
self.setup_signal_forwarding()
self.program_running = True
# Imitate the behaviour of the original docker exec up to a point
except docker.errors.NotFound as e:
print("Error: No such container: {}".format(container_name))
os._exit(1)
# Start a thread that monitors if the program died so we can end this when this happens
self.monitor_thread = threading.Thread(target=self.monitor_exec)
self.monitor_thread.start()
self.output_manager_thread = None
if self.interactive:
# Deal with stdout and stderr in a thread and let the main thread deal with input
self.output_manager_thread = threading.Thread(target=self.manage_stdout_and_stderr)
self.output_manager_thread.start()
self.manage_stdin()
else:
self.manage_stdout_and_stderr()
def monitor_exec(self):
"""
We loop (very slowly) to check if the underlaying command died, this is useful for
commands executed in a remote docker daemon. It 'should' not happen locally, but it may.
"""
try:
# Check if the process is dead, the 'Running' key must become false
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id)
while exec_inspect_dict.get('Running'):
# Generous sleep, as this is to catch the program dieing by something else than this wrapper
time.sleep(10.0)
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id)
# If it's dead, we should exit with its exit code
os._exit(exec_inspect_dict.get('ExitCode'))
except docker.errors.APIError as e:
# API error, we can't access anymore, exit
raise RuntimeError("Docker API error when monitoring exec process ({})".format(e))
def forward_signal(self, signal_number, frame):
"""
Forward the signal signal_number to the container,
we first need to find what's the in-container PID of the process we docker exec-ed
then we docker exec a kill signal with it.
"""
# print("Forwarding signal {}".format(signal_number))
# Using a lock to attempt to deal with Control+C spam
with self.signal_lock:
pid_in_container = self.get_container_pid()
kill_command = ["kill", "-{}".format(signal_number), str(pid_in_container)]
try:
exit_code, output = self.container.exec_run(kill_command,
# Do it always as root
user='root')
except docker.errors.NotFound as e:
raise RuntimeError("Container doesn't exist, can't forward signal {} (Exception: {})".format(
signal_number, e))
if exit_code != 0:
raise RuntimeError(
'When forwarding signal {}, kill command to PID in container {} failed with exit code {}, output was: {}'.format(
signal_number, pid_in_container, exit_code, output))
def get_container_pid(self):
"""
Return the in-container PID of the exec-ed process.
"""
try:
# I wish the stored PID of exec was the container PID (which is what I expected)
# but it's actually the host PID so in the following lines we deal with it
pid_in_host = self.client.api.exec_inspect(self.exec_id).get('Pid')
except docker.errors.NotFound as e:
raise RuntimeError("Container doesn't exist, can't get exec PID (Exception: {})".format(e))
# We need to translate the host PID into the container PID, there is no general mapping for it in Docker
# If we are running in the same host, this is easier, we can get the Docker PID by just doing:
# cat /proc/PID/status | grep NSpid | awk '{print $3}'
# If the docker container is running in a different machine we need to execute that command in that machine
# which implies using SSH to execute the command
# Here we can only support DOCKER_HOST=ssh://user@host to use ssh to execute this command
# as if we are using ssh:// to access the docker daemon it's fair to assume we have SSH keys setup
# if docker host is tcp:// on another host or a socket file with SSH tunneling there isn't much we can do
docker_host = os.environ.get('DOCKER_HOST', None)
# If using SSH execute the command remotely
if docker_host and 'ssh://' in docker_host:
ssh_user_at_host = docker_host.replace('ssh://', '')
get_pid_in_container_cmd = "ssh -q -o StrictHostKeyChecking=no {} ".format(ssh_user_at_host)
get_pid_in_container_cmd += "cat /proc/{}/status | grep NSpid | awk '{{print $3}}'".format(pid_in_host)
# Otherwise, execute the command locally
else:
get_pid_in_container_cmd = "cat /proc/{}/status | grep NSpid | awk '{{print $3}}'".format(pid_in_host)
# Execute the command that gets the in-Docker PID
try:
pid_in_container = subprocess.check_output(get_pid_in_container_cmd, shell=True)
except subprocess.CalledProcessError as e:
raise RuntimeError(
"CalledProcessError exception while trying to get the in-docker PID of the process ({})".format(e))
return int(pid_in_container)
def setup_signal_forwarding(self):
"""
Forward all signals to the docker exec-ed process.
If it dies, this process will die too as self.manage_stdout_and_stderr will finish
and forward the exit code.
"""
self.signal_lock = threading.Lock()
# Forward all signals, even though we are most interested just in SIGTERM and SIGINT
signal.signal(signal.SIGHUP, self.forward_signal)
signal.signal(signal.SIGINT, self.forward_signal)
signal.signal(signal.SIGQUIT, self.forward_signal)
signal.signal(signal.SIGILL, self.forward_signal)
signal.signal(signal.SIGTRAP, self.forward_signal)
signal.signal(signal.SIGABRT, self.forward_signal)
signal.signal(signal.SIGBUS, self.forward_signal)
signal.signal(signal.SIGFPE, self.forward_signal)
# Can't be captured, but for clarity leaving it here
# signal.signal(signal.SIGKILL, self.forward_signal)
signal.signal(signal.SIGUSR1, self.forward_signal)
signal.signal(signal.SIGUSR2, self.forward_signal)
signal.signal(signal.SIGSEGV, self.forward_signal)
signal.signal(signal.SIGPIPE, self.forward_signal)
signal.signal(signal.SIGALRM, self.forward_signal)
signal.signal(signal.SIGTERM, self.forward_signal)
def manage_stdout_and_stderr(self):
"""
Print stdout and stderr as the generator provides it.
When the generator finishes we exit the program forwarding the exit code.
"""
# Note that if the application prints a lot, this will use some CPU
# but there is no way around it as we are forced to read from the socket and decode to print
for stdout, stderr in self.exec_output:
# Note that if choosing tty=True output is always in stdout
if stdout:
print(stdout.decode("utf-8"), file=sys.stdout, end='')
if stderr:
print(stderr.decode("utf-8"), file=sys.stderr, end='')
# When we come out of this loop, the program we exec-ed has terminated
# so we can exit with its exit code just here
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id)
exit_code = exec_inspect_dict.get('ExitCode')
os._exit(exit_code)
def manage_stdin(self):
"""
Forward the input of this program to the docker exec-ed program.
"""
raise NotImplemented("Managing stdin is not implemented.")
# print(dir(self.exec_socket))
# print(self.exec_socket.readable())
# print(self.exec_socket.writable())
# print(dir(self.exec_socket._sock))
# self.exec_socket._writing = True
# print(self.exec_socket.writable())
# def write(sock, str):
# while len(str) > 0:
# written = sock.write(str)
# str = str[written:]
# while True:
# # self.exec_socket._sock.sendall(input().encode('utf-8'))
# # self.exec_socket.flush()
# #print("sent")
# # Doesn't work either
# write(self.exec_socket, input().encode('utf-8'))
# print("--written--")
# #os.write(self.exec_socket._sock.fileno(), input().encode('utf-8'))
# #print("sent")
# #print("Received: {}".format(self.exec_socket._sock.recv(1)))
# # try:
# # print(os.read(self.exec_socket._sock.fileno(), 4096))
# # except BlockingIOError as b:
# # print("BlockingIOError: {} ".format(b))
# # print(self.client.api.exec_inspect(self.exec_id))
def __del__(self):
"""
When the program ends this gets called so we can cleanup resources
and exit with the exit code from the exec-ed command.
Note it is unlikely this gets ever called.
"""
# print("Calling __del__")
# Wait for the output thread in case there are more prints to show
if self.output_manager_thread:
self.output_manager_thread.join()
# Try to wait for the process to be dead in case it isn't yet
try:
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id)
while exec_inspect_dict.get('Running'):
time.sleep(0.1)
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id)
except docker.errors.APIError as e:
# We may get an API error here, if so, return an exit code other than 0
os._exit(127)
pass
# Forward the exit code of the exec-ed command if we got here
exit_code = exec_inspect_dict.get('ExitCode')
os._exit(exit_code)
if __name__ == '__main__':
# Original docker exec --help
"""
Usage: docker exec [OPTIONS] CONTAINER COMMAND [ARG...]
Run a command in a running container
Options:
-d, --detach Detached mode: run command in the background
--detach-keys string Override the key sequence for detaching a container
-e, --env list Set environment variables
-i, --interactive Keep STDIN open even if not attached
--privileged Give extended privileges to the command
-t, --tty Allocate a pseudo-TTY
-u, --user string Username or UID (format: <name|uid>[:<group|gid>])
-w, --workdir string Working directory inside the container
"""
parser = argparse.ArgumentParser(description="Run a command in a running container")
parser.add_argument("container", help="Container name")
parser.add_argument("command_and_args", help="Command and arguments", nargs=argparse.REMAINDER)
parser.add_argument("-d", "--detach", action='store_true',
help="Detached mode: run command in the background")
# We only support environment variables as a long string if there must be more than one
# I.e. -e USER=user for one or -e "USER=user SOMETHING_ELSE=1"
# Supporting multiple -e didn't work for me
parser.add_argument("-e", "--env",
type=str, help="Set environment variables (like 'VAR1=1 VAR2=2')")
# Interactive is not supported, but leaving it here just in case it is implemented in the future
parser.add_argument("-i", "--interactive", action='store_true',
help="Keep STDIN open even if not attached (Note: not implemented, use 'docker exec')")
parser.add_argument("--privileged", action='store_true',
help="Give extended privileges to the command")
parser.add_argument("-t", "--tty", action='store_true',
help="Allocate a pseudo-TTY")
parser.add_argument("-u", "--user",
type=str, help="Username or UID (format: <name|uid>[:<group|gid>])")
parser.add_argument("-w", "--workdir",
type=str, help="Working directory inside the container")
args = parser.parse_args()
if len(args.command_and_args) < 1:
print("dockerexec requires at least 2 arguments")
parser.print_help()
exit(1)
if args.interactive:
raise NotImplemented("Interactive mode not implemented, you should just use docker exec")
dewsf = DockerExecWithSignalForwarding(args.container,
args.command_and_args,
detach=args.detach,
# Note detach-keys is not implemented
environment=args.env,
interactive=args.interactive,
privileged=args.privileged,
tty=args.tty,
user=args.user,
workdir=args.workdir)
# The following lines are tests done with a container running:
# docker run --rm -t --name exec_signal_problem python:3 sleep 999
# Proper testing should be implemented based on this
# # Forward error test
# de = DockerExec('exec_signal_problem',
# 'ls asdf',
# tty=True,
# interactive=False)
# # simple working test
# de = DockerExec('exec_signal_problem',
# 'ls',
# tty=True,
# interactive=False)
# Test signal forwarding SIGINT Control C
# de = DockerExec('exec_signal_problem',
# 'python -c "import sys;import signal;signal.signal(signal.SIGINT, print);print(\'hello\', file=sys.stderr);import time; time.sleep(600)"',
# tty=True,
# interactive=False)
# Test signal forwarding SIGTERM
# de = DockerExec('exec_signal_problem',
# 'python -c "import sys;import signal;signal.signal(signal.SIGTERM, print);print(\'hello\', file=sys.stderr);import time; time.sleep(600)"',
# tty=True,
# interactive=False)
# Test output in stderr
# de = DockerExec('exec_signal_problem',
# 'python -c "import sys; print(\'hello stderr\', file=sys.stderr);print(\'hello stdout\', file=sys.stdout)"',
# tty=False,
# interactive=False)
# test input, doesn't work, not supported (not needed anyways)
# de = DockerExec('exec_signal_problem',
# 'cat',
# tty=True,
# interactive=True)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment