Last active
March 14, 2023 22:51
-
-
Save awesomebytes/a1e8b613c6c53fd2ff39bf6b130cbc5e to your computer and use it in GitHub Desktop.
docker exec -t forwarding signals wrapper
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python3 | |
import sys | |
import argparse | |
import os | |
import subprocess | |
import signal | |
import time | |
import threading | |
import shlex | |
import docker | |
""" | |
This program is meant to substitute running 'docker exec <FLAGS> CONTAINER_NAME COMMAND' | |
to overcome the limitation of docker exec not forwarding signals to the | |
executed process. | |
This was reported here in Nov 2014: https://github.com/moby/moby/issues/9098#issuecomment-312152980) | |
Furthermore, here: https://github.com/docker/cli/pull/1841 they say | |
moby/moby#9098 Kill docker exec command will not terminate the spawned process | |
This patch does not fix the docker exec case; it looks like there's no API to kill an exec'd process, | |
so there's no signal-proxy for this yet | |
Note: -i is not supported, couldn't make it work but got as close as I could (maybe it can't work?). | |
Author: Sammy Pfeiffer <sam.pfeiffer at hullbot.com> | |
""" | |
class DockerExecWithSignalForwarding(object): | |
def __init__(self, container_name, command, | |
# Offer the rest of the options from docker exec | |
detach=False, | |
# Note detach-keys is not implemented | |
environment=None, | |
# Not supported | |
interactive=False, | |
privileged=False, | |
tty=False, | |
# Leaving these as None makes the inner docker API deal with them correctly | |
user=None, | |
workdir=None, | |
# By default the timeout of Python's docker exec is 60s, change it to 1 year-ish | |
socket_timeout=60 * 60 * 24 * 365): | |
""" | |
Provided a set of flags (same ones of docker exec), a container name and a command, | |
do 'docker exec' but managing signals to be forwarded to the exec-ed process. | |
""" | |
if interactive: | |
raise RuntimeError("Interactive mode not supported, use docker exec.") | |
# We inherit the rest of the configuration (including DOCKER_HOST) from the environment | |
self.client = docker.from_env(timeout=socket_timeout) | |
# Sanity check on the command, should be a string which we split with shlex or already a list/tuple | |
if isinstance(command, str): | |
command = shlex.split(command) | |
if not (isinstance(command, list) or isinstance(command, tuple)): | |
raise TypeError("Command is of type {} and it must be str/list/tuple. (command: {})".format( | |
type(command), | |
command)) | |
# Translate docker exec style arguments into exec_run arguments | |
try: | |
# Get a reference to the container | |
self.container = self.client.containers.get(container_name) | |
# Get the Id of the 'docker exec' instance (that is not yet being executed) so we can start it | |
exec_create_response = self.client.api.exec_create(self.container.id, | |
command, | |
stdout=True, | |
stderr=True, | |
stdin=self.interactive, | |
tty=tty, | |
privileged=privileged, | |
user=user, | |
environment=environment, | |
workdir=workdir) | |
self.exec_id = exec_create_response['Id'] | |
# The following block of code is to manage the situation of an interactive session | |
# We would like to support it but the underlying API doesn't allow for it (writing into the socket | |
# simply does not work as far as I could test) it was a lot of work to figure out the bits | |
# to get this to this state, so I'm leaving it here | |
# if interactive: | |
# # Because we want to support stdin we need to access the lower-level socket | |
# # instead of being able to use exec_start with stream=True | |
# self.exec_socket = self.client.api.exec_start(self.exec_id, | |
# detach=detach, | |
# tty=tty, | |
# stream=False, | |
# socket=True, | |
# demux=True) | |
# # Recreate the function that offers the generator for output usually when using stream=True | |
# def _read_from_socket(socket, stream, tty=True, demux=False): | |
# """ | |
# Adapted from docker/client.py in order to enable stdin... tricky. | |
# """ | |
# gen = docker.api.client.frames_iter(socket, tty) | |
# if demux: | |
# # The generator will output tuples (stdout, stderr) | |
# gen = (docker.api.client.demux_adaptor(*frame) for frame in gen) | |
# else: | |
# # The generator will output strings | |
# gen = (data for (_, data) in gen) | |
# if stream: | |
# return gen | |
# else: | |
# # Wait for all the frames, concatenate them, and return the result | |
# return docker.api.client.consume_socket_output(gen, demux=demux) | |
# self.exec_output = _read_from_socket(self.exec_socket, True, tty, True) | |
# else: | |
self.exec_output = self.client.api.exec_start(self.exec_id, | |
detach=detach, | |
tty=tty, | |
stream=True, | |
socket=False, | |
demux=True) | |
self.setup_signal_forwarding() | |
self.program_running = True | |
# Imitate the behaviour of the original docker exec up to a point | |
except docker.errors.NotFound as e: | |
print("Error: No such container: {}".format(container_name)) | |
os._exit(1) | |
# Start a thread that monitors if the program died so we can end this when this happens | |
self.monitor_thread = threading.Thread(target=self.monitor_exec) | |
self.monitor_thread.start() | |
self.output_manager_thread = None | |
if self.interactive: | |
# Deal with stdout and stderr in a thread and let the main thread deal with input | |
self.output_manager_thread = threading.Thread(target=self.manage_stdout_and_stderr) | |
self.output_manager_thread.start() | |
self.manage_stdin() | |
else: | |
self.manage_stdout_and_stderr() | |
def monitor_exec(self): | |
""" | |
We loop (very slowly) to check if the underlaying command died, this is useful for | |
commands executed in a remote docker daemon. It 'should' not happen locally, but it may. | |
""" | |
try: | |
# Check if the process is dead, the 'Running' key must become false | |
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id) | |
while exec_inspect_dict.get('Running'): | |
# Generous sleep, as this is to catch the program dieing by something else than this wrapper | |
time.sleep(10.0) | |
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id) | |
# If it's dead, we should exit with its exit code | |
os._exit(exec_inspect_dict.get('ExitCode')) | |
except docker.errors.APIError as e: | |
# API error, we can't access anymore, exit | |
raise RuntimeError("Docker API error when monitoring exec process ({})".format(e)) | |
def forward_signal(self, signal_number, frame): | |
""" | |
Forward the signal signal_number to the container, | |
we first need to find what's the in-container PID of the process we docker exec-ed | |
then we docker exec a kill signal with it. | |
""" | |
# print("Forwarding signal {}".format(signal_number)) | |
# Using a lock to attempt to deal with Control+C spam | |
with self.signal_lock: | |
pid_in_container = self.get_container_pid() | |
kill_command = ["kill", "-{}".format(signal_number), str(pid_in_container)] | |
try: | |
exit_code, output = self.container.exec_run(kill_command, | |
# Do it always as root | |
user='root') | |
except docker.errors.NotFound as e: | |
raise RuntimeError("Container doesn't exist, can't forward signal {} (Exception: {})".format( | |
signal_number, e)) | |
if exit_code != 0: | |
raise RuntimeError( | |
'When forwarding signal {}, kill command to PID in container {} failed with exit code {}, output was: {}'.format( | |
signal_number, pid_in_container, exit_code, output)) | |
def get_container_pid(self): | |
""" | |
Return the in-container PID of the exec-ed process. | |
""" | |
try: | |
# I wish the stored PID of exec was the container PID (which is what I expected) | |
# but it's actually the host PID so in the following lines we deal with it | |
pid_in_host = self.client.api.exec_inspect(self.exec_id).get('Pid') | |
except docker.errors.NotFound as e: | |
raise RuntimeError("Container doesn't exist, can't get exec PID (Exception: {})".format(e)) | |
# We need to translate the host PID into the container PID, there is no general mapping for it in Docker | |
# If we are running in the same host, this is easier, we can get the Docker PID by just doing: | |
# cat /proc/PID/status | grep NSpid | awk '{print $3}' | |
# If the docker container is running in a different machine we need to execute that command in that machine | |
# which implies using SSH to execute the command | |
# Here we can only support DOCKER_HOST=ssh://user@host to use ssh to execute this command | |
# as if we are using ssh:// to access the docker daemon it's fair to assume we have SSH keys setup | |
# if docker host is tcp:// on another host or a socket file with SSH tunneling there isn't much we can do | |
docker_host = os.environ.get('DOCKER_HOST', None) | |
# If using SSH execute the command remotely | |
if docker_host and 'ssh://' in docker_host: | |
ssh_user_at_host = docker_host.replace('ssh://', '') | |
get_pid_in_container_cmd = "ssh -q -o StrictHostKeyChecking=no {} ".format(ssh_user_at_host) | |
get_pid_in_container_cmd += "cat /proc/{}/status | grep NSpid | awk '{{print $3}}'".format(pid_in_host) | |
# Otherwise, execute the command locally | |
else: | |
get_pid_in_container_cmd = "cat /proc/{}/status | grep NSpid | awk '{{print $3}}'".format(pid_in_host) | |
# Execute the command that gets the in-Docker PID | |
try: | |
pid_in_container = subprocess.check_output(get_pid_in_container_cmd, shell=True) | |
except subprocess.CalledProcessError as e: | |
raise RuntimeError( | |
"CalledProcessError exception while trying to get the in-docker PID of the process ({})".format(e)) | |
return int(pid_in_container) | |
def setup_signal_forwarding(self): | |
""" | |
Forward all signals to the docker exec-ed process. | |
If it dies, this process will die too as self.manage_stdout_and_stderr will finish | |
and forward the exit code. | |
""" | |
self.signal_lock = threading.Lock() | |
# Forward all signals, even though we are most interested just in SIGTERM and SIGINT | |
signal.signal(signal.SIGHUP, self.forward_signal) | |
signal.signal(signal.SIGINT, self.forward_signal) | |
signal.signal(signal.SIGQUIT, self.forward_signal) | |
signal.signal(signal.SIGILL, self.forward_signal) | |
signal.signal(signal.SIGTRAP, self.forward_signal) | |
signal.signal(signal.SIGABRT, self.forward_signal) | |
signal.signal(signal.SIGBUS, self.forward_signal) | |
signal.signal(signal.SIGFPE, self.forward_signal) | |
# Can't be captured, but for clarity leaving it here | |
# signal.signal(signal.SIGKILL, self.forward_signal) | |
signal.signal(signal.SIGUSR1, self.forward_signal) | |
signal.signal(signal.SIGUSR2, self.forward_signal) | |
signal.signal(signal.SIGSEGV, self.forward_signal) | |
signal.signal(signal.SIGPIPE, self.forward_signal) | |
signal.signal(signal.SIGALRM, self.forward_signal) | |
signal.signal(signal.SIGTERM, self.forward_signal) | |
def manage_stdout_and_stderr(self): | |
""" | |
Print stdout and stderr as the generator provides it. | |
When the generator finishes we exit the program forwarding the exit code. | |
""" | |
# Note that if the application prints a lot, this will use some CPU | |
# but there is no way around it as we are forced to read from the socket and decode to print | |
for stdout, stderr in self.exec_output: | |
# Note that if choosing tty=True output is always in stdout | |
if stdout: | |
print(stdout.decode("utf-8"), file=sys.stdout, end='') | |
if stderr: | |
print(stderr.decode("utf-8"), file=sys.stderr, end='') | |
# When we come out of this loop, the program we exec-ed has terminated | |
# so we can exit with its exit code just here | |
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id) | |
exit_code = exec_inspect_dict.get('ExitCode') | |
os._exit(exit_code) | |
def manage_stdin(self): | |
""" | |
Forward the input of this program to the docker exec-ed program. | |
""" | |
raise NotImplemented("Managing stdin is not implemented.") | |
# print(dir(self.exec_socket)) | |
# print(self.exec_socket.readable()) | |
# print(self.exec_socket.writable()) | |
# print(dir(self.exec_socket._sock)) | |
# self.exec_socket._writing = True | |
# print(self.exec_socket.writable()) | |
# def write(sock, str): | |
# while len(str) > 0: | |
# written = sock.write(str) | |
# str = str[written:] | |
# while True: | |
# # self.exec_socket._sock.sendall(input().encode('utf-8')) | |
# # self.exec_socket.flush() | |
# #print("sent") | |
# # Doesn't work either | |
# write(self.exec_socket, input().encode('utf-8')) | |
# print("--written--") | |
# #os.write(self.exec_socket._sock.fileno(), input().encode('utf-8')) | |
# #print("sent") | |
# #print("Received: {}".format(self.exec_socket._sock.recv(1))) | |
# # try: | |
# # print(os.read(self.exec_socket._sock.fileno(), 4096)) | |
# # except BlockingIOError as b: | |
# # print("BlockingIOError: {} ".format(b)) | |
# # print(self.client.api.exec_inspect(self.exec_id)) | |
def __del__(self): | |
""" | |
When the program ends this gets called so we can cleanup resources | |
and exit with the exit code from the exec-ed command. | |
Note it is unlikely this gets ever called. | |
""" | |
# print("Calling __del__") | |
# Wait for the output thread in case there are more prints to show | |
if self.output_manager_thread: | |
self.output_manager_thread.join() | |
# Try to wait for the process to be dead in case it isn't yet | |
try: | |
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id) | |
while exec_inspect_dict.get('Running'): | |
time.sleep(0.1) | |
exec_inspect_dict = self.client.api.exec_inspect(self.exec_id) | |
except docker.errors.APIError as e: | |
# We may get an API error here, if so, return an exit code other than 0 | |
os._exit(127) | |
pass | |
# Forward the exit code of the exec-ed command if we got here | |
exit_code = exec_inspect_dict.get('ExitCode') | |
os._exit(exit_code) | |
if __name__ == '__main__': | |
# Original docker exec --help | |
""" | |
Usage: docker exec [OPTIONS] CONTAINER COMMAND [ARG...] | |
Run a command in a running container | |
Options: | |
-d, --detach Detached mode: run command in the background | |
--detach-keys string Override the key sequence for detaching a container | |
-e, --env list Set environment variables | |
-i, --interactive Keep STDIN open even if not attached | |
--privileged Give extended privileges to the command | |
-t, --tty Allocate a pseudo-TTY | |
-u, --user string Username or UID (format: <name|uid>[:<group|gid>]) | |
-w, --workdir string Working directory inside the container | |
""" | |
parser = argparse.ArgumentParser(description="Run a command in a running container") | |
parser.add_argument("container", help="Container name") | |
parser.add_argument("command_and_args", help="Command and arguments", nargs=argparse.REMAINDER) | |
parser.add_argument("-d", "--detach", action='store_true', | |
help="Detached mode: run command in the background") | |
# We only support environment variables as a long string if there must be more than one | |
# I.e. -e USER=user for one or -e "USER=user SOMETHING_ELSE=1" | |
# Supporting multiple -e didn't work for me | |
parser.add_argument("-e", "--env", | |
type=str, help="Set environment variables (like 'VAR1=1 VAR2=2')") | |
# Interactive is not supported, but leaving it here just in case it is implemented in the future | |
parser.add_argument("-i", "--interactive", action='store_true', | |
help="Keep STDIN open even if not attached (Note: not implemented, use 'docker exec')") | |
parser.add_argument("--privileged", action='store_true', | |
help="Give extended privileges to the command") | |
parser.add_argument("-t", "--tty", action='store_true', | |
help="Allocate a pseudo-TTY") | |
parser.add_argument("-u", "--user", | |
type=str, help="Username or UID (format: <name|uid>[:<group|gid>])") | |
parser.add_argument("-w", "--workdir", | |
type=str, help="Working directory inside the container") | |
args = parser.parse_args() | |
if len(args.command_and_args) < 1: | |
print("dockerexec requires at least 2 arguments") | |
parser.print_help() | |
exit(1) | |
if args.interactive: | |
raise NotImplemented("Interactive mode not implemented, you should just use docker exec") | |
dewsf = DockerExecWithSignalForwarding(args.container, | |
args.command_and_args, | |
detach=args.detach, | |
# Note detach-keys is not implemented | |
environment=args.env, | |
interactive=args.interactive, | |
privileged=args.privileged, | |
tty=args.tty, | |
user=args.user, | |
workdir=args.workdir) | |
# The following lines are tests done with a container running: | |
# docker run --rm -t --name exec_signal_problem python:3 sleep 999 | |
# Proper testing should be implemented based on this | |
# # Forward error test | |
# de = DockerExec('exec_signal_problem', | |
# 'ls asdf', | |
# tty=True, | |
# interactive=False) | |
# # simple working test | |
# de = DockerExec('exec_signal_problem', | |
# 'ls', | |
# tty=True, | |
# interactive=False) | |
# Test signal forwarding SIGINT Control C | |
# de = DockerExec('exec_signal_problem', | |
# 'python -c "import sys;import signal;signal.signal(signal.SIGINT, print);print(\'hello\', file=sys.stderr);import time; time.sleep(600)"', | |
# tty=True, | |
# interactive=False) | |
# Test signal forwarding SIGTERM | |
# de = DockerExec('exec_signal_problem', | |
# 'python -c "import sys;import signal;signal.signal(signal.SIGTERM, print);print(\'hello\', file=sys.stderr);import time; time.sleep(600)"', | |
# tty=True, | |
# interactive=False) | |
# Test output in stderr | |
# de = DockerExec('exec_signal_problem', | |
# 'python -c "import sys; print(\'hello stderr\', file=sys.stderr);print(\'hello stdout\', file=sys.stdout)"', | |
# tty=False, | |
# interactive=False) | |
# test input, doesn't work, not supported (not needed anyways) | |
# de = DockerExec('exec_signal_problem', | |
# 'cat', | |
# tty=True, | |
# interactive=True) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment