[FIX] Try to #60. introduce retry mechanism when container removal failed

This commit is contained in:
Sylvain LE GAL 2024-10-08 09:47:58 +02:00
parent 5b49b2028f
commit 1d15b1f8d3

View File

@ -1,3 +1,5 @@
import time
import docker import docker
from loguru import logger from loguru import logger
@ -110,6 +112,28 @@ def exec_container(container, command):
def kill_container(container_name): def kill_container(container_name):
# In some situation, containers.list return
# containers with removal already in progress
# when we call container.remove(), it is raising an
# docker.errors.APIError
# "removal of container xx is already in progress".
# so, we retry a few seconds after
# and raise an exception after five failures.
for i in [1, 5, 10, 60, False]:
try:
_kill_container(container_name)
return
except docker.errors.APIError as e:
if not i:
logger.error(f"Fail to kill {container_name} after 5 retries")
raise e
logger.warning(
f"Fail to kill {container_name}. Retrying in {i} seconds"
)
time.sleep(i)
def _kill_container(container_name):
client = get_docker_client() client = get_docker_client()
try: try: