[FIX] Try to #60. introduce retry mechanism when container removal failed
This commit is contained in:
parent
5b49b2028f
commit
1d15b1f8d3
|
|
@ -1,3 +1,5 @@
|
|||
import time
|
||||
|
||||
import docker
|
||||
from loguru import logger
|
||||
|
||||
|
|
@ -110,6 +112,28 @@ def exec_container(container, command):
|
|||
|
||||
|
||||
def kill_container(container_name):
|
||||
# In some situation, containers.list return
|
||||
# containers with removal already in progress
|
||||
# when we call container.remove(), it is raising an
|
||||
# docker.errors.APIError
|
||||
# "removal of container xx is already in progress".
|
||||
# so, we retry a few seconds after
|
||||
# and raise an exception after five failures.
|
||||
for i in [1, 5, 10, 60, False]:
|
||||
try:
|
||||
_kill_container(container_name)
|
||||
return
|
||||
except docker.errors.APIError as e:
|
||||
if not i:
|
||||
logger.error(f"Fail to kill {container_name} after 5 retries")
|
||||
raise e
|
||||
logger.warning(
|
||||
f"Fail to kill {container_name}. Retrying in {i} seconds"
|
||||
)
|
||||
time.sleep(i)
|
||||
|
||||
|
||||
def _kill_container(container_name):
|
||||
client = get_docker_client()
|
||||
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user