[FIX] Try to #60. introduce retry mechanism when container removal failed
This commit is contained in:
parent
5b49b2028f
commit
1d15b1f8d3
|
|
@ -1,3 +1,5 @@
|
||||||
|
import time
|
||||||
|
|
||||||
import docker
|
import docker
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
|
|
@ -110,6 +112,28 @@ def exec_container(container, command):
|
||||||
|
|
||||||
|
|
||||||
def kill_container(container_name):
|
def kill_container(container_name):
|
||||||
|
# In some situation, containers.list return
|
||||||
|
# containers with removal already in progress
|
||||||
|
# when we call container.remove(), it is raising an
|
||||||
|
# docker.errors.APIError
|
||||||
|
# "removal of container xx is already in progress".
|
||||||
|
# so, we retry a few seconds after
|
||||||
|
# and raise an exception after five failures.
|
||||||
|
for i in [1, 5, 10, 60, False]:
|
||||||
|
try:
|
||||||
|
_kill_container(container_name)
|
||||||
|
return
|
||||||
|
except docker.errors.APIError as e:
|
||||||
|
if not i:
|
||||||
|
logger.error(f"Fail to kill {container_name} after 5 retries")
|
||||||
|
raise e
|
||||||
|
logger.warning(
|
||||||
|
f"Fail to kill {container_name}. Retrying in {i} seconds"
|
||||||
|
)
|
||||||
|
time.sleep(i)
|
||||||
|
|
||||||
|
|
||||||
|
def _kill_container(container_name):
|
||||||
client = get_docker_client()
|
client = get_docker_client()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user