Docker : Conflict (container is already paused)
klieret opened this issue · comments
___________________________ test_execute_environment ___________________________
tmp_path = Path('/tmp/pytest-of-runner/pytest-0/test_execute_environment0')
test_env_args = EnvironmentArguments(data_path='/tmp/pytest-of-runner/pytest-0/swe-agent-test-repo/problem_statements/1.md', image_nam...er/pytest-0/test_execute_environment0/env_config.yml'), repo_path='/tmp/pytest-of-runner/pytest-0/swe-agent-test-repo')
@pytest.mark.slow()
def test_execute_environment(tmp_path, test_env_args):
test_env = {
"python": "3.6",
"packages": "pytest",
"pip_packages": ["tox"],
"install": "echo 'installing'",
}
env_config_path = Path(tmp_path / "env_config.yml")
env_config_path.write_text(yaml.dump(test_env))
test_env_args = dataclasses.replace(test_env_args, environment_setup=env_config_path)
with swe_env_context(test_env_args) as env:
> env.reset()
tests/test_env.py:114:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sweagent/environment/swe_env.py:340: in reset
self.install_env()
sweagent/environment/swe_env.py:881: in install_env
self.communicate_with_handling(f"conda activate {env_name}", error_msg="Failed to activate conda environment")
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <sweagent.environment.swe_env.SWEEnv object at 0x7ff8be667af0>
input = 'conda activate __tmp__pytest-of-runner__pytest-0__swe-agent-test-repo__a3d7505'
error_msg = 'Failed to activate conda environment', timeout_duration = 25
/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/docker/utils/decorators.py:19: in wrapped
return f(self, resource_id, *args, **kwargs)
/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/docker/api/container.py:917: in pause
self._raise_for_status(res)
/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/docker/api/client.py:277: in _raise_for_status
raise create_api_error_from_http_exception(e) from e
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
e = HTTPError('409 Client Error: Conflict for url: http+docker://localhost/v1.43/containers/32f20f508076c37d70cdfeb9fb71cd6630671f77b3a9d15394eb99bbeb5415a7/pause')
def create_api_error_from_http_exception(e):
"""
Create a suitable APIError from requests.exceptions.HTTPError.
"""
response = e.response
try:
explanation = response.json()['message']
except ValueError:
explanation = (response.text or '').strip()
cls = APIError
if response.status_code == 404:
explanation_msg = (explanation or '').lower()
if any(fragment in explanation_msg
for fragment in _image_not_found_explanation_fragments):
cls = ImageNotFound
else:
cls = NotFound
> raise cls(e, response=response, explanation=explanation) from e
E docker.errors.APIError: 409 Client Error for http+docker://localhost/v1.43/containers/32f20f508076c37d70cdfeb9fb71cd6630671f77b3a9d15394eb99bbeb5415a7/pause: Conflict ("Container 32f20f508076c37d70cdfeb9fb71cd6630671f77b3a9d15394eb99bbeb5415a7 is already paused")
/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/docker/errors.py:39: APIError
Could be a race condition with
if self.persistent:
if self.container_obj.status not in {"paused", "exited"}:
self.container_obj.pause()
self.logger.info("Agent container paused")
else:
self.logger.info(f"Agent container status: {self.container_obj.status}")Seems to persist and always be tied to cleanup after an error:
=================================== FAILURES ===================================
___________________ test_execute_environment_clone_python_uv ___________________
tmp_path = Path('/tmp/pytest-of-runner/pytest-0/test_execute_environment_clone0')
test_env_args = EnvironmentArguments(data_path='/tmp/pytest-of-runner/pytest-0/swe-agent-test-repo/problem_statements/1.md', image_nam...est-0/test_execute_environment_clone0/env_config.yml'), repo_path='/tmp/pytest-of-runner/pytest-0/swe-agent-test-repo')
@pytest.mark.slow()
def test_execute_environment_clone_python_uv(tmp_path, test_env_args):
"""This should clone the existing python 3.10 conda environment for speedup
and also use uv pip.
"""
test_env = {
"python": "3.10",
"packages": "pytest",
"pip_packages": ["tox"],
"install": "uv pip install -e .",
}
env_config_path = Path(tmp_path / "env_config.yml")
env_config_path.write_text(yaml.dump(test_env))
test_env_args = dataclasses.replace(test_env_args, environment_setup=env_config_path)
with swe_env_context(test_env_args) as env:
> env.reset()
tests/test_env.py:140:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sweagent/environment/swe_env.py:371: in reset
self.install_env()
sweagent/environment/swe_env.py:979: in install_env
self.communicate_with_handling(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <sweagent.environment.swe_env.SWEEnv object at 0x7f50a6e08a60>
input = 'uv pip install -e .'
error_msg = 'Install command failed to execute successfully'
timeout_duration = 500
def communicate_with_handling(self, input: str, error_msg: str, timeout_duration=25) -> str:
"""
Wrapper for communicate function that raises error if return code is non-zero
Args:
input: input to send to container
error_msg: error message to raise if return code is non-zero
timeout_duration: duration to wait for output
Returns:
output: output from container
"""
logs = self.communicate(input, timeout_duration=timeout_duration)
if self.returncode != 0:
self.logger.error(f"{error_msg}: {logs}")
self.close()
msg = f"{error_msg}: {logs}"
> raise RuntimeError(msg)
E RuntimeError: Install command failed to execute successfully: error: No Python interpreters found in virtual environments
sweagent/environment/swe_env.py:739: RuntimeError
----------------------------- Captured stdout call -----------------------------
INFO 💽 Loaded dataset from
/tmp/pytest-of-runner/pytest-0/swe-agent-test-repo/problem_statements/1
.md
INFO Found image sweagent/swe-agent:latest with tags:
['sweagent/swe-agent:latest'], created: 2024-06-04T00:18:04.167366673Z
for linux amd64.
DEBUG Starting container with command: docker exec -i
test-container-this-is-a-random-string /bin/bash -l
INFO 🌱 Environment Initialized
DEBUG Environment initialization took 1.43 seconds
INFO Installing __tmp__pytest-of-runner__pytest-0__swe-agent-test-repo at
base commit...
ERROR Install command failed to execute successfully: error: No Python
interpreters found in virtual environments
INFO Beginning environment shutdown...
INFO Agent container paused
INFO Beginning environment shutdown...
WARNING Failed to pause container.
Traceback (most recent call last):
File "/home/runner/work/SWE-agent/SWE-agent/tests/test_env.py", line
48, in swe_env_context
yield env
File "/home/runner/work/SWE-agent/SWE-agent/tests/test_env.py", line
140, in test_execute_environment_clone_python_uv
env.reset()
File
"/home/runner/work/SWE-agent/SWE-agent/sweagent/environment/swe_env.py"
, line 371, in reset
self.install_env()
File
"/home/runner/work/SWE-agent/SWE-agent/sweagent/environment/swe_env.py"
, line 979, in install_env
self.communicate_with_handling(
File
"/home/runner/work/SWE-agent/SWE-agent/sweagent/environment/swe_env.py"
, line 739, in communicate_with_handling
raise RuntimeError(msg)
RuntimeError: Install command failed to execute successfully: error: No
Python interpreters found in virtual environments
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/api/client.py", line 275, in _raise_for_status
response.raise_for_status()
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/req
uests/models.py", line 1024, in raise_for_status
raise HTTPError(http_error_msg, response=self)
requests.exceptions.HTTPError: 409 Client Error: Conflict for url:
http+docker://localhost/v1.43/containers/14f3254c509a38706f263bf5e713ff
87dbf22805fe8466159b7626241781e21d/pause
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File
"/home/runner/work/SWE-agent/SWE-agent/sweagent/environment/swe_env.py"
, line 506, in close
self.container_obj.pause()
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/models/containers.py", line 332, in pause
return self.client.api.pause(self.id)
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/utils/decorators.py", line 19, in wrapped
return f(self, resource_id, *args, **kwargs)
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/api/container.py", line 917, in pause
self._raise_for_status(res)
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/api/client.py", line 277, in _raise_for_status
raise create_api_error_from_http_exception(e) from e
File
"/opt/hostedtoolcache/Python/3.9.19/x64/lib/python3.9/site-packages/doc
ker/errors.py", line 39, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation) from e
docker.errors.APIError: 409 Client Error for
http+docker://localhost/v1.43/containers/14f3254c509a38706f263bf5e713ff
87dbf22805fe8466159b7626241781e21d/pause: Conflict ("Container
14f3254c509a38706f263bf5e713ff87dbf22805fe8466159b7626241781e21d is
already paused")
The reason that
if self.container_obj.status not in {"paused", "exited"}:
doesn't work is that self.container_obj.status apparently doesn't get updated after the initialization:
x = _get_persistent_container("coa1", "sweagent/swe-agent:latest")
client = docker.from_env()
co = client.containers.get("coa1")
co.status
# running
co.pause()
# let's wait a bit
co.status
# still running