Lists not returned with __iter__ and not compatible with an async for loop
SelfhostedPro opened this issue · comments
Long story short
I'm attempting to gather stats on all of my running docker containers then sending them (via a websocket) back to my frontend. I can't get an async for loop going using docker.containers.list() as it returns a standard list without an aiter method.
- Expected behaviour: Returns a list that is iterable in an
async forloop - Actual behaviour: Returns a standard list that is blocking
How to reproduce (Ignore the fastapi/websocket stuff)
Running the following (without websocket/fastapi stuff) will only stream stats for a single container (if you remove async from the first for loop), not all of them at the same time.
@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
# auth_success = await websocket_auth(websocket=websocket)
# if auth_success:
await websocket.accept()
dclient = aiodocker.Docker()
async for app in await dclient.containers.list():
async with aiodocker.Docker() as docker:
cpu_total = 0.0
cpu_system = 0.0
cpu_percent = 0.0
_name = app._container['Names'][0][1:]
container: DockerContainer = await docker.containers.get(_name)
stats = container.stats(stream=True)
async for line in stats:
mem_current = line["memory_stats"]["usage"]
mem_total = line["memory_stats"]["limit"]
try:
cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
except KeyError as e:
print("error while getting new CPU stats: %r, falling back")
cpu_percent = await calculate_cpu_percent(line)
full_stats = {
"name": _name,
"time": line['read'],
"cpu_percent": cpu_percent,
"mem_current": mem_current,
"mem_total": line["memory_stats"]["limit"],
"mem_percent": (mem_current / mem_total) * 100.0,
}
await websocket.send_text(json.dumps(full_stats))Your environment
Running the latest version of Manjaro, Python 3.8.3, FastAPI. Not sure if anything in my env is going to be useful but if you need anything specifically feel free to ask.
Tried changing things around some but still am not having any luck:
@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
# auth_success = await websocket_auth(websocket=websocket)
# if auth_success:
await websocket.accept()
tasks = []
async with aiodocker.Docker() as docker:
containers = await docker.containers.list()
for app in containers:
_name = app._container['Names'][0][1:]
container: DockerContainer = await docker.containers.get(_name)
stats = container.stats(stream=True)
tasks.append(process_container(_name, stats))
await asyncio.gather(*tasks)
async def process_container(name, stats):
cpu_total = 0.0
cpu_system = 0.0
cpu_percent = 0.0
async for line in stats:
mem_current = line["memory_stats"]["usage"]
mem_total = line["memory_stats"]["limit"]
try:
cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
except KeyError as e:
print("error while getting new CPU stats: %r, falling back")
cpu_percent = await calculate_cpu_percent(line)
full_stats = {
"name": _name,
"time": line['read'],
"cpu_percent": cpu_percent,
"mem_current": mem_current,
"mem_total": line["memory_stats"]["limit"],
"mem_percent": (mem_current / mem_total) * 100.0,
}
await websocket.send_text(json.dumps(full_stats))ERROR: Exception in ASGI application
Traceback (most recent call last):
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/uvicorn/protocols/websockets/websockets_impl.py", line 154, in run_asgi
result = await self.app(self.scope, self.asgi_receive, self.asgi_send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/uvicorn/middleware/proxy_headers.py", line 45, in __call__
return await self.app(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/fastapi/applications.py", line 180, in __call__
await super().__call__(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/applications.py", line 111, in __call__
await self.middleware_stack(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/middleware/errors.py", line 146, in __call__
await self.app(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/exceptions.py", line 58, in __call__
await self.app(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 566, in __call__
await route.handle(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 283, in handle
await self.app(scope, receive, send)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/starlette/routing.py", line 57, in app
await func(session)
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/fastapi/routing.py", line 242, in app
await dependant.call(**values)
File "./backend/api/routers/apps.py", line 133, in dashboard
await asyncio.gather(*tasks)
File "./backend/api/routers/apps.py", line 139, in process_container
async for line in stats:
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/containers.py", line 299, in _stats_stream
async with cm as response:
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/utils.py", line 309, in __aenter__
resp = await self._coro
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiodocker/docker.py", line 241, in _do_query
response = await self.session.request(
File "/home/user/dev/Yacht-work/Yacht/backend/venv/lib/python3.8/site-packages/aiohttp/client.py", line 357, in _request
raise RuntimeError('Session is closed')
RuntimeError: Session is closed
This fixed it (indentation error)
@router.websocket("/stats")
async def dashboard(websocket: WebSocket):
# auth_success = await websocket_auth(websocket=websocket)
# if auth_success:
await websocket.accept()
tasks = []
async with aiodocker.Docker() as docker:
containers = await docker.containers.list()
for app in containers:
_name = app._container['Names'][0][1:]
container: DockerContainer = await docker.containers.get(_name)
stats = container.stats(stream=True)
tasks.append(process_container(_name, stats, websocket))
await asyncio.gather(*tasks)
async def process_container(name, stats, websocket):
cpu_total = 0.0
cpu_system = 0.0
cpu_percent = 0.0
async for line in stats:
mem_current = line["memory_stats"]["usage"]
mem_total = line["memory_stats"]["limit"]
try:
cpu_percent, cpu_system, cpu_total = await calculate_cpu_percent2(line, cpu_total, cpu_system)
except KeyError as e:
print("error while getting new CPU stats: %r, falling back")
cpu_percent = await calculate_cpu_percent(line)
full_stats = {
"name": name,
"time": line['read'],
"cpu_percent": cpu_percent,
"mem_current": mem_current,
"mem_total": line["memory_stats"]["limit"],
"mem_percent": (mem_current / mem_total) * 100.0,
}
await websocket.send_text(json.dumps(full_stats))