devmapper command fails when setting up a specific instance type
odesenfans opened this issue · comments
Scheduling a specific instance on a CRN results in a 500. The instance is not started properly.
Instance:
{
"chain": "ETH",
"channel": "ALEPH-CLOUDSOLUTIONS",
"confirmations": [],
"confirmed": false,
"content": {
"address": "0xf379B2F8c29EeDc1fC7d189240d0aD7c9164aF64",
"allow_amend": false,
"environment": {
"aleph_api": true,
"internet": true,
"reproducible": false,
"shared_cache": false
},
"metadata": {
"name": "MyAwAZ"
},
"resources": {
"memory": 8000,
"seconds": 30,
"vcpus": 4
},
"rootfs": {
"parent": {
"ref": "77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27",
"use_latest": true
},
"persistence": "host",
"size_mib": 80000
},
"time": 1689843819.372,
"volumes": [
{
"mount": "/mount/opt",
"name": "MyVolume",
"persistence": "host",
"size_mib": 64000
}
]
},
"item_content": "{\"address\":\"0xf379B2F8c29EeDc1fC7d189240d0aD7c9164aF64\",\"time\":1689843819.372,\"metadata\":{\"name\":\"MyAwAZ\"},\"volumes\":[{\"persistence\":\"host\",\"mount\":\"/mount/opt\"
,\"size_mib\":64000,\"name\":\"MyVolume\"}],\"allow_amend\":false,\"resources\":{\"memory\":8000,\"vcpus\":4,\"seconds\":30},\"environment\":{\"reproducible\":false,\"internet\":true,\"aleph_api\":true,\
"shared_cache\":false},\"rootfs\":{\"parent\":{\"ref\":\"77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27\",\"use_latest\":true},\"persistence\":\"host\",\"size_mib\":80000}}",
"item_hash": "fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9",
"item_type": "inline",
"sender": "0xf379B2F8c29EeDc1fC7d189240d0aD7c9164aF64",
"signature": "0x6b5c8519b235b102ede3cf6c5e1aaea378204ff1d10a2c0f9628d2c45203605b77368e8b71be44ba3f041af6b235e56f5c50258cf869668aeeb0149cb76697f71c",
"size": 511,
"time": 1689843819.372,
"type": "INSTANCE"
}
Error:
143737406.418562 | DEBUG | File already exists: /var/cache/aleph/vm/runtime/77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27
143737406.525135 | DEBUG | command: chown jailman:jailman /var/cache/aleph/vm/runtime/77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27
143737408.357143 | DEBUG | File already exists, skipping ext4 creation on /var/lib/aleph/vm/volumes/persistent/fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9/MyVolume.ext4
143737409.245729 | DEBUG | command: losetup --find --show --read-only /var/cache/aleph/vm/runtime/77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27
143737410.745859 | DEBUG | command: blockdev --getsz /var/cache/aleph/vm/runtime/77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27
143737413.549662 | DEBUG | command: losetup --find --show /var/lib/aleph/vm/volumes/persistent/fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9/rootfs.btrfs
143737417.740822 | DEBUG | command: blockdev --getsz /var/lib/aleph/vm/volumes/persistent/fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9/rootfs.btrfs
143737420.428991 | DEBUG | command: dmsetup create fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9_rootfs
143737432.645321 | ERROR | Command failed with error code 1:
stdin = b'0 163840000 snapshot /dev/mapper/77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27 /dev/loop15 P 8'
command = ['dmsetup', 'create', 'fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9_rootfs']
stdout = b'device-mapper: reload ioctl on fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9_rootfs (253:3) failed: Invalid argument\nCommand failed.\n'
143737432.922125 | ERROR | Error handling request
Traceback (most recent call last):
File "/usr/lib/python3/dist-packages/aiohttp/web_protocol.py", line 433, in _handle_request
resp = await request_handler(request)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/aiohttp/web_app.py", line 504, in _handle
resp = await handler(request)
^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3/dist-packages/aiohttp/web_middlewares.py", line 117, in impl
return await handler(request)
^^^^^^^^^^^^^^^^^^^^^^
File "/root/git/aleph-vm/vm_supervisor/supervisor.py", line 42, in server_version_middleware
resp: web.StreamResponse = await handler(request)
^^^^^^^^^^^^^^^^^^^^^^
File "/root/git/aleph-vm/vm_supervisor/views/init.py", line 214, in update_allocations
await start_persistent_vm(instance_hash, pubsub)
File "/root/git/aleph-vm/vm_supervisor/run.py", line 235, in start_persistent_vm
execution = await create_vm_execution(vm_hash=vm_hash)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/git/aleph-vm/vm_supervisor/run.py", line 59, in create_vm_execution
execution = await pool.create_a_vm(
^^^^^^^^^^^^^^^^^^^^^^^
File "/root/git/aleph-vm/vm_supervisor/pool.py", line 68, in create_a_vm
await execution.prepare()
File "/root/git/aleph-vm/vm_supervisor/models.py", line 131, in prepare
await resources.download_all()
File "/root/git/aleph-vm/vm_supervisor/vm/firecracker/instance.py", line 55, in download_all
await asyncio.gather(
File "/root/git/aleph-vm/vm_supervisor/vm/firecracker/instance.py", line 47, in download_runtime
self.rootfs_path = await create_devmapper(
^^^^^^^^^^^^^^^^^^^^^^^
File "/root/git/aleph-vm/vm_supervisor/storage.py", line 280, in create_devmapper
await create_mapped_device(mapped_volume_name, snapshot_table_command)
File "/root/git/aleph-vm/vm_supervisor/storage.py", line 243, in create_mapped_device
await run_in_subprocess(command, stdin_input=table_command.encode())
File "/root/git/aleph-vm/vm_supervisor/utils.py", line 95, in run_in_subprocess
raise subprocess.CalledProcessError(
subprocess.CalledProcessError: Command '['dmsetup', 'create', 'fb02c5b2019413d7f0d39b803e4ad764fae48427ec6c2c934a1d6b8d5aea55f9_rootfs']' returned non-zero exit status 1.
The instance is based on the official Ubuntu image, which is known to work. At least one other Ubuntu instance runs on the same node.