task manager: task id is reported as not found when the task id is available in system.tablets table

[Asias He]

• Start tablet repair
• Check the task id is available in system.tablets table
• Abort the task id
• The abort_task reports error that the task id is not found.

Can be reproduced with the test code in PR: #21973.
Remove the get_task_status loop below and abort as soon as the tablet_task_id is obtained.

@pytest.mark.asyncio
@skip_mode('release', 'error injections are not supported in release mode')
async def test_tablet_repair_error_delete(manager: ManagerClient):
    servers, cql, hosts, table_id = await create_table_insert_data_for_repair(manager)

    token = -1
    async def repair_task():
        await inject_error_on(manager, "repair_tablet_fail_on_rpc_call", servers)
        # Check failed repair request can be deleted
        await manager.api.tablet_repair(servers[0].ip_addr, "test", "test", token)

    async def del_repair_task():
        tablet_task_id = None
        while tablet_task_id == None:
            tablet_task_id = await get_tablet_task_id(cql, hosts[0], table_id, token)
        status = None
        while status == None:
            try:
                status = await manager.api.get_task_status(servers[0].ip_addr, tablet_task_id)
            except:
                status == None
        await manager.api.abort_task(servers[0].ip_addr, tablet_task_id)

    await asyncio.gather(repair_task(), del_repair_task());
    await inject_error_off(manager, "repair_tablet_fail_on_rpc_call", servers)

[Asias He]

The reproducer can be found here:

[e-mail]:asias/scylla.git reproducer_for_21975

            if allow_failed:                                                                                       
                return await resp.json()                                                                           
            if resp.status != 200:                                                                                                                                                                                                     
                text = await resp.text()                                                                                                                                                                                               
>               raise HTTPError(uri, resp.status, params, json, text)       
E               test.pylib.rest_client.HTTPError: HTTP error 400, uri: http://127.226.137.1:10000/task_manager/abort_task/26614b60-bddc-11ef-8112-adb2f382c014, params: None, json: None, body:                                        E               {"message": "task with id 26614b60-bddc-11ef-8112-adb2f382c014 not found", "code": 400}