Sometimes rq's job queue will loss job which cend a recursive function with `enqueue_in` to schedule.
b7wch opened this issue · comments
This is my code snippet from a custom django app's command line.
if __name__ == "__main__":
scheduler.enqueue_in(time_delta=timedelta(seconds=1), func=loop, job_id="id-test", on_failure=loop_failure)
@job("high", result_ttl=600)
def loop():
try:
logger.info(f"loop wakeup at {datetime.now().isoformat()}")
sleep = service_loop_schedule()
except Exception as err:
logger.error(f"loop err:{err} \n{traceback.format_exc()}")
sleep = [1]
logger.info(f"start_schedule after loop sleep {sleep}")
def _requeue():
try:
scheduler: DjangoScheduler = django_rq.get_scheduler()
scheduler.enqueue_in(time_delta=timedelta(seconds=min(sleep)), func=loop, job_id="id-test", on_failure=loop_failure)
except Exception as err:
logger.error(f"loop _requeue err:{err} \n{traceback.format_exc()}")
return err
start = time.time()
failed = _requeue()
while failed:
time.sleep(random.randint(1, 2))
failed = _requeue()
if time.time()-start > 10 and failed:
logger.critical(f"loop _requeue error: {failed}")
break
def loop_failure(job: Job, conn: Redis, exec_info):
logger.error(f"loop_failure job:{job.id} info:{exec_info}")
The code in django app's command will run normally for a moment, but it will loss 'loop' job at some time without any other error occur.
Anyone knows the reason why?