Test stuck: s3_queue_wait_for_location

[Tatsiana Yaumenenka]

Tested on s3

t/example_test.py:

import json
import logging
import os
import time
from threading import Thread
from typing import Optional

import boto3
from botocore import UNSIGNED
from botocore.config import Config
from moto.core import set_initial_no_auth_action_count
from moto.server import DomainDispatcherApplication, create_backend_app
from testgres.enums import NodeStatus
from werkzeug.serving import (BaseWSGIServer, make_ssl_devcert,
                              make_server)

from .base_test import BaseTest

log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)

class S3Test(BaseTest):
	bucket_name = "test-bucket"
	host="localhost"
	port=5000
	iam_port=5001
	dir_path = os.path.dirname(os.path.realpath(__file__))
	user="ORDB_USER"
	region="us-east-1"

	@classmethod
	@set_initial_no_auth_action_count(4)
	def setUpClass(cls):
		cls.ssl_key = make_ssl_devcert('/tmp/ordb_test_key', cn=cls.host)
		cls.s3_server = MotoServerSSL(ssl_context=cls.ssl_key)
		cls.s3_server.start()
		cls.iam_server = MotoServerSSL(port=cls.iam_port, service='iam',
									   ssl_context=cls.ssl_key)
		cls.iam_server.start()

		iam_config = Config(signature_version = UNSIGNED)

		iam = boto3.client('iam', config=iam_config,
						   endpoint_url=f"https://{cls.host}:{cls.iam_port}",
						   verify=cls.ssl_key[0])
		iam.create_user(UserName=cls.user)
		policy_document = {
			"Version": "2012-10-17",
			"Statement": {"Effect": "Allow", "Action": "*", "Resource": "*"}
		}
		policy = iam.create_policy(PolicyName="ORDB_POLICY",
				 				   PolicyDocument=json.dumps(policy_document))
		policy_arn = policy["Policy"]["Arn"]
		iam.attach_user_policy(UserName=cls.user, PolicyArn=policy_arn)
		response = iam.create_access_key(UserName=cls.user)
		cls.access_key_id = response["AccessKey"]["AccessKeyId"]
		cls.secret_access_key = response["AccessKey"]["SecretAccessKey"]

	@classmethod
	def tearDownClass(cls):
		cls.s3_server.stop()
		cls.iam_server.stop()

	def setUp(self):
		super().setUp()

		session = boto3.Session(
			aws_access_key_id=self.access_key_id,
			aws_secret_access_key=self.secret_access_key,
			region_name=self.region
		)
		host_port = f"https://{self.host}:{self.port}"
		self.client = session.client("s3", endpoint_url=host_port,
									 verify=self.ssl_key[0])
		try:
			self.client.head_bucket(Bucket=self.bucket_name)
		except:
			self.client.create_bucket(Bucket=self.bucket_name)

	def tearDown(self):
		super().tearDown()
		objects = self.client.list_objects(Bucket=self.bucket_name)
		objects = objects.get("Contents", [])
		while objects != []:
			objects = list({"Key": x["Key"]} for x in objects)
			self.client.delete_objects(Bucket=self.bucket_name,
									   Delete={"Objects":objects})
			objects = self.client.list_objects(Bucket=self.bucket_name)
			objects = objects.get("Contents", [])

		self.client.delete_bucket(Bucket=self.bucket_name)
		self.client.close()
	
	
	def test_1(self):
		
		node = self.node
		node.append_conf('postgresql.conf', f"""
			orioledb.s3_mode = true
			orioledb.s3_host = '{self.host}:{self.port}/{self.bucket_name}'
			orioledb.s3_region = '{self.region}'
			orioledb.s3_accesskey = '{self.access_key_id}'
			orioledb.s3_secretkey = '{self.secret_access_key}'
			orioledb.s3_cainfo = '{self.ssl_key[0]}'
			orioledb.s3_num_workers = 1
		""")
		node.start()
		
		with self.node as master:
			with self.getReplica().start() as replica:
				with master.connect() as con1:
					con1.begin()
					
					con1.execute("""

						CREATE EXTENSION IF NOT EXISTS orioledb;			

						CREATE TABLE o_test_1(
							val_1 text PRIMARY KEY
						)USING orioledb;
			
						ALTER TABLE o_test_1 ALTER val_1 TYPE varchar;
					""")
				
					con1.commit()

					catchup_orioledb(replica)

class MotoServerSSL:
	def __init__(self, host: str = "localhost", port: int = 5000,
				 service: Optional[str] = None, ssl_context=None):
		self._host = host
		self._port = port
		self._service = service
		self._thread: Optional[Thread] = None
		self._server: Optional[BaseWSGIServer] = None
		self._server_ready = False
		self._ssl_context = ssl_context

	def _server_entry(self) -> None:
		app = DomainDispatcherApplication(create_backend_app, self._service)

		self._server = make_server(self._host, self._port, app, False,
								   ssl_context=self._ssl_context,
								   passthrough_errors=True)
		self._server_ready = True
		self._server.serve_forever()

	def start(self) -> None:
		self._thread = Thread(target=self._server_entry, daemon=True)
		self._thread.start()
		while not self._server_ready:
			time.sleep(0.1)

	def stop(self) -> None:
		self._server_ready = False
		if self._server:
			self._server.shutdown()

		self._thread.join() 

Stuck stack

0x00007ff57518545a in epoll_wait (epfd=8, events=0x55779bbad228, maxevents=1, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
30      ../sysdeps/unix/sysv/linux/epoll_wait.c: No such file or directory.
#0  0x00007ff57518545a in epoll_wait (epfd=8, events=0x55779bbad228, maxevents=1, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
#1  0x000055779b06792d in WaitEventSetWaitBlock (nevents=1, occurred_events=0x7ffefdcc9e40, cur_timeout=-1, set=0x55779bbad1c8) at latch.c:1489
#2  WaitEventSetWait (set=0x55779bbad1c8, timeout=-1, occurred_events=0x7ffefdcc9e40, nevents=1, wait_event_info=<optimized out>) at latch.c:1435
#3  0x000055779b067d48 in WaitLatch (latch=<optimized out>, wakeEvents=wakeEvents@entry=33, timeout=timeout@entry=-1, wait_event_info=wait_event_info@entry=134217758) at latch.c:497
#4  0x000055779b075f8f in ConditionVariableTimedSleep (wait_event_info=134217758, timeout=-1, cv=0x7ff57336b4a8) at condition_variable.c:163
#5  ConditionVariableTimedSleep (cv=0x7ff57336b4a8, timeout=-1, wait_event_info=134217758) at condition_variable.c:112
#6  0x00007ff5748140c6 in s3_queue_wait_for_location (location=location@entry=3512) at src/s3/queue.c:315
#7  0x00007ff574813641 in s3_perform_backup (maxLocation=3512) at src/s3/checkpoint.c:272
#8  0x00007ff574805d0c in o_perform_checkpoint (redo_pos=50774088, flags=102) at src/checkpoint/checkpoint.c:1071
#9  0x000055779addee7b in CheckPointGuts (checkPointRedo=50774088, flags=flags@entry=102) at xlog.c:6877
#10 0x000055779ade57d3 in CreateCheckPoint (flags=102) at xlog.c:6566
#11 0x000055779aff9717 in CheckpointerMain () at checkpointer.c:480
#12 0x000055779aff75bb in AuxiliaryProcessMain (auxtype=auxtype@entry=CheckpointerProcess) at auxprocess.c:153
#13 0x000055779affd28a in StartChildProcess (type=<optimized out>) at postmaster.c:5446
#14 0x000055779afff5da in ServerLoop () at postmaster.c:1839
#15 0x000055779b000d7b in PostmasterMain (argc=<optimized out>, argv=0x55779bbabac0) at postmaster.c:1485
#16 0x000055779ad208d0 in main (argc=3, argv=0x55779bbabac0) at main.c:202
[Inferior 1 (process 15253) detached]

Run as:

make USE_PGXS=1 testgrescheck_part_1 TESTGRESCHECKS_PART_1="t.s3_test.S3Test.test_1"

Where:

• t - folder with python tests
• s3_test - test file name
• S3Test - test class name
• test_1 - test name

[Alexander Korotkov]

Thank you for reporting. The primary and replica shouldn't connect to the same bucket. This is the current limitation, which is to be removed in the future. Now it's documented.
https://github.com/orioledb/orioledb/blob/main/doc/usage.md#s3-database-storage-experimental