segfault after ctx:setStore

[daurnimator]

This sample:

local openssl_ctx = require "openssl.ssl.context"
local ctx = openssl_ctx.new("TLS", true)
local store = require "openssl.x509.store".new()
ctx:setStore(store)

Fails with this backtrace on process exit:

#0  0x00007ffff68c1b60 in X509_LOOKUP_shutdown ()
   from /usr/lib/libcrypto.so.1.1
#1  0x00007ffff68c1dc2 in X509_STORE_free () from /usr/lib/libcrypto.so.1.1
#2  0x00007ffff6bb10d7 in SSL_CTX_free () from /usr/lib/libssl.so.1.1
#3  0x00007ffff6dfb214 in ?? () from /usr/lib/lua/5.3/_openssl.so
#4  0x0000000000408e16 in ?? ()
#5  0x00000000004090e3 in ?? ()
#6  0x0000000000409141 in ?? ()
#7  0x0000000000408562 in ?? ()
#8  0x000000000040943b in ?? ()
#9  0x000000000040a6cd in ?? ()
#10 0x000000000040bdea in ?? ()
#11 0x000000000041064e in ?? ()
#12 0x0000000000404099 in main ()

[daurnimator]

This C program likewise fails (at least with my installed openssl 1.1.0.g).
The issue seems to be in luaossl's defines?

#include <openssl/crypto.h>
#include <openssl/ssl.h>
#include <openssl/x509_vfy.h>

#ifdef LIBRESSL_VERSION_NUMBER
#define OPENSSL_PREREQ(M, m, p) (0)
#define LIBRESSL_PREREQ(M, m, p) \
	(LIBRESSL_VERSION_NUMBER >= (((M) << 28) | ((m) << 20) | ((p) << 12)))
#else
#define OPENSSL_PREREQ(M, m, p) \
	(OPENSSL_VERSION_NUMBER >= (((M) << 28) | ((m) << 20) | ((p) << 12)))
#define LIBRESSL_PREREQ(M, m, p) (0)
#endif

#ifndef HAVE_SSL_CTX_SET1_CERT_STORE
#define HAVE_SSL_CTX_SET1_CERT_STORE (HAVE_SSL_CTX_set1_cert_store || 0) /* backwards compatible with old macro name */
#endif

#ifndef HAVE_SSL_CTX_CERT_STORE
#define HAVE_SSL_CTX_CERT_STORE (!OPENSSL_PREREQ(1,1,0))
#endif

#ifndef HAVE_X509_STORE_REFERENCES
#define HAVE_X509_STORE_REFERENCES (!OPENSSL_PREREQ(1,1,0))
#endif

#if !HAVE_SSL_CTX_SET1_CERT_STORE
#if !HAVE_SSL_CTX_CERT_STORE || !HAVE_X509_STORE_REFERENCES
#define SSL_CTX_set1_cert_store(ctx, store) \
	SSL_CTX_set_cert_store((ctx), (store))
#else
#define SSL_CTX_set1_cert_store(ctx, store) \
	compat_SSL_CTX_set1_cert_store((ctx), (store))

/* to support preprocessor detection below */
#define compat_SSL_CTX_set1_cert_store(ctx, store) \
	compat_SSL_CTX_set1_cert_store((ctx), (store))

static void (compat_SSL_CTX_set1_cert_store)(SSL_CTX *ctx, X509_STORE *store) {
	int n;

	/*
	 * This isn't thead-safe, but using X509_STORE or SSL_CTX objects
	 * from different threads isn't safe generally.
	 */
	if (ctx->cert_store) {
		X509_STORE_free(ctx->cert_store);
		ctx->cert_store = NULL;
	}

	n = store->references;

	SSL_CTX_set_cert_store(ctx, store);

	if (n == store->references)
		CRYPTO_add(&store->references, 1, CRYPTO_LOCK_X509_STORE);
} /* compat_SSL_CTX_set1_cert_store() */
#endif
#endif


int main() {
	SSL_CTX* ctx = SSL_CTX_new(TLS_server_method());
	if (!ctx) return 1;
	X509_STORE *store = X509_STORE_new();
	if (!store) return 2;
	SSL_CTX_set1_cert_store(ctx, store);
	X509_STORE_free(store);
	SSL_CTX_free(ctx);
	return 0;
}

[daurnimator]

Btw, unreleased openssl 1.1.1 actually has a SSL_CTX_set1_cert_store function. Added via openssl/openssl#1755