mbedtls_md_setup memory leak if allocation fails

[Guido Vranken]

Description

• Type: Bug
• Priority: Minor

---

Bug

The following code produces a memory leak. This happens because I've modified the allocator to return NULL on the second allocation request.

#include <mbedtls/md.h>
#include <mbedtls/platform.h>

#include <string.h>
#include <stdint.h>
#include <stdlib.h>
#include <stddef.h>

#define CF_CHECK_EQ(expr, res) if ( (expr) != (res) ) { goto end; }
#define CF_CHECK_NE(expr, res) if ( (expr) == (res) ) { goto end; }

static void* mbedTLS_custom_calloc(size_t A, size_t B) {
    static int i;
    i++;
    if ( i == 2 ) return NULL;
    const size_t size = A*B;
    void* p = malloc(size);
    if ( size ) {
        memset(p, 0x00, size);
    }
    return p;
}

static void mbedTLS_custom_free(void* ptr) {
    free(ptr);
}

int main(void)
{
    if ( mbedtls_platform_set_calloc_free(mbedTLS_custom_calloc, mbedTLS_custom_free) != 0 ) {
        abort();
    }

    mbedtls_md_info_t const* md_info = NULL;
    mbedtls_md_context_t md_ctx;

    mbedtls_md_init(&md_ctx);

    CF_CHECK_NE(md_info = mbedtls_md_info_from_type(MBEDTLS_MD_SHA512), NULL);
    CF_CHECK_EQ(mbedtls_md_setup(&md_ctx, md_info, 1), 0 );

end:
    mbedtls_md_free(&md_ctx);

    return 0;
}

Fix it by moving

https://github.com/ARMmbed/mbedtls/blob/3ee91f47f44d4133d3f155b113abfdf7bef98c4e/library/md.c#L471

to before line 461