Memory leak in quantile function

[johanvdw]

Running:

#!/usr/bin/env python3
import numpy as np
import datetime
from crick import TDigest 


td = TDigest()
for j in range(1000000):
    arr = np.array(1)
    td.update(arr)
    x = td.quantile(0.88)

Leads to this memory usage pattern:

[johanvdw]

Some extra info: the graph was made using memory-profiler.
Python 3.7.3
crick: '0.0.3'
Cython: '0.29.21'

[djgagne]

Bumping this issue because I have run into the same problem. I suspect the issue coming from the C function that wraps the quantile call for ndarrays. I ended up rewriting the quantile function in python with a numba guvectorize decorator and got similar speed sans memory leak. I would like to move everything to numba eventually.

crick/crick/tdigest_stubs.c

Lines 519 to 589 in 8ec0b07

	CRICK_INLINE PyArrayObject *tdigest_quantile_ndarray(tdigest_t *T,
	PyArrayObject *q) {
	NpyIter *iter = NULL;
	NpyIter_IterNextFunc *iternext = NULL;
	PyArrayObject *ret = NULL;
	PyArrayObject *op[2] = {NULL};
	npy_uint32 flags;
	npy_uint32 op_flags[2];
	PyArray_Descr *dtypes[2] = {NULL};
	npy_intp *innersizeptr, *strideptr;
	char **dataptr;
	op[0] = q;
	op[1] = NULL;
	flags = NPY_ITER_EXTERNAL_LOOP | NPY_ITER_BUFFERED | NPY_ITER_ZEROSIZE_OK;
	op_flags[0] = NPY_ITER_READONLY | NPY_ITER_ALIGNED;
	op_flags[1] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_ALIGNED;
	dtypes[0] = PyArray_DescrFromType(NPY_FLOAT64);
	if (dtypes[0] == NULL) {
	goto finish;
	}
	dtypes[1] = dtypes[0];
	Py_INCREF(dtypes[1]);
	iter = NpyIter_MultiNew(2, op, flags, NPY_KEEPORDER, NPY_SAFE_CASTING,
	op_flags, dtypes);
	if (iter == NULL) {
	goto finish;
	}
	iternext = NpyIter_GetIterNext(iter, NULL);
	if (iternext == NULL) {
	goto finish;
	}
	dataptr = NpyIter_GetDataPtrArray(iter);
	strideptr = NpyIter_GetInnerStrideArray(iter);
	innersizeptr = NpyIter_GetInnerLoopSizePtr(iter);
	/* Preprocess the centroids */
	tdigest_query_prep(T);
	do {
	char *data_q = dataptr[0];
	char *data_out = dataptr[1];
	npy_intp stride_q = strideptr[0];
	npy_intp stride_out = strideptr[1];
	npy_intp count = *innersizeptr;
	while (count--) {
	*(npy_float64 *)data_out = tdigest_quantile(T, *(npy_float64 *)data_q);
	data_q += stride_q;
	data_out += stride_out;
	}
	} while (iternext(iter));
	ret = NpyIter_GetOperandArray(iter)[1];
	Py_INCREF(ret);
	finish:
	Py_XDECREF(dtypes[0]);
	Py_XDECREF(dtypes[1]);
	if (iter != NULL) {
	if (NpyIter_Deallocate(iter) != NPY_SUCCEED) {
	Py_XDECREF(ret);
	ret = NULL;
	}
	}
	return ret;
	}

[dcherian]

@djgagne did you also convert the rest of t-digest code to numba or just the quantile at the end?

[djgagne]

I have written numba versions of cdf and quantile so far but have not tackled the code for updating and merging the tdigest. I have the code in a PR for my bridgescaler package.