Skip to content

Commit

Permalink
Merge pull request #136 from jakirkham/ret_ndarray_encode_decode
Browse files Browse the repository at this point in the history
Return `ndarray`s from `encode` and (where possible) `decode`
  • Loading branch information
alimanfoo authored Nov 29, 2018
2 parents ecb03c6 + 2012553 commit fd46c4c
Show file tree
Hide file tree
Showing 12 changed files with 573 additions and 366 deletions.
4 changes: 4 additions & 0 deletions docs/release.rst
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ Release notes
code readability and maintainability. By :user:`John Kirkham <jakirkham>` and
:user:`Alistair Miles <alimanfoo>`; :issue:`119`, :issue:`121`, :issue:`128`.

* Return values from encode() and decode() methods are now returned as numpy
arrays for consistency across codecs. By :user:`John Kirkham <jakirkham>`,
:issue:`136`.

* Improvements to handling of errors in the :class:`numcodecs.blosc.Blosc` and
:class:`numcodecs.lz4.LZ4` codecs when the maximum allowed size of an input
buffer is exceeded. By :user:`Jerome Kelleher <jeromekelleher>`, :issue:`80`,
Expand Down
300 changes: 184 additions & 116 deletions numcodecs/blosc.c

Large diffs are not rendered by default.

8 changes: 5 additions & 3 deletions numcodecs/blosc.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING

from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import PY2, text_type, ensure_contiguous_ndarray
from .compat import PY2, text_type, ensure_ndarray, ensure_contiguous_ndarray
from .abc import Codec


Expand Down Expand Up @@ -488,11 +488,13 @@ class Blosc(Codec):

def encode(self, buf):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize)
out = compress(buf, self._cname_bytes, self.clevel, self.shuffle, self.blocksize)
return ensure_ndarray(out)

def decode(self, buf, out=None):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return decompress(buf, out)
out = decompress(buf, out)
return ensure_ndarray(out)

def __repr__(self):
r = '%s(cname=%r, clevel=%r, shuffle=%s, blocksize=%s)' % \
Expand Down
4 changes: 2 additions & 2 deletions numcodecs/bz2.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


from numcodecs.abc import Codec
from numcodecs.compat import ndarray_copy, ensure_contiguous_ndarray
from numcodecs.compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray


class BZ2(Codec):
Expand All @@ -28,7 +28,7 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _bz2.compress(buf, self.level)
return ensure_ndarray(_bz2.compress(buf, self.level))

# noinspection PyMethodMayBeStatic
def decode(self, buf, out=None):
Expand Down
222 changes: 137 additions & 85 deletions numcodecs/lz4.c

Large diffs are not rendered by default.

6 changes: 3 additions & 3 deletions numcodecs/lz4.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ from cpython.bytes cimport PyBytes_FromStringAndSize, PyBytes_AS_STRING

from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import PY2, ensure_contiguous_ndarray
from .compat import PY2, ensure_ndarray, ensure_contiguous_ndarray
from .abc import Codec


Expand Down Expand Up @@ -219,11 +219,11 @@ class LZ4(Codec):

def encode(self, buf):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return compress(buf, self.acceleration)
return ensure_ndarray(compress(buf, self.acceleration))

def decode(self, buf, out=None):
buf = ensure_contiguous_ndarray(buf, self.max_buffer_size)
return decompress(buf, out)
return ensure_ndarray(decompress(buf, out))

def __repr__(self):
r = '%s(acceleration=%r)' % \
Expand Down
6 changes: 3 additions & 3 deletions numcodecs/lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
if _lzma:

from .abc import Codec
from .compat import ndarray_copy, ensure_contiguous_ndarray
from .compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray

# noinspection PyShadowingBuiltins
class LZMA(Codec):
Expand Down Expand Up @@ -51,8 +51,8 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _lzma.compress(buf, format=self.format, check=self.check,
preset=self.preset, filters=self.filters)
return ensure_ndarray(_lzma.compress(buf, format=self.format, check=self.check,
preset=self.preset, filters=self.filters))

def decode(self, buf, out=None):

Expand Down
107 changes: 67 additions & 40 deletions numcodecs/vlen.c

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions numcodecs/vlen.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import numpy as np
from .abc import Codec
from .compat_ext cimport Buffer
from .compat_ext import Buffer
from .compat import ensure_contiguous_ndarray
from .compat import ensure_ndarray, ensure_contiguous_ndarray
from cpython cimport (PyBytes_GET_SIZE, PyBytes_AS_STRING, PyBytes_Check,
PyBytes_FromStringAndSize, PyUnicode_AsUTF8String)
from cpython.buffer cimport PyBUF_ANY_CONTIGUOUS
Expand Down Expand Up @@ -130,7 +130,7 @@ class VLenUTF8(Codec):
memcpy(data, encv, l)
data += l

return out
return ensure_ndarray(out)

@cython.wraparound(False)
@cython.boundscheck(False)
Expand Down
4 changes: 2 additions & 2 deletions numcodecs/zlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@


from .abc import Codec
from .compat import ndarray_copy, ensure_contiguous_ndarray
from .compat import ndarray_copy, ensure_ndarray, ensure_contiguous_ndarray


class Zlib(Codec):
Expand All @@ -28,7 +28,7 @@ def encode(self, buf):
buf = ensure_contiguous_ndarray(buf)

# do compression
return _zlib.compress(buf, self.level)
return ensure_ndarray(_zlib.compress(buf, self.level))

# noinspection PyMethodMayBeStatic
def decode(self, buf, out=None):
Expand Down
Loading

0 comments on commit fd46c4c

Please sign in to comment.