From 77a7dc3453306f5a800c238b36efbff53c1dad40 Mon Sep 17 00:00:00 2001 From: skoudoro Date: Fri, 16 Feb 2018 10:36:12 -0500 Subject: [PATCH 1/7] fix rows_per_buf definition --- nibabel/streamlines/array_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/array_sequence.py b/nibabel/streamlines/array_sequence.py index d892b9b91d..01d9ff0990 100644 --- a/nibabel/streamlines/array_sequence.py +++ b/nibabel/streamlines/array_sequence.py @@ -37,7 +37,7 @@ def __init__(self, arr_seq, common_shape, dtype): self.common_shape = common_shape n_in_row = reduce(mul, common_shape, 1) bytes_per_row = n_in_row * dtype.itemsize - self.rows_per_buf = bytes_per_row / self.bytes_per_buf + self.rows_per_buf = self.bytes_per_buf / bytes_per_row def update_seq(self, arr_seq): arr_seq._offsets = np.array(self.offsets) From 3724fcfb3c953105cf706c615b4e535f98909734 Mon Sep 17 00:00:00 2001 From: skoudoro Date: Fri, 16 Feb 2018 11:55:30 -0500 Subject: [PATCH 2/7] add a test to check buffer_size effect --- nibabel/streamlines/tests/test_array_sequence.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nibabel/streamlines/tests/test_array_sequence.py b/nibabel/streamlines/tests/test_array_sequence.py index 6925f58b35..9e234a6bc4 100644 --- a/nibabel/streamlines/tests/test_array_sequence.py +++ b/nibabel/streamlines/tests/test_array_sequence.py @@ -2,6 +2,7 @@ import sys import unittest import tempfile +import itertools import numpy as np from nose.tools import assert_equal, assert_raises, assert_true @@ -91,11 +92,20 @@ def test_creating_arraysequence_from_list(self): SEQ_DATA['data']) def test_creating_arraysequence_from_generator(self): - gen = (e for e in SEQ_DATA['data']) - check_arr_seq(ArraySequence(gen), SEQ_DATA['data']) + gen_1, gen_2 = itertools.tee((e for e in SEQ_DATA['data'])) + seq = ArraySequence(gen_1) + seq_with_buffer = ArraySequence(gen_2, buffer_size=256) + + # Check buffer size effect + assert_true(seq_with_buffer.data.shape[0] > seq.data.shape[0]) + assert_equal(seq_with_buffer.common_shape, seq.common_shape) + + # Check generator result + check_arr_seq(seq, SEQ_DATA['data']) + check_arr_seq(seq_with_buffer, SEQ_DATA['data']) # Already consumed generator - check_empty_arr_seq(ArraySequence(gen)) + check_empty_arr_seq(ArraySequence(gen_1)) def test_creating_arraysequence_from_arraysequence(self): seq = ArraySequence(SEQ_DATA['data']) From bc96348e7e8c3bc5c827744c7ac48e79fbfcd16e Mon Sep 17 00:00:00 2001 From: skoudoro Date: Fri, 16 Feb 2018 17:34:49 -0500 Subject: [PATCH 3/7] adressed @effigies suggestion : get a integer by rounding rows_per_buf --- nibabel/streamlines/array_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/array_sequence.py b/nibabel/streamlines/array_sequence.py index 01d9ff0990..4042ccdc9d 100644 --- a/nibabel/streamlines/array_sequence.py +++ b/nibabel/streamlines/array_sequence.py @@ -37,7 +37,7 @@ def __init__(self, arr_seq, common_shape, dtype): self.common_shape = common_shape n_in_row = reduce(mul, common_shape, 1) bytes_per_row = n_in_row * dtype.itemsize - self.rows_per_buf = self.bytes_per_buf / bytes_per_row + self.rows_per_buf = int(np.ceil(self.bytes_per_buf / bytes_per_row)) def update_seq(self, arr_seq): arr_seq._offsets = np.array(self.offsets) From 2ace3efc5e9bc95b335bd8685ded5b6a014744f0 Mon Sep 17 00:00:00 2001 From: skoudoro Date: Fri, 16 Feb 2018 17:35:26 -0500 Subject: [PATCH 4/7] fix concatenate test --- nibabel/streamlines/tests/test_array_sequence.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nibabel/streamlines/tests/test_array_sequence.py b/nibabel/streamlines/tests/test_array_sequence.py index 9e234a6bc4..b63b8eab7b 100644 --- a/nibabel/streamlines/tests/test_array_sequence.py +++ b/nibabel/streamlines/tests/test_array_sequence.py @@ -328,4 +328,4 @@ def test_concatenate(): seqs = [seq[:, [i]] for i in range(seq.common_shape[0])] new_seq = concatenate(seqs, axis=0) assert_true(len(new_seq), seq.common_shape[0] * len(seq)) - assert_array_equal(new_seq._data, seq._data.T.reshape((-1, 1))) + assert_array_equal(new_seq._data, seq._data.T.reshape((-1, 0))) From 7ce7d0f1af5b6628e4f2f2d19cbbbece2a7ba712 Mon Sep 17 00:00:00 2001 From: skoudoro Date: Mon, 19 Feb 2018 17:09:40 -0500 Subject: [PATCH 5/7] addressed matthew comment, and adding shrink to finalize_append step --- nibabel/streamlines/array_sequence.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nibabel/streamlines/array_sequence.py b/nibabel/streamlines/array_sequence.py index 4042ccdc9d..91e94d7ac1 100644 --- a/nibabel/streamlines/array_sequence.py +++ b/nibabel/streamlines/array_sequence.py @@ -37,7 +37,7 @@ def __init__(self, arr_seq, common_shape, dtype): self.common_shape = common_shape n_in_row = reduce(mul, common_shape, 1) bytes_per_row = n_in_row * dtype.itemsize - self.rows_per_buf = int(np.ceil(self.bytes_per_buf / bytes_per_row)) + self.rows_per_buf = max(1, self.bytes_per_buf // bytes_per_row) def update_seq(self, arr_seq): arr_seq._offsets = np.array(self.offsets) @@ -185,6 +185,7 @@ def finalize_append(self): return self._build_cache.update_seq(self) self._build_cache = None + self.shrink_data() def _resize_data_to(self, n_rows, build_cache): """ Resize data array if required """ From c7861d2f1712e6e0382e2453073168a9fb677924 Mon Sep 17 00:00:00 2001 From: skoudoro Date: Mon, 19 Feb 2018 17:12:27 -0500 Subject: [PATCH 6/7] test_correction --- nibabel/streamlines/tests/test_array_sequence.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nibabel/streamlines/tests/test_array_sequence.py b/nibabel/streamlines/tests/test_array_sequence.py index b63b8eab7b..d11cce856a 100644 --- a/nibabel/streamlines/tests/test_array_sequence.py +++ b/nibabel/streamlines/tests/test_array_sequence.py @@ -97,8 +97,8 @@ def test_creating_arraysequence_from_generator(self): seq_with_buffer = ArraySequence(gen_2, buffer_size=256) # Check buffer size effect - assert_true(seq_with_buffer.data.shape[0] > seq.data.shape[0]) - assert_equal(seq_with_buffer.common_shape, seq.common_shape) + assert_equal(seq_with_buffer.data.shape, seq.data.shape) + assert_true(seq_with_buffer._buffer_size > seq._buffer_size) # Check generator result check_arr_seq(seq, SEQ_DATA['data']) @@ -328,4 +328,5 @@ def test_concatenate(): seqs = [seq[:, [i]] for i in range(seq.common_shape[0])] new_seq = concatenate(seqs, axis=0) assert_true(len(new_seq), seq.common_shape[0] * len(seq)) - assert_array_equal(new_seq._data, seq._data.T.reshape((-1, 0))) + assert_array_equal(new_seq._data, seq._data.T.reshape((-1, 1))) + \ No newline at end of file From ddd72408d90b0029f648318237087a45f8ad5d91 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Mon, 19 Feb 2018 21:08:21 -0500 Subject: [PATCH 7/7] STY: Remove spaces after final newline --- nibabel/streamlines/tests/test_array_sequence.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nibabel/streamlines/tests/test_array_sequence.py b/nibabel/streamlines/tests/test_array_sequence.py index d11cce856a..45f50075f8 100644 --- a/nibabel/streamlines/tests/test_array_sequence.py +++ b/nibabel/streamlines/tests/test_array_sequence.py @@ -329,4 +329,3 @@ def test_concatenate(): new_seq = concatenate(seqs, axis=0) assert_true(len(new_seq), seq.common_shape[0] * len(seq)) assert_array_equal(new_seq._data, seq._data.T.reshape((-1, 1))) - \ No newline at end of file