From e0dbcbd163e0737de817dd4a4482496205d2f97b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Tue, 31 Oct 2017 15:00:10 +0100 Subject: [PATCH 01/18] added example of big 1D array --- larray_editor/api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/larray_editor/api.py b/larray_editor/api.py index 5bdd4768..594376f6 100644 --- a/larray_editor/api.py +++ b/larray_editor/api.py @@ -363,6 +363,9 @@ def make_demo(width=20, ball_radius=5, path_radius=5, steps=30): arr_obj = ndtest((2, 3)).astype(object) arr_str = ndtest((2, 3)).astype(str) big = ndtest((1000, 1000, 500)) + big1d = ndrange(1000000) + # force big1d.axes[0]._mapping to be created so that we do not measure that delay in the editor + big1d[{}] # test autoresizing long_labels = zeros('a=a_long_label,another_long_label; b=this_is_a_label,this_is_another_one') From 5660a998819b26bda5df725c16b8c4cd25cf31ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Fri, 10 Nov 2017 11:42:59 +0100 Subject: [PATCH 02/18] when using the scrollwheel on filter comboboxes, only emit one signal (issue #93) (so that we compute & redraw the new visible data only once) --- larray_editor/combo.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/larray_editor/combo.py b/larray_editor/combo.py index 080854fd..1d56bed2 100644 --- a/larray_editor/combo.py +++ b/larray_editor/combo.py @@ -115,14 +115,14 @@ def on_model_item_changed(self, item): else: model[0].checked = 'partial' model.blockSignals(False) - is_checked = [i for i, item in enumerate(model[1:]) if item.checked] - self.checkedItemsChanged.emit(is_checked) + checked_indices = [i for i, item in enumerate(model[1:]) if item.checked] + self.checkedItemsChanged.emit(checked_indices) def select_offset(self, offset): """offset: 1 for next, -1 for previous""" model = self._model - # model.blockSignals(True) + model.blockSignals(True) indices_checked = [i for i, item in enumerate(model) if item.checked] first_checked = indices_checked[0] # check first_checked + offset, uncheck the rest @@ -135,6 +135,8 @@ def select_offset(self, offset): is_checked = ["partial"] + [i == to_check for i in range(1, len(model))] for checked, item in zip(is_checked, model): item.checked = checked + model.blockSignals(False) + self.checkedItemsChanged.emit([to_check - 1]) def addItem(self, text): item = StandardItem(text) From 09f492784a9bf0c5f7515b055b1a1575af97fa87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 15:08:34 +0100 Subject: [PATCH 03/18] fixed diff_checkbox state being ignored when switching arrays --- larray_editor/comparator.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/larray_editor/comparator.py b/larray_editor/comparator.py index 2f34be7a..f640ef2a 100644 --- a/larray_editor/comparator.py +++ b/larray_editor/comparator.py @@ -23,10 +23,10 @@ def __init__(self, parent=None): maxdiff_layout.addStretch() layout.addLayout(maxdiff_layout) - self.arraywidget = ArrayEditorWidget(self, np.array([]), readonly=True, bg_gradient='red-white-blue') + self.arraywidget = ArrayEditorWidget(self, data=None, readonly=True, bg_gradient='red-white-blue') diff_checkbox = QCheckBox(_('Differences Only')) - diff_checkbox.stateChanged.connect(self.show_differences_only) + diff_checkbox.stateChanged.connect(self.display) self.diff_checkbox = diff_checkbox self.arraywidget.btn_layout.addWidget(diff_checkbox) @@ -72,11 +72,10 @@ def set_data(self, arrays, stack_axis): self.bg_value = full_like(self.array, 0.5) self.maxdiff_label.setText(str(maxabsreldiff)) - self.arraywidget.set_data(self.array, bg_value=self.bg_value) + self.display(self.diff_checkbox.isChecked()) - def show_differences_only(self, yes): - if yes: - # only show rows with a difference. For some reason, this is abysmally slow though. + def display(self, diff_only): + if diff_only: row_filter = (~self.isequal).any(self.stack_axis.name) self.arraywidget.set_data(self.array[row_filter], bg_value=self.bg_value[row_filter]) else: From 49a5d1191bd511702873760190d80ba0627dd4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 15:31:01 +0100 Subject: [PATCH 04/18] fixed compare+diffonly when stack failed also display the exception (even though it is currently unreadable) --- larray_editor/comparator.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/larray_editor/comparator.py b/larray_editor/comparator.py index f640ef2a..ab099e45 100644 --- a/larray_editor/comparator.py +++ b/larray_editor/comparator.py @@ -3,7 +3,8 @@ from qtpy.QtWidgets import (QWidget, QVBoxLayout, QListWidget, QSplitter, QDialogButtonBox, QHBoxLayout, QDialog, QLabel, QCheckBox) -from larray import LArray, Session, Axis, X, stack, full_like, nan, zeros_like, isnan, larray_nan_equal, nan_equal +from larray import (LArray, Session, Axis, X, stack, full, full_like, zeros_like, + nan, isnan, larray_nan_equal, nan_equal) from larray_editor.utils import ima, replace_inf, _ from larray_editor.arraywidget import ArrayEditorWidget @@ -43,8 +44,8 @@ def set_data(self, arrays, stack_axis): try: self.array = stack(arrays, stack_axis) array0 = self.array[stack_axis.i[0]] - except: - self.array = LArray([np.nan]) + except Exception as e: + self.array = LArray(str(e)) array0 = self.array try: self.isequal = nan_equal(self.array, array0) @@ -68,18 +69,20 @@ def set_data(self, arrays, stack_axis): self.bg_value = full_like(self.array, 0.5) except TypeError: # str/object array - maxabsreldiff = np.nan + maxabsreldiff = nan self.bg_value = full_like(self.array, 0.5) self.maxdiff_label.setText(str(maxabsreldiff)) self.display(self.diff_checkbox.isChecked()) def display(self, diff_only): - if diff_only: - row_filter = (~self.isequal).any(self.stack_axis.name) - self.arraywidget.set_data(self.array[row_filter], bg_value=self.bg_value[row_filter]) - else: - self.arraywidget.set_data(self.array, bg_value=self.bg_value) + array = self.array + bg_value = self.bg_value + if diff_only and self.isequal.ndim > 0: + row_filter = (~self.isequal).any(self.stack_axis) + array = array[row_filter] + bg_value = bg_value[row_filter] + self.arraywidget.set_data(array, bg_value=bg_value) class ArrayComparator(QDialog): From f5a046522cb0b08f3de7927222e8e18a0c709c2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 15:36:45 +0100 Subject: [PATCH 05/18] fixed LArrayDataAdapter.get_xlabels being slow on large 1D arrays (issue #93) --- larray_editor/arrayadapter.py | 4 +++- larray_editor/arraymodel.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/larray_editor/arrayadapter.py b/larray_editor/arrayadapter.py index 8cfbae9c..c60dff1a 100644 --- a/larray_editor/arrayadapter.py +++ b/larray_editor/arrayadapter.py @@ -51,7 +51,9 @@ def get_xlabels(self): if self.filtered_data.size == 0 or len(axes) == 0: return None else: - return [[label] for label in axes.labels[-1]] + # this is a lazy equivalent of: + # return [(label,) for label in axes.labels[-1]] + return Product([axes.labels[-1]]) def get_ylabels(self): axes = self.filtered_data.axes diff --git a/larray_editor/arraymodel.py b/larray_editor/arraymodel.py index 96a49f82..f91905e0 100644 --- a/larray_editor/arraymodel.py +++ b/larray_editor/arraymodel.py @@ -2,7 +2,8 @@ import numpy as np from larray_editor.utils import (get_font, from_qvariant, to_qvariant, to_text_string, - is_float, is_number, LinearGradient, SUPPORTED_FORMATS, scale_to_01range) + is_float, is_number, LinearGradient, SUPPORTED_FORMATS, scale_to_01range, + Product) from qtpy.QtCore import Qt, QModelIndex, QAbstractTableModel from qtpy.QtGui import QColor from qtpy.QtWidgets import QMessageBox @@ -132,8 +133,9 @@ def __init__(self, parent=None, data=None, readonly=False, font=None): def _set_data(self, data, changes=None): if data is None: data = [[]] - if not isinstance(data, (list, tuple)): - QMessageBox.critical(self.dialog, "Error", "Expected list or tuple.") + # TODO: use sequence instead + if not isinstance(data, (list, tuple, Product)): + QMessageBox.critical(self.dialog, "Error", "Expected list, tuple or Product") data = [[]] self._data = data self.total_rows = len(data[0]) From 713869ae260f8b2ce08c3cfeff71eba4eb182300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:25:13 +0100 Subject: [PATCH 06/18] prevent "filters" label from appearing for scalar (0D) arrays --- larray_editor/arraywidget.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index ea9064f6..826749cb 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -766,8 +766,8 @@ def set_data(self, data=None, bg_value=None): # update filters filters_layout = self.filters_layout clear_layout(filters_layout) - # len(axes) > 0 is not good because we can have len 0 axes - if la_data.size > 0: + # data.size > 0 to avoid arrays with length 0 axes and len(axes) > 0 to avoid scalars (scalar.size == 1) + if la_data.size > 0 and len(axes) > 0: filters_layout.addWidget(QLabel(_("Filters"))) for axis, display_name in zip(axes, display_names): filters_layout.addWidget(QLabel(display_name)) From 244151ca243c32d4d4ccc0d47ac2c5e535c3b8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:26:11 +0100 Subject: [PATCH 07/18] made large 1D arrays display faster by not creating the filter combobox for axes >= 10000 elements (issue #93) --- larray_editor/arraywidget.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 826749cb..0e95ce21 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -771,7 +771,12 @@ def set_data(self, data=None, bg_value=None): filters_layout.addWidget(QLabel(_("Filters"))) for axis, display_name in zip(axes, display_names): filters_layout.addWidget(QLabel(display_name)) - filters_layout.addWidget(self.create_filter_combo(axis)) + # FIXME: on very large axes, this is getting too slow. Ideally the combobox should use a model which + # only fetch labels when they are needed to be displayed + if len(axis) < 10000: + filters_layout.addWidget(self.create_filter_combo(axis)) + else: + filters_layout.addWidget(QLabel("too big to be filtered")) filters_layout.addStretch() self.data_adapter.update_filtered_data({}) From ef17c339771d37c22232e48b4577046b71921117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:44:36 +0100 Subject: [PATCH 08/18] use min/max instead of nanmin/nanmax because nans are already filtered out by isfinite --- larray_editor/arraymodel.py | 4 ++-- larray_editor/arraywidget.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/larray_editor/arraymodel.py b/larray_editor/arraymodel.py index f91905e0..5fcc41af 100644 --- a/larray_editor/arraymodel.py +++ b/larray_editor/arraymodel.py @@ -272,8 +272,8 @@ def reset_minmax(self): color_value = self.color_func(data) if self.color_func is not None else data # ignore nan, -inf, inf (setting them to 0 or to very large numbers is not an option) color_value = color_value[np.isfinite(color_value)] - self.vmin = float(np.nanmin(color_value)) - self.vmax = float(np.nanmax(color_value)) + self.vmin = float(np.min(color_value)) + self.vmax = float(np.max(color_value)) self.bgcolor_possible = True # ValueError for empty arrays, TypeError for object/string arrays except (TypeError, ValueError): diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 0e95ce21..91623f63 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -864,7 +864,7 @@ def format_helper(self, data): if not data.size: return 0, 0, False data = np.where(np.isfinite(data), data, 0) - vmin, vmax = np.nanmin(data), np.nanmax(data) + vmin, vmax = np.min(data), np.max(data) absmax = max(abs(vmin), abs(vmax)) logabsmax = math.log10(absmax) if absmax else 0 # minimum number of zeros before meaningful fractional part From d3901b1bb7dbe0409789742d9cfe459561ae8aa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:45:23 +0100 Subject: [PATCH 09/18] select first row in compare(sessions) --- larray_editor/comparator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/larray_editor/comparator.py b/larray_editor/comparator.py index ab099e45..5503fbd6 100644 --- a/larray_editor/comparator.py +++ b/larray_editor/comparator.py @@ -183,7 +183,6 @@ def setup_and_check(self, sessions, names, title='', colors='red-white-blue'): self.listwidget = listwidget comparatorwidget = ComparatorWidget(self) - comparatorwidget.set_data(self.get_arrays(array_names[0]), self.stack_axis) self.comparatorwidget = comparatorwidget main_splitter = QSplitter(Qt.Horizontal) @@ -209,6 +208,7 @@ def setup_and_check(self, sessions, names, title='', colors='red-white-blue'): # Make the dialog act as a window self.setWindowFlags(Qt.Window) + self.listwidget.setCurrentRow(0) return True def get_arrays(self, name): From c5351580a042cc17b865bd0ec967ea1c3dda889e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:47:39 +0100 Subject: [PATCH 10/18] fixed readonly being always False when editing arrays (doh!) --- larray_editor/arraywidget.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 91623f63..6687eb46 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -530,7 +530,8 @@ def __init__(self, parent, data=None, readonly=False, bg_value=None, bg_gradient minvalue=None, maxvalue=None): QWidget.__init__(self, parent) assert bg_gradient in gradient_map - readonly = np.isscalar(data) + if data is not None and np.isscalar(data): + readonly = True self.readonly = readonly self.model_axes = LabelsArrayModel(parent=self, readonly=readonly) From 4a4b2fd09daa8b9d6be5a58fc9184ffda6afc9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:50:05 +0100 Subject: [PATCH 11/18] do not update_digits_scientific on accept_changes (it makes no sense) --- larray_editor/arraywidget.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 6687eb46..1dd7ce3d 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -947,8 +947,7 @@ def dirty(self): def accept_changes(self): """Accept changes""" - la_data = self.data_adapter.accept_changes() - self._update_digits_scientific(la_data) + self.data_adapter.accept_changes() def reject_changes(self): """Reject changes""" From 049edec288b6c15f1845aa7ca5362b2294585e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Wed, 8 Nov 2017 16:51:02 +0100 Subject: [PATCH 12/18] made LinearGradient more foregiving (unsure it is a good idea) --- larray_editor/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/larray_editor/utils.py b/larray_editor/utils.py index e91b47c1..24cb352a 100644 --- a/larray_editor/utils.py +++ b/larray_editor/utils.py @@ -282,10 +282,8 @@ def __getitem__(self, key): ------- QColor """ - if np.isnan(key): + if np.isnan(key) or key < 0 or key > 1: return self.nan_color - # this is enough to also avoid nan, inf & -inf - assert 0 <= key <= 1 pos_idx = np.searchsorted(self.positions, key, side='right') - 1 # if we are exactly on one of the bounds if pos_idx > 0 and key in self.positions: From 21ef4c890230abb776591871eeeea6a2fbc2d020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Thu, 26 Oct 2017 11:00:08 +0200 Subject: [PATCH 13/18] reworked editor internals to avoid calling update methods several times per change (issue #93) --- larray_editor/arraywidget.py | 155 ++++++++++++++++++----------------- larray_editor/editor.py | 19 ++--- 2 files changed, 88 insertions(+), 86 deletions(-) diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 1dd7ce3d..3ff8e776 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -524,7 +524,6 @@ def __init__(self, parent, data_scrollbar): gradient_map = dict(available_gradients) - class ArrayEditorWidget(QWidget): def __init__(self, parent, data=None, readonly=False, bg_value=None, bg_gradient='blue-red', minvalue=None, maxvalue=None): @@ -756,19 +755,21 @@ def dropEvent(self, event): def set_data(self, data=None, bg_value=None): # update adapter - self.data_adapter.set_data(data, bg_value=bg_value) - la_data = self.data_adapter.get_data() - axes = la_data.axes + if data is None: + data = la.LArray([]) + else: + data = la.aslarray(data) + axes = data.axes display_names = axes.display_names - # update data format and bgcolor - self._update_digits_scientific(la_data) + # update data format + self._update_digits_scientific(data) # update filters filters_layout = self.filters_layout clear_layout(filters_layout) # data.size > 0 to avoid arrays with length 0 axes and len(axes) > 0 to avoid scalars (scalar.size == 1) - if la_data.size > 0 and len(axes) > 0: + if data.size > 0 and len(axes) > 0: filters_layout.addWidget(QLabel(_("Filters"))) for axis, display_name in zip(axes, display_names): filters_layout.addWidget(QLabel(display_name)) @@ -779,7 +780,10 @@ def set_data(self, data=None, bg_value=None): else: filters_layout.addWidget(QLabel("too big to be filtered")) filters_layout.addStretch() - self.data_adapter.update_filtered_data({}) + + self.gradient_chooser.setEnabled(self.model_data.bgcolor_possible) + + self.data_adapter.set_data(data, bg_value=bg_value) # reset default size self.view_axes.set_default_size() @@ -787,79 +791,85 @@ def set_data(self, data=None, bg_value=None): self.view_xlabels.set_default_size() self.view_data.set_default_size() - def _update_digits_scientific(self, data): + # called by set_data and ArrayEditorWidget.accept_changes (this should not be the case IMO) + # two cases: + # * set_data should update both scientific and ndigits + # * toggling scientific checkbox should update only ndigits + def _update_digits_scientific(self, data, scientific=None): """ data : LArray """ - # TODO: Adapter must provide a method to return a data sample as a Numpy array - assert isinstance(data, la.LArray) - data = data.data - size, dtype = data.size, data.dtype - # this will yield a data sample of max 199 - step = (size // 100) if size > 100 else 1 - data_sample = data.flat[::step] - - # TODO: refactor so that the expensive format_helper is not called - # twice (or the values are cached) - use_scientific = self.choose_scientific(data_sample) + dtype = data.dtype + if dtype.type in (np.str, np.str_, np.bool_, np.bool, np.object_): + scientific = False + ndecimals = 0 + else: + # XXX: move this to the adapter (return a data sample as a Numpy array) + data = self._get_sample(data) + + # max_digits = self.get_max_digits() + # default width can fit 8 chars + # FIXME: use max_digits? + avail_digits = 8 + frac_zeros, int_digits, has_negative = self.format_helper(data) + + # choose whether or not to use scientific notation + # ================================================ + if scientific is None: + # use scientific format if there are more integer digits than we can display or if we can display more + # information that way (scientific format "uses" 4 digits, so we have a net win if we have >= 4 zeros -- + # *including the integer one*) + # TODO: only do so if we would actually display more information + # 0.00001 can be displayed with 8 chars + # 1e-05 + # would + scientific = int_digits > avail_digits or frac_zeros >= 4 + + # determine best number of decimals to display + # ============================================ + # TODO: ndecimals vs self.digits => rename self.digits to either frac_digits or ndecimals + data_frac_digits = self._data_digits(data) + if scientific: + int_digits = 2 if has_negative else 1 + exp_digits = 4 + else: + exp_digits = 0 + # - 1 for the dot + ndecimals = avail_digits - 1 - int_digits - exp_digits - # XXX: self.ndecimals vs self.digits - self.digits = self.choose_ndecimals(data_sample, use_scientific) - self.use_scientific = use_scientific - self.model_data.set_format(self.cell_format) + if ndecimals < 0: + ndecimals = 0 - self.digits_spinbox.setValue(self.digits) - self.digits_spinbox.setEnabled(is_number(dtype)) + if data_frac_digits < ndecimals: + ndecimals = data_frac_digits - self.scientific_checkbox.setChecked(use_scientific) - self.scientific_checkbox.setEnabled(is_number(dtype)) + self.digits = ndecimals - self.gradient_chooser.setEnabled(self.model_data.bgcolor_possible) + self.use_scientific = scientific - def choose_scientific(self, data): - # max_digits = self.get_max_digits() - # default width can fit 8 chars - # FIXME: use max_digits? - avail_digits = 8 - if data.dtype.type in (np.str, np.str_, np.bool_, np.bool, np.object_): - return False - - frac_zeros, int_digits, _ = self.format_helper(data) - - # if there are more integer digits than we can display or we can - # display more information by using scientific format, do so - # (scientific format "uses" 4 digits, so we win if have >= 4 zeros - # -- *including the integer one*) - # TODO: only do so if we would actually display more information - # 0.00001 can be displayed with 8 chars - # 1e-05 - # would - return int_digits > avail_digits or frac_zeros >= 4 - - def choose_ndecimals(self, data, scientific): - if data.dtype.type in (np.str, np.str_, np.bool_, np.bool, np.object_): - return 0 + self.digits_spinbox.blockSignals(True) + self.digits_spinbox.setValue(ndecimals) + self.digits_spinbox.setEnabled(is_number(dtype)) + self.digits_spinbox.blockSignals(False) - # max_digits = self.get_max_digits() - # default width can fit 8 chars - # FIXME: use max_digits? - avail_digits = 8 - data_frac_digits = self._data_digits(data) - _, int_digits, negative = self.format_helper(data) - if scientific: - int_digits = 2 if negative else 1 - exp_digits = 4 - else: - exp_digits = 0 - # - 1 for the dot - ndecimals = avail_digits - 1 - int_digits - exp_digits + self.scientific_checkbox.blockSignals(True) + self.scientific_checkbox.setChecked(scientific) + self.scientific_checkbox.setEnabled(is_number(dtype)) + self.scientific_checkbox.blockSignals(False) - if ndecimals < 0: - ndecimals = 0 + # setting the format explicitly instead of relying on digits_spinbox.digits_changed to set it because + # digits_changed is only triggered when digits actually changed, not when passing from scientific -> non + # scientific or number -> object + self.model_data.set_format(self.cell_format) - if data_frac_digits < ndecimals: - ndecimals = data_frac_digits - return ndecimals + def _get_sample(self, data): + assert isinstance(data, la.LArray) + data = data.data + size = data.size + # this will yield a data sample of max 199 + step = (size // 100) if size > 100 else 1 + sample = data.flat[::step] + return sample[np.isfinite(sample)] def format_helper(self, data): if not data.size: @@ -963,10 +973,7 @@ def cell_format(self): return '%%.%d%s' % (self.digits, format_letter) def scientific_changed(self, value): - self.use_scientific = value - self.digits = self.choose_ndecimals(self.data_adapter.get_data(), value) - self.digits_spinbox.setValue(self.digits) - self.model_data.set_format(self.cell_format) + self._update_digits_scientific(self.data_adapter.get_data(), value) def digits_changed(self, value): self.digits = value diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 28529dc0..5f8e3435 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -108,9 +108,7 @@ def setup_and_check(self, data, title='', readonly=False, minvalue=None, maxvalu widget.setLayout(layout) self._listwidget = QListWidget(self) - self._listwidget.currentItemChanged.connect(self.on_item_changed) - # this is a workaround for the fact that no currentItemChanged signal is emitted when no item was selected - # before + # this is a bit more reliable than currentItemChanged which is not emitted when no item was selected before self._listwidget.itemSelectionChanged.connect(self.on_selection_changed) self._listwidget.setMinimumWidth(45) @@ -340,16 +338,14 @@ def select_list_item(self, to_display): # we need to update the array widget explicitly self.set_current_array(self.data[to_display], to_display) else: - # for some reason, on_item_changed is not triggered when no item was selected - if not prev_selected: - self.set_current_array(self.data[to_display], to_display) self._listwidget.setCurrentItem(changed_items[0]) def update_mapping(self, value): # XXX: use ordered set so that the order is non-random if the underlying container is ordered? keys_before = set(self.data.keys()) keys_after = set(value.keys()) - # contains both new and updated keys (but not deleted keys) + # Contains both new and keys for which the object id changed (but not deleted keys nor inplace modified keys). + # Inplace modified arrays should be already handled in ipython_cell_executed by the setitem_pattern. changed_keys = [k for k in keys_after if value[k] is not self.data.get(k)] # when a key is re-assigned, it can switch from being displayable to non-displayable or vice versa @@ -432,6 +428,8 @@ def ipython_cell_executed(self): # otherwise it should have failed at this point, but let us be sure if varname in clean_ns: if self._display_in_grid(varname, clean_ns[varname]): + # XXX: this completely refreshes the array, including detecting scientific & ndigits, which might + # not be what we want in this case self.select_list_item(varname) else: # not setitem => assume expr or normal assignment @@ -442,6 +440,7 @@ def ipython_cell_executed(self): self.select_list_item(last_input) else: # any statement can contain a call to a function which updates globals + # this will select (or refresh) the "first" changed array self.update_mapping(clean_ns) # if the statement produced any output (probably because it is a simple expression), display it. @@ -463,11 +462,7 @@ def on_selection_changed(self, *args, **kwargs): assert len(selected) == 1 selected_item = selected[0] assert isinstance(selected_item, QListWidgetItem) - self.on_item_changed(selected_item, None) - - def on_item_changed(self, curr, prev): - if curr is not None: - name = str(curr.text()) + name = str(selected_item.text()) array = self.data[name] self.set_current_array(array, name) expr = self.expressions.get(name, name) From f8bc4af04c4f365d16874a7a4cd15ff46da31843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Thu, 9 Nov 2017 13:26:46 +0100 Subject: [PATCH 14/18] simplified and optimized larray editor internals (issue #93) * one of the goal was to make switching from one array to another as fast as possible by cutting down on the repeated calls (to various set_data and datamodel.reset()) * tightened what is accepted by each internal class. There is only one expected type for the data. External facing classes should still accept the same objects. * all internal classes which "hold" data are created without any data in __init__ but require a set_data before they can function. * data_model.reset_minmax needs to be called explicitly. * data_model.reset() needs to be called explicitly when "everything is done" --- larray_editor/arrayadapter.py | 65 +++++++++++++--------------- larray_editor/arraymodel.py | 79 +++++++++++++++++------------------ larray_editor/arraywidget.py | 79 +++++++++++++++++++++-------------- larray_editor/editor.py | 13 +++--- 4 files changed, 120 insertions(+), 116 deletions(-) diff --git a/larray_editor/arrayadapter.py b/larray_editor/arrayadapter.py index c60dff1a..32d8f380 100644 --- a/larray_editor/arrayadapter.py +++ b/larray_editor/arrayadapter.py @@ -2,34 +2,24 @@ import numpy as np import larray as la -from larray_editor.utils import Product, _LazyNone, _LazyDimLabels + +from larray_editor.utils import Product, _LazyDimLabels class LArrayDataAdapter(object): - def __init__(self, axes_model, xlabels_model, ylabels_model, data_model, - data=None, changes=None, current_filter=None, bg_gradient=None, bg_value=None): + def __init__(self, axes_model, xlabels_model, ylabels_model, data_model): # set models self.axes_model = axes_model self.xlabels_model = xlabels_model self.ylabels_model = ylabels_model self.data_model = data_model - # set current filter - if current_filter is None: - current_filter = {} - assert isinstance(current_filter, dict) - self.current_filter = current_filter - # set changes - if changes is None: - changes = {} - self.set_changes(changes) - # set data - if data is None: - data = np.empty((0, 0), dtype=np.int8) - self.set_data(data, bg_value, current_filter) - - def set_changes(self, changes=None): - assert isinstance(changes, dict) - self.changes = changes + + # these are not valid values, but should overwritten by set_data which must be called before the adapter is used + self.current_filter = None + self.changes = None + self.la_data = None + self.bg_value = None + self.filtered_data = None def get_axes_names(self): return self.filtered_data.axes.display_names @@ -39,7 +29,7 @@ def get_axes(self): # test self.filtered_data.size == 0 is required in case an instance built as LArray([]) is passed # test len(axes) == 0 is required when a user filters until to get a scalar if self.filtered_data.size == 0 or len(axes) == 0: - return None + return [[]] else: axes_names = axes.display_names if len(axes_names) >= 2: @@ -49,7 +39,7 @@ def get_axes(self): def get_xlabels(self): axes = self.filtered_data.axes if self.filtered_data.size == 0 or len(axes) == 0: - return None + return [[]] else: # this is a lazy equivalent of: # return [(label,) for label in axes.labels[-1]] @@ -58,7 +48,7 @@ def get_xlabels(self): def get_ylabels(self): axes = self.filtered_data.axes if self.filtered_data.size == 0 or len(axes) == 0: - return None + return [[]] elif len(axes) == 1: return [['']] else: @@ -101,34 +91,37 @@ def get_bg_value_2D(self, shape_2D): # XXX: or create two methods?: # - set_data (which reset the current filter) # - update_data (which sets new data but keeps current filter unchanged) - def set_data(self, data, bg_value=None, current_filter=None): - if data is None: - data = la.LArray([]) - if current_filter is None: - self.current_filter = {} + def set_data(self, data, bg_value=None): + assert isinstance(data, la.LArray) + self.current_filter = {} self.changes = {} self.la_data = la.aslarray(data) self.bg_value = la.aslarray(bg_value) if bg_value is not None else None - self.update_filtered_data(current_filter, reset_minmax=True) + self.update_filtered_data() + self.data_model.reset_minmax() + self.data_model.reset() - def update_filtered_data(self, current_filter=None, reset_minmax=False): - if current_filter is not None: - assert isinstance(current_filter, dict) - self.current_filter = current_filter + def update_filtered_data(self): + assert isinstance(self.la_data, la.LArray) self.filtered_data = self.la_data[self.current_filter] + if np.isscalar(self.filtered_data): self.filtered_data = la.aslarray(self.filtered_data) + axes = self.get_axes() xlabels = self.get_xlabels() ylabels = self.get_ylabels() data_2D = self.get_2D_data() changes_2D = self.get_changes_2D() bg_value_2D = self.get_bg_value_2D(data_2D.shape) + self.axes_model.set_data(axes) self.xlabels_model.set_data(xlabels) self.ylabels_model.set_data(ylabels) - self.data_model.set_data(data_2D, changes_2D, reset_minmax=reset_minmax) - self.data_model.set_bg_value(bg_value_2D) + # using the protected version of the method to avoid calling reset() several times + self.data_model._set_data(data_2D) + self.data_model._set_changes(changes_2D) + self.data_model._set_bg_value(bg_value_2D) def get_data(self): return self.la_data @@ -223,6 +216,7 @@ def change_filter(self, axis, indices): else: self.current_filter[axis_id] = axis.labels[indices] self.update_filtered_data() + self.data_model.reset() def clear_changes(self): self.changes.clear() @@ -238,6 +232,7 @@ def accept_changes(self): self.la_data.i[axes.translate_full_key(k)] = v # update models self.update_filtered_data() + self.data_model.reset() # clear changes self.clear_changes() # return modified data diff --git a/larray_editor/arraymodel.py b/larray_editor/arraymodel.py index 5fcc41af..abd2eb3f 100644 --- a/larray_editor/arraymodel.py +++ b/larray_editor/arraymodel.py @@ -20,8 +20,6 @@ class AbstractArrayModel(QAbstractTableModel): ---------- parent : QWidget, optional Parent Widget. - data : array-like, optional - Input data. readonly : bool, optional If True, data cannot be changed. False by default. font : QFont, optional @@ -30,7 +28,7 @@ class AbstractArrayModel(QAbstractTableModel): ROWS_TO_LOAD = 500 COLS_TO_LOAD = 40 - def __init__(self, parent=None, data=None, readonly=False, font=None): + def __init__(self, parent=None, readonly=False, font=None): QAbstractTableModel.__init__(self) self.dialog = parent @@ -45,13 +43,12 @@ def __init__(self, parent=None, data=None, readonly=False, font=None): self.cols_loaded = 0 self.total_rows = 0 self.total_cols = 0 - self.set_data(data) - def _set_data(self, data, changes=None): + def _set_data(self, data): raise NotImplementedError() - def set_data(self, data, changes=None, **kwargs): - self._set_data(data, changes, **kwargs) + def set_data(self, data): + self._set_data(data) self.reset() def rowCount(self, parent=QModelIndex()): @@ -119,20 +116,16 @@ class LabelsArrayModel(AbstractArrayModel): ---------- parent : QWidget, optional Parent Widget. - data : nested list or tuple, optional - Input data. readonly : bool, optional If True, data cannot be changed. False by default. font : QFont, optional Font. Default is `Calibri` with size 11. """ - def __init__(self, parent=None, data=None, readonly=False, font=None): - AbstractArrayModel.__init__(self, parent, data, readonly, font) + def __init__(self, parent=None, readonly=False, font=None): + AbstractArrayModel.__init__(self, parent, readonly, font) self.font.setBold(True) - def _set_data(self, data, changes=None): - if data is None: - data = [[]] + def _set_data(self, data): # TODO: use sequence instead if not isinstance(data, (list, tuple, Product)): QMessageBox.critical(self.dialog, "Error", "Expected list, tuple or Product") @@ -188,42 +181,44 @@ class DataArrayModel(AbstractArrayModel): Parameters ---------- - data : Numpy ndarray, optional - Input 2D array. - format : str, optional - Indicates how data are represented in cells. - By default, they are represented as floats with 3 decimal points. + parent : QWidget, optional + Parent Widget. readonly : bool, optional If True, data cannot be changed. False by default. + format : str, optional + Indicates how data is represented in cells. + By default, they are represented as floats with 3 decimal points. font : QFont, optional Font. Default is `Calibri` with size 11. - parent : QWidget, optional - Parent Widget. bg_gradient : LinearGradient, optional Background color gradient bg_value : Numpy ndarray, optional Background color value. Must have the shape as data - minvalue : scalar + minvalue : scalar, optional Minimum value allowed. - maxvalue : scalar + maxvalue : scalar, optional Maximum value allowed. """ ROWS_TO_LOAD = 500 COLS_TO_LOAD = 40 - def __init__(self, parent=None, data=None, readonly=False, format="%.3f", font=None, - bg_gradient=None, bg_value=None, minvalue=None, maxvalue=None): - AbstractArrayModel.__init__(self, parent, data, readonly, font) + def __init__(self, parent=None, readonly=False, format="%.3f", font=None, minvalue=None, maxvalue=None): + AbstractArrayModel.__init__(self, parent, readonly, font) self._format = format self.minvalue = minvalue self.maxvalue = maxvalue - self._set_data(data) - self._set_bg_gradient(bg_gradient) - self._set_bg_value(bg_value) - # XXX: unsure this is necessary at all in __init__ - self.reset() + + self.changes = None + self.color_func = None + + self.vmin = None + self.vmax = None + self.bgcolor_possible = False + + self.bg_value = None + self.bg_gradient = None def get_format(self): """Return current format""" @@ -234,16 +229,12 @@ def get_data(self): """Return data""" return self._data - def _set_data(self, data, changes=None, reset_minmax=True): - if changes is None: - changes = {} + def _set_changes(self, changes): self.changes = changes + def _set_data(self, data): # TODO: check that data respects minvalue/maxvalue - if data is None: - data = np.empty((0, 0), dtype=np.int8) - if not (isinstance(data, np.ndarray) and data.ndim == 2): - QMessageBox.critical(self.dialog, "Error", "Expect Numpy ndarray of 2 dimensions") + assert isinstance(data, np.ndarray) and data.ndim == 2 self._data = data dtype = data.dtype @@ -262,8 +253,6 @@ def _set_data(self, data, changes=None, reset_minmax=True): self.color_func = None # -------------------------------------- self.total_rows, self.total_cols = self._data.shape - if reset_minmax: - self.reset_minmax() self._compute_rows_cols_loaded() def reset_minmax(self): @@ -283,9 +272,12 @@ def reset_minmax(self): def set_format(self, format): """Change display format""" - self._format = format + self._set_format(format) self.reset() + def _set_format(self, format): + self._format = format + def set_bg_gradient(self, bg_gradient): self._set_bg_gradient(bg_gradient) self.reset() @@ -459,17 +451,22 @@ def set_values(self, left, top, right, bottom, values): # Update vmin/vmax if necessary if self.vmin is not None and self.vmax is not None: + # FIXME: -inf/+inf and non-number values should be ignored here too colorval = self.color_func(values) if self.color_func is not None else values old_colorval = self.color_func(oldvalues) if self.color_func is not None else oldvalues + # we need to lower vmax or increase vmin if np.any(((old_colorval == self.vmax) & (colorval < self.vmax)) | ((old_colorval == self.vmin) & (colorval > self.vmin))): self.reset_minmax() + self.reset() # this is faster, when the condition is False (which should be most of the cases) than computing # subset_max and checking if subset_max > self.vmax if np.any(colorval > self.vmax): self.vmax = float(np.nanmax(colorval)) + self.reset() if np.any(colorval < self.vmin): self.vmin = float(np.nanmin(colorval)) + self.reset() top_left = self.index(left, top) # -1 because Qt index end bounds are inclusive diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index 3ff8e776..a2a02849 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -93,10 +93,12 @@ from larray_editor.combo import FilterComboBox, FilterMenu import larray as la + # XXX: define Enum instead ? TOP, BOTTOM = 0, 1 LEFT, RIGHT = 0, 1 + class LabelsView(QTableView): """"Labels view class""" @@ -208,6 +210,7 @@ def __init__(self, dtype, parent=None, font=None, def createEditor(self, parent, option, index): """Create editor widget""" model = index.model() + # TODO: dtype should be taken from the model instead (or even from the actual value?) value = model.get_value(index) if self.dtype.name == "bool": # toggle value @@ -271,20 +274,16 @@ class DataView(QTableView): signal_paste = Signal() signal_plot = Signal() - def __init__(self, parent, model, dtype, shape): + def __init__(self, parent, model): QTableView.__init__(self, parent) # set model if not isinstance(model, DataArrayModel): raise TypeError("Expected model of type {}. Received {} instead" .format(DataArrayModel.__name__, type(model).__name__)) self.setModel(model) - # set array delegate - delegate = ArrayDelegate(dtype, self, minvalue=model.minvalue, maxvalue=model.maxvalue) - self.setItemDelegate(delegate) self.setSelectionMode(QTableView.ContiguousSelection) - self.shape = shape self.context_menu = self.setup_context_menu() # TODO: find a cleaner way to do this @@ -320,6 +319,11 @@ def __init__(self, parent, model, dtype, shape): # self.horizontalHeader().sectionClicked.connect(self.on_horizontal_header_clicked) + def set_dtype(self, dtype): + model = self.model() + delegate = ArrayDelegate(dtype, self, minvalue=model.minvalue, maxvalue=model.maxvalue) + self.setItemDelegate(delegate) + def set_default_size(self): # make the grid a bit more compact self.horizontalHeader().setDefaultSectionSize(64) @@ -543,11 +547,10 @@ def __init__(self, parent, data=None, readonly=False, bg_value=None, bg_gradient self.view_ylabels = LabelsView(parent=self, model=self.model_ylabels, position=(BOTTOM, LEFT)) self.model_data = DataArrayModel(parent=self, readonly=readonly, minvalue=minvalue, maxvalue=maxvalue) - self.view_data = DataView(parent=self, model=self.model_data, dtype=data.dtype, shape=data.shape) + self.view_data = DataView(parent=self, model=self.model_data) self.data_adapter = LArrayDataAdapter(axes_model=self.model_axes, xlabels_model=self.model_xlabels, - ylabels_model=self.model_ylabels, data_model=self.model_data, data=data, - bg_value=bg_value, bg_gradient=bg_gradient) + ylabels_model=self.model_ylabels, data_model=self.model_data) # Create vertical and horizontal scrollbars self.vscrollbar = ScrollBar(self, self.view_data.verticalScrollBar()) @@ -628,11 +631,13 @@ def __init__(self, parent, data=None, readonly=False, bg_value=None, bg_gradient spin.valueChanged.connect(self.digits_changed) self.digits_spinbox = spin self.btn_layout.addWidget(spin) + self.digits = 0 scientific = QCheckBox(_('Scientific')) scientific.stateChanged.connect(self.scientific_changed) self.scientific_checkbox = scientific self.btn_layout.addWidget(scientific) + self.use_scientific = False gradient_chooser = QComboBox() gradient_chooser.setMaximumSize(120, 20) @@ -669,8 +674,10 @@ def __init__(self, parent, data=None, readonly=False, bg_value=None, bg_gradient layout.addLayout(self.btn_layout) layout.setContentsMargins(0, 0, 0, 0) self.setLayout(layout) - self.set_data(data, bg_value=bg_value) + self.model_data.set_bg_gradient(gradient_map[bg_gradient]) + if data is not None: + self.set_data(data, bg_value=bg_value) # See http://doc.qt.io/qt-4.8/qt-draganddrop-fridgemagnets-dragwidget-cpp.html for an example self.setAcceptDrops(True) @@ -753,12 +760,12 @@ def dropEvent(self, event): else: event.ignore() - def set_data(self, data=None, bg_value=None): - # update adapter - if data is None: - data = la.LArray([]) - else: - data = la.aslarray(data) + def set_data(self, data, bg_value=None): + # TODO: in the future, data should either be an adapter directly or we should instantiate one here depending + # on the type of data it received. Having a single adapter instance and using set_data on it like we do now + # cannot work because we will need a different adapter class for different data types. + data = la.aslarray(data) + axes = data.axes display_names = axes.display_names @@ -781,9 +788,8 @@ def set_data(self, data=None, bg_value=None): filters_layout.addWidget(QLabel("too big to be filtered")) filters_layout.addStretch() - self.gradient_chooser.setEnabled(self.model_data.bgcolor_possible) - self.data_adapter.set_data(data, bg_value=bg_value) + self.gradient_chooser.setEnabled(self.model_data.bgcolor_possible) # reset default size self.view_axes.set_default_size() @@ -791,6 +797,8 @@ def set_data(self, data=None, bg_value=None): self.view_xlabels.set_default_size() self.view_data.set_default_size() + self.view_data.set_dtype(data.dtype) + # called by set_data and ArrayEditorWidget.accept_changes (this should not be the case IMO) # two cases: # * set_data should update both scientific and ndigits @@ -844,23 +852,24 @@ def _update_digits_scientific(self, data, scientific=None): ndecimals = data_frac_digits self.digits = ndecimals - self.use_scientific = scientific + # avoid triggering digits_changed which would cause a useless redraw self.digits_spinbox.blockSignals(True) self.digits_spinbox.setValue(ndecimals) self.digits_spinbox.setEnabled(is_number(dtype)) self.digits_spinbox.blockSignals(False) + # avoid triggering scientific_changed which would call this function a second time self.scientific_checkbox.blockSignals(True) self.scientific_checkbox.setChecked(scientific) self.scientific_checkbox.setEnabled(is_number(dtype)) self.scientific_checkbox.blockSignals(False) # setting the format explicitly instead of relying on digits_spinbox.digits_changed to set it because - # digits_changed is only triggered when digits actually changed, not when passing from scientific -> non - # scientific or number -> object - self.model_data.set_format(self.cell_format) + # digits_changed is only triggered when digits actually changed, not when passing from + # scientific -> non scientific or number -> object + self.set_format(data, ndecimals, scientific) def _get_sample(self, data): assert isinstance(data, la.LArray) @@ -963,21 +972,27 @@ def reject_changes(self): """Reject changes""" self.data_adapter.reject_changes() - @property - def cell_format(self): - type = self.data_adapter.dtype.type - if type in (np.str, np.str_, np.bool_, np.bool, np.object_): - return '%s' - else: - format_letter = 'e' if self.use_scientific else 'f' - return '%%.%d%s' % (self.digits, format_letter) - def scientific_changed(self, value): - self._update_digits_scientific(self.data_adapter.get_data(), value) + self._update_digits_scientific(self.data_adapter.get_data(), scientific=value) + self.model_data.reset() def digits_changed(self, value): self.digits = value - self.model_data.set_format(self.cell_format) + self.set_format(self.data_adapter, value, self.use_scientific) + self.model_data.reset() + + def set_format(self, data, digits, scientific): + """data: object with a dtype attribute""" + type = data.dtype.type + if type in (np.str, np.str_, np.bool_, np.bool, np.object_): + fmt = '%s' + else: + # XXX: use self.digits_spinbox.getValue() and instead? + # XXX: use self.digits_spinbox.getValue() instead? + format_letter = 'e' if scientific else 'f' + fmt = '%%.%d%s' % (digits, format_letter) + # this does not call model_data.reset() so it should be called by the caller + self.model_data._set_format(fmt) def create_filter_combo(self, axis): def filter_changed(checked_items): diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 5f8e3435..53f27220 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -3,7 +3,7 @@ import matplotlib import numpy as np -from larray import LArray, Session, zeros +from larray import LArray, Session, zeros, empty from larray_editor.utils import (PY2, PYQT5, _, create_action, show_figure, ima, commonpath, dependencies, get_versions, urls) from larray_editor.arraywidget import ArrayEditorWidget @@ -116,7 +116,8 @@ def setup_and_check(self, data, title='', readonly=False, minvalue=None, maxvalu del_item_shortcut.activated.connect(self.delete_current_item) self.data = Session() - self.arraywidget = ArrayEditorWidget(self, zeros(0), readonly) + self.arraywidget = ArrayEditorWidget(self, readonly=readonly) + self.arraywidget.model_data.dataChanged.connect(self.data_changed) if qtconsole_available: # Create an in-process kernel @@ -226,10 +227,6 @@ def void_formatter(array, *args, **kwargs): arrays = [k for k, v in self.data.items() if self._display_in_grid(k, v)] self.add_list_items(arrays) self._listwidget.setCurrentRow(0) - - # tracking data changes - self.arraywidget.model_data.dataChanged.connect(self.data_changed) - return True def _reset(self): @@ -456,7 +453,7 @@ def ipython_cell_executed(self): if isinstance(cur_output, matplotlib.axes.Subplot) and 'inline' not in matplotlib.get_backend(): show_figure(self, cur_output.figure) - def on_selection_changed(self, *args, **kwargs): + def on_selection_changed(self): selected = self._listwidget.selectedItems() if selected: assert len(selected) == 1 @@ -559,7 +556,7 @@ def _ask_to_save_if_unsaved_modifications(self): def new(self): if self._ask_to_save_if_unsaved_modifications(): self._reset() - self.arraywidget.set_data() + self.arraywidget.set_data(empty(0)) self.set_current_file(None) self.unsaved_modifications = False self.statusBar().showMessage("Viewer has been reset", 4000) From eea054d903fcfb88ef586f05e62c4eefdf644aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=83=C2=ABtan=20de=20Menten?= Date: Thu, 9 Nov 2017 13:36:45 +0100 Subject: [PATCH 15/18] made switching to a big array (much) faster by not computing its min/max on the whole array but on a sample (fixes #93) This means we can miss the actual min/max of the displayed part, so it complexifies the code. Most of this awful code will go away after : * we invert how changes work (store old values instead of new values) * we get decent buffering (in that case the min/max should only be updated when "moving" the buffer. --- larray_editor/arraymodel.py | 56 ++++++++++++++++++++++++++++-------- larray_editor/arraywidget.py | 1 + larray_editor/utils.py | 40 ++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/larray_editor/arraymodel.py b/larray_editor/arraymodel.py index abd2eb3f..7ce56940 100644 --- a/larray_editor/arraymodel.py +++ b/larray_editor/arraymodel.py @@ -3,7 +3,7 @@ import numpy as np from larray_editor.utils import (get_font, from_qvariant, to_qvariant, to_text_string, is_float, is_number, LinearGradient, SUPPORTED_FORMATS, scale_to_01range, - Product) + Product, is_number_value, get_sample, get_sample_indices) from qtpy.QtCore import Qt, QModelIndex, QAbstractTableModel from qtpy.QtGui import QColor from qtpy.QtWidgets import QMessageBox @@ -256,8 +256,8 @@ def _set_data(self, data): self._compute_rows_cols_loaded() def reset_minmax(self): - data = self.get_values() try: + data = self.get_values(sample=True) color_value = self.color_func(data) if self.color_func is not None else data # ignore nan, -inf, inf (setting them to 0 or to very large numbers is not an option) color_value = color_value[np.isfinite(color_value)] @@ -336,8 +336,21 @@ def data(self, index, role=Qt.DisplayRole): elif role == Qt.BackgroundColorRole: if self.bgcolor_possible and self.bg_gradient is not None and value is not np.ma.masked: if self.bg_value is None: - v = float(self.color_func(value) if self.color_func is not None else value) - v = scale_to_01range(v, self.vmin, self.vmax) + try: + v = self.color_func(value) if self.color_func is not None else value + if -np.inf < v < self.vmin: + # TODO: this is suboptimal, as it can reset many times (though in practice, it is usually + # ok). When we get buffering, we will need to compute vmin/vmax on the whole buffer + # at once, eliminating this problem (and we could even compute final colors directly + # all at once) + self.vmin = v + self.reset() + elif self.vmax < v < np.inf: + self.vmax = v + self.reset() + v = scale_to_01range(v, self.vmin, self.vmax) + except TypeError: + v = np.nan else: i, j = index.row(), index.column() v = self.bg_value[i, j] @@ -346,26 +359,45 @@ def data(self, index, role=Qt.DisplayRole): # return to_qvariant("{}\n{}".format(repr(value),self.get_labels(index))) return to_qvariant() - def get_values(self, left=0, top=0, right=None, bottom=None): + def get_values(self, left=0, top=0, right=None, bottom=None, sample=False): width, height = self.total_rows, self.total_cols if right is None: right = width if bottom is None: bottom = height - values = self._data[left:right, top:bottom].copy() - # both versions get the same result, but depending on inputs, the - # speed difference can be large. + # this whole bullshit will disappear when we implement undo/redo + values = self._data[left:right, top:bottom] + # both versions get the same result, but depending on inputs, the speed difference can be large. if values.size < len(self.changes): + # changes are supposedly relatively small so this case should not be too slow even if we just want a sample + values = values.copy() for i in range(left, right): for j in range(top, bottom): pos = i, j if pos in self.changes: values[i - left, j - top] = self.changes[pos] + if sample: + return get_sample(values, 500) + else: + return values else: - for (i, j), value in self.changes.items(): - if left <= i < right and top <= j < bottom: - values[i - left, j - top] = value - return values + if sample: + sample_indices = get_sample_indices(values, 500) + changes = self.changes + + def get_val(idx): + i, j = idx + changes_idx = (i + left, j + top) + return changes[changes_idx] if changes_idx in changes else values[idx] + + # we need to keep the dtype, otherwise numpy might convert mixed object arrays to strings + return np.array([get_val(idx) for idx in zip(*sample_indices)], dtype=values.dtype) + else: + values = values.copy() + for (i, j), value in self.changes.items(): + if left <= i < right and top <= j < bottom: + values[i - left, j - top] = value + return values def convert_value(self, value): """ diff --git a/larray_editor/arraywidget.py b/larray_editor/arraywidget.py index a2a02849..59728495 100644 --- a/larray_editor/arraywidget.py +++ b/larray_editor/arraywidget.py @@ -874,6 +874,7 @@ def _update_digits_scientific(self, data, scientific=None): def _get_sample(self, data): assert isinstance(data, la.LArray) data = data.data + # TODO: use utils.get_sample instead size = data.size # this will yield a data sample of max 199 step = (size // 100) if size > 100 else 1 diff --git a/larray_editor/utils.py b/larray_editor/utils.py index 24cb352a..1518fa7d 100644 --- a/larray_editor/utils.py +++ b/larray_editor/utils.py @@ -2,6 +2,8 @@ import os import sys +import math + import numpy as np from qtpy import PYQT5 @@ -547,3 +549,41 @@ def scale_to_01range(value, vmin, vmax): else: assert vmin < vmax return (value - vmin) / (vmax - vmin) + + +is_number_value = np.vectorize(lambda x: isinstance(x, (int, float, np.number))) + + +def get_sample_step(data, maxsize): + size = data.size + if not size: + return None + return math.ceil(size / maxsize) + + +def get_sample(data, maxsize): + """return sample. View in all cases. + + if data.size < maxsize: + sample_size == data.size + else: + (maxsize // 2) < sample_size <= maxsize + + Parameters + ---------- + data + maxsize + + Returns + ------- + view + """ + size = data.size + if not size: + return data + return data.flat[::get_sample_step(data, maxsize)] + + +def get_sample_indices(data, maxsize): + flat_indices = np.arange(0, data.size, get_sample_step(data, maxsize)) + return np.unravel_index(flat_indices, data.shape) From 77ede674730d5e13ef2d295323bd3d68d894f28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Fri, 10 Nov 2017 11:36:06 +0100 Subject: [PATCH 16/18] cosmetic improvements to about dialog --- larray_editor/editor.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index 53f27220..c5705e78 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -721,10 +721,12 @@ def about(self): kwargs = get_versions() kwargs.update(urls) message = """\ -LArray Editor {editor}: The Graphical User Interface for LArray. -
Licensed under the terms of the GNU GENERAL PUBLIC LICENSE Version 3. -

Developed and maintained by the Federal Planning Bureau (Belgium). -

Versions of underlying libraries: +

LArray Editor {editor} +
The Graphical User Interface for LArray +

Licensed under the terms of the GNU General Public License Version 3. +

Developed and maintained by the Federal Planning Bureau (Belgium). +

  +

Versions of underlying libraries

  • Python {python} on {system} {bitness:d}bits
  • Qt {qt}, {qt_api} {qt_api_ver}
    • @@ -733,7 +735,7 @@ def about(self): if kwargs[dep] != 'N/A': message += "
  • {dep} {{{dep}}}
    • \n".format(dep=dep) message += "
" - QMessageBox.about(self, _("About Larray Editor"), message.format(**kwargs)) + QMessageBox.about(self, _("About LArray Editor"), message.format(**kwargs)) def set_current_file(self, filepath): self.update_recent_files([filepath]) From 52eaf2bba5fa5a14488e2051e7cd5471bbf53792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Fri, 10 Nov 2017 11:48:07 +0100 Subject: [PATCH 17/18] removed useless line (view, edit and compare are also in larray) --- larray_editor/editor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/larray_editor/editor.py b/larray_editor/editor.py index c5705e78..de4d3a7b 100644 --- a/larray_editor/editor.py +++ b/larray_editor/editor.py @@ -125,8 +125,8 @@ def setup_and_check(self, data, title='', readonly=False, minvalue=None, maxvalu kernel_manager.start_kernel(show_banner=False) kernel = kernel_manager.kernel + # TODO: use self._reset() instead kernel.shell.run_cell('from larray import *') - kernel.shell.run_cell('from larray_editor import *') text_formatter = kernel.shell.display_formatter.formatters['text/plain'] def void_formatter(array, *args, **kwargs): From febd9f558d163b29378e7b5a5d54f4ebb772ef4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABtan=20de=20Menten?= Date: Fri, 10 Nov 2017 11:49:53 +0100 Subject: [PATCH 18/18] added support for coloring object arrays (only use numeric values) --- larray_editor/arraymodel.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/larray_editor/arraymodel.py b/larray_editor/arraymodel.py index 7ce56940..f6dbb1a4 100644 --- a/larray_editor/arraymodel.py +++ b/larray_editor/arraymodel.py @@ -259,6 +259,11 @@ def reset_minmax(self): try: data = self.get_values(sample=True) color_value = self.color_func(data) if self.color_func is not None else data + if color_value.dtype.type == np.object_: + color_value = color_value[is_number_value(color_value)] + # this is probably broken if we have complex numbers stored as objects but I don't foresee + # this case happening anytime soon. + color_value = color_value.astype(float) # ignore nan, -inf, inf (setting them to 0 or to very large numbers is not an option) color_value = color_value[np.isfinite(color_value)] self.vmin = float(np.min(color_value))