From d916e1c9a6d2370794ff1143991e56bd848cefd1 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 1 Oct 2025 16:43:52 -0500 Subject: [PATCH 01/12] init solution for converting localization_dump field lables to a list of tuples rather than a dictionary --- .../backend/stored_queries/batch_edit.py | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index ce073f28b42..376f5449bb5 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -887,7 +887,8 @@ def _flatten(_: str, _self: "RowPlanCanonical"): def to_upload_plan( self, base_table: Table, - localization_dump: dict[str, dict[str, str]], + # localization_dump: dict[str, dict[str, str]], + localization_dump: dict[str, list[tuple[str, str, bool]]], query_fields: list[QueryField], fields_added: dict[str, int], get_column_id: Callable[[str], int], @@ -909,13 +910,30 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): _id - 1 ] # Need to go off by 1, bc we added 1 to account for id fields table_name, field_name = _get_table_and_field(field) - field_labels = localization_dump.get(table_name, {}) + field_labels = localization_dump.get(table_name, []) # It could happen that the field we saw doesn't exist. # Plus, the default options get chosen in the cases of - if field_name not in field_labels or field.fieldspec.contains_tree_rank(): + if field_name not in [fl[0] for fl in field_labels] or field.fieldspec.contains_tree_rank(): localized_label = naive_field_format(field.fieldspec) else: - localized_label = field_labels[field_name] + localized_label = None + # localized_label_idx = None + # localized_label_idx = 0 + localized_label_idx = len(field_labels) - 1 + for i, fl in enumerate(field_labels): + if fl[0] == field_name and not fl[2]: + localized_label = fl[1] + # set the localized_label that was just retrieved from field_lables to a tuple ending in True to mark it as used + # field_labels[i] = (fl[0], fl[1], True) + localized_label_idx = i + break + + field_labels[localized_label_idx] = (field_labels[localized_label_idx][0], field_labels[localized_label_idx][1], True) + localization_dump[table_name] = field_labels + + if localized_label is None: + localized_label = field_labels[localized_label_idx][1] + string_id = field.fieldspec.to_stringid() fields_added[localized_label] = fields_added.get(localized_label, 0) + 1 _count = fields_added[localized_label] @@ -1159,11 +1177,29 @@ def run_batch_edit_query(props: BatchEditProps): ), "Got misaligned captions!" localization_dump = {} + # if captions: + # for (field, caption) in zip(visible_fields, captions): + # table_name, field_name = _get_table_and_field(field) + # field_labels = localization_dump.get(table_name, {}) + # field_labels[field_name] = caption + # if captions: + # for (field, caption) in zip(visible_fields, captions): + # table_name, field_name = _get_table_and_field(field) + # field_labels = localization_dump.get(table_name, {}) + # while field_name in field_labels.keys(): + # field_name += "_" + # field_labels[field_name] = caption + # if table_name not in localization_dump.keys(): + # localization_dump[table_name] = field_labels + # else: + # existing_field_labels = localization_dump[table_name] + # localization_dump[table_name] = {**existing_field_labels, **field_labels} if captions: for (field, caption) in zip(visible_fields, captions): table_name, field_name = _get_table_and_field(field) - field_labels = localization_dump.get(table_name, {}) - field_labels[field_name] = caption + field_labels = localization_dump.get(table_name, []) + new_field_label = (field_name, caption, False) + field_labels.append(new_field_label) localization_dump[table_name] = field_labels naive_row_plan = RowPlanMap.get_row_plan(visible_fields) From c2c747805074aa13fa8fa9cc4ea408a037e1ae79 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 3 Oct 2025 15:35:08 -0500 Subject: [PATCH 02/12] solution with BatchEditMetaTables --- .../backend/stored_queries/batch_edit.py | 123 ++++++++++++------ 1 file changed, 81 insertions(+), 42 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 376f5449bb5..189aeb8a203 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -606,6 +606,71 @@ def rewrite( # RowPlanMap is just a map, this stores actual data (to many is a dict of list, rather than just a dict) # maybe unify that with RowPlanMap? +class FieldLabel(NamedTuple): + field_name: str + caption: str + is_used: bool = False + +class TableFieldLabels: + table_name: str + field_labels: list[FieldLabel] + + def __init__(self, table_name: str): + self.table_name = table_name + self.field_labels = [] + + def __iter__(self): + return iter(self.field_labels) + + def add_field_label(self, field_label: FieldLabel): + self.field_labels.append(field_label) + + def get_field_label_names(self) -> list[str]: + return [field.field_name for field in self.field_labels] + + def has_field_label(self, field_name: str) -> bool: + return any(field.field_name == field_name for field in self.field_labels) + + def use_field_label(self, field_name: str) -> FieldLabel | None: + last_match_idx: int | None = None + for idx, field_label in enumerate(self.field_labels): + if field_label.field_name != field_name: + continue + last_match_idx = idx + if not field_label.is_used: + updated = field_label._replace(is_used=True) + self.field_labels[idx] = updated + return updated + if last_match_idx is not None: + updated = self.field_labels[last_match_idx]._replace(is_used=True) + self.field_labels[last_match_idx] = updated + return updated + return None + +class BatchEditMetaTables: + table_labels: dict[str, TableFieldLabels] = {} + + def __init__(self): + self.table_labels = {} + + def __init__(self, localization_dump: dict[str, list[tuple[str, str, bool]]]): + self.table_labels = {} + for table_name, field_labels in localization_dump.items(): + table_field_labels = TableFieldLabels(table_name) + for field_label in field_labels: + table_field_labels.add_field_label(FieldLabel(field_label[0], field_label[1], field_label[2])) + self.table_labels[table_name] = table_field_labels + + def add_field_label(self, table_name: str, field_label: FieldLabel): + if table_name not in self.table_labels: + self.table_labels[table_name] = TableFieldLabels(table_name) + self.table_labels[table_name].add_field_label(field_label) + + def get_table_field_labels(self, table_name: str) -> TableFieldLabels | None: + return self.table_labels.get(table_name, None) + + def get_all_table_names(self) -> list[str]: + return list(self.table_labels.keys()) class RowPlanCanonical(NamedTuple): batch_edit_pack: BatchEditPack @@ -887,8 +952,7 @@ def _flatten(_: str, _self: "RowPlanCanonical"): def to_upload_plan( self, base_table: Table, - # localization_dump: dict[str, dict[str, str]], - localization_dump: dict[str, list[tuple[str, str, bool]]], + batch_edit_meta_tables: BatchEditMetaTables, query_fields: list[QueryField], fields_added: dict[str, int], get_column_id: Callable[[str], int], @@ -909,30 +973,21 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): field = query_fields[ _id - 1 ] # Need to go off by 1, bc we added 1 to account for id fields - table_name, field_name = _get_table_and_field(field) - field_labels = localization_dump.get(table_name, []) # It could happen that the field we saw doesn't exist. # Plus, the default options get chosen in the cases of - if field_name not in [fl[0] for fl in field_labels] or field.fieldspec.contains_tree_rank(): + table_name, field_name = _get_table_and_field(field) + table_field_labels = batch_edit_meta_tables.get_table_field_labels(table_name) + if ( + table_field_labels is None + or not table_field_labels.has_field_label(field_name) + or field.fieldspec.contains_tree_rank() + ): localized_label = naive_field_format(field.fieldspec) else: - localized_label = None - # localized_label_idx = None - # localized_label_idx = 0 - localized_label_idx = len(field_labels) - 1 - for i, fl in enumerate(field_labels): - if fl[0] == field_name and not fl[2]: - localized_label = fl[1] - # set the localized_label that was just retrieved from field_lables to a tuple ending in True to mark it as used - # field_labels[i] = (fl[0], fl[1], True) - localized_label_idx = i - break - - field_labels[localized_label_idx] = (field_labels[localized_label_idx][0], field_labels[localized_label_idx][1], True) - localization_dump[table_name] = field_labels - - if localized_label is None: - localized_label = field_labels[localized_label_idx][1] + field_label = table_field_labels.use_field_label(field_name) + localized_label = ( + field_label.caption if field_label is not None else naive_field_format(field.fieldspec) + ) string_id = field.fieldspec.to_stringid() fields_added[localized_label] = fields_added.get(localized_label, 0) + 1 @@ -976,7 +1031,7 @@ def _to_upload_plan(rel_name: str, _self: "RowPlanCanonical"): ) return _self.to_upload_plan( related_model, - localization_dump, + batch_edit_meta_tables, query_fields, fields_added, get_column_id, @@ -1177,23 +1232,6 @@ def run_batch_edit_query(props: BatchEditProps): ), "Got misaligned captions!" localization_dump = {} - # if captions: - # for (field, caption) in zip(visible_fields, captions): - # table_name, field_name = _get_table_and_field(field) - # field_labels = localization_dump.get(table_name, {}) - # field_labels[field_name] = caption - # if captions: - # for (field, caption) in zip(visible_fields, captions): - # table_name, field_name = _get_table_and_field(field) - # field_labels = localization_dump.get(table_name, {}) - # while field_name in field_labels.keys(): - # field_name += "_" - # field_labels[field_name] = caption - # if table_name not in localization_dump.keys(): - # localization_dump[table_name] = field_labels - # else: - # existing_field_labels = localization_dump[table_name] - # localization_dump[table_name] = {**existing_field_labels, **field_labels} if captions: for (field, caption) in zip(visible_fields, captions): table_name, field_name = _get_table_and_field(field) @@ -1202,6 +1240,7 @@ def run_batch_edit_query(props: BatchEditProps): field_labels.append(new_field_label) localization_dump[table_name] = field_labels + batch_edit_meta_tables = BatchEditMetaTables(localization_dump) naive_row_plan = RowPlanMap.get_row_plan(visible_fields) all_tree_info = get_all_tree_information(props["collection"], props["user"].id) base_table = datamodel.get_table_by_id_strict(tableid, strict=True) @@ -1295,7 +1334,7 @@ def _get_orig_column(string_id: str): # The keys are lookups into original query field (not modified by us). Used to get ids in the original one. key_and_headers, upload_plan = extend_row.to_upload_plan( base_table, - localization_dump, + batch_edit_meta_tables, query_fields, {}, _get_orig_column, @@ -1366,4 +1405,4 @@ def filter_tree_info(filters: dict[str, list[int]], all_tree_info: dict[str, lis tree_filter = set(filters[tablename]) all_tree_info[treetable_key] = list(filter(lambda tree_info : tree_info['definition']['id'] in tree_filter, all_tree_info[treetable_key])) - return all_tree_info \ No newline at end of file + return all_tree_info From 03e2b3b72038436eb498963c844aa5479a0209db Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 10:12:29 -0500 Subject: [PATCH 03/12] reorganize batch edit helper classes --- .../backend/stored_queries/batch_edit.py | 303 +---------------- .../batch_edit_helper_classes.py | 316 ++++++++++++++++++ 2 files changed, 332 insertions(+), 287 deletions(-) create mode 100644 specifyweb/backend/stored_queries/batch_edit_helper_classes.py diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 189aeb8a203..6704a9d9d0a 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -1,5 +1,3 @@ - - # type: ignore # ^^ The above is because we etensively use recursive typedefs of named tuple in this file not supported on our MyPy 0.97 version. @@ -15,9 +13,22 @@ from collections.abc import Callable from specifyweb.backend.permissions.permissions import has_target_permission -from specifyweb.specify.api.filter_by_col import CONCRETE_HIERARCHY +from specifyweb.backend.stored_queries.batch_edit_helper_classes import ( + BATCH_EDIT_NULL_RECORD_DESCRIPTION, + BATCH_EDIT_READONLY_TABLES, + BATCH_EDIT_SHARED_READONLY_FIELDS, + BATCH_EDIT_SHARED_READONLY_RELATIONSHIPS, + EMPTY_FIELD, + EMPTY_PACK, + FLOAT_FIELDS, + BatchEditFieldPack, + BatchEditMetaTables, + BatchEditPack, + BatchEditProps, + MaybeField +) from specifyweb.specify.models import datamodel -from specifyweb.specify.models_utils.load_datamodel import Field, Relationship, Table +from specifyweb.specify.models_utils.load_datamodel import Relationship, Table from specifyweb.backend.trees.views import TREE_INFORMATION, get_all_tree_information from specifyweb.backend.trees.utils import SPECIFY_TREES from specifyweb.specify.datamodel import is_tree_table @@ -30,7 +41,7 @@ ) from specifyweb.backend.workbench.models import Spdataset from specifyweb.backend.workbench.permissions import BatchEditDataSetPT -from specifyweb.backend.workbench.upload.treerecord import TreeRecord, TreeRankRecord, RANK_KEY_DELIMITER +from specifyweb.backend.workbench.upload.treerecord import TreeRecord, TreeRankRecord from specifyweb.backend.workbench.upload.upload_plan_schema import parse_column_options from specifyweb.backend.workbench.upload.upload_table import UploadTable from specifyweb.backend.workbench.upload.uploadable import NULL_RECORD, Uploadable @@ -45,37 +56,6 @@ from django.db import transaction -MaybeField = Callable[[QueryFieldSpec], Field | None] - -# TODO: -# Investigate if any/some/most of the logic for making an upload plan could be moved to frontend and reused. -# - does generation of upload plan in the backend bc upload plan is not known (we don't know count of to-many). -# - seemed complicated to merge upload plan from the frontend -# - need to place id markers at correct level, so need to follow upload plan anyways. -# REFACTOR: Break this file into smaller pieaces - -# TODO: Play-around with localizing -BATCH_EDIT_NULL_RECORD_DESCRIPTION = "" - -# TODO: add backend support for making system tables readonly -BATCH_EDIT_READONLY_TABLES = [*CONCRETE_HIERARCHY] - -BATCH_EDIT_SHARED_READONLY_FIELDS = [ - "timestampcreated", - "timestampmodified", - "version", - "nodenumber", - "highestchildnodenumber", - "rankid", - "fullname", - "age", -] - -BATCH_EDIT_SHARED_READONLY_RELATIONSHIPS = ["createdbyagent", "modifiedbyagent"] - -BATCH_EDIT_REQUIRED_TREE_FIELDS = ["name"] - - def get_readonly_fields(table: Table): fields = [*BATCH_EDIT_SHARED_READONLY_FIELDS, table.idFieldName.lower()] @@ -100,10 +80,6 @@ def get_readonly_fields(table: Table): return fields, [*BATCH_EDIT_SHARED_READONLY_RELATIONSHIPS, *relationships] - -FLOAT_FIELDS = ["java.lang.Float", "java.lang.Double", "java.math.BigDecimal"] - - def parse(value: Any | None, query_field: QueryField) -> Any: field = query_field.fieldspec.get_field() if field is None or value is None: @@ -117,168 +93,6 @@ def parse(value: Any | None, query_field: QueryField) -> Any: 'longitude2')): return float(value) return value - - -def _get_nested_order(field_spec: QueryFieldSpec): - # don't care about ordernumber if it ain't nested - # won't affect logic, just data being saved. - if len(field_spec.join_path) == 0: - return None - return field_spec.table.get_field("ordernumber") - - -batch_edit_fields: dict[str, tuple[MaybeField, int]] = { - # technically, if version updates are correct, this is useless beyond base tables - # and to-manys. TODO: Do just that. remove it. sorts asc. using sort, the optimized - # dataset construction takes place. - "id": (lambda field_spec: field_spec.table.idField, 1), - # version control gets added here. no sort. - "version": (lambda field_spec: field_spec.table.get_field("version"), 0), - # ordernumber. no sort (actually adding a sort here is useless) - "order": (_get_nested_order, 1), -} - - -class BatchEditFieldPack(NamedTuple): - field: QueryField | None = None - idx: int | None = None # default value not there, for type safety - value: Any = None # stricten this? - - -class BatchEditPack(NamedTuple): - id: BatchEditFieldPack - order: BatchEditFieldPack - version: BatchEditFieldPack - - # extends a path to contain the last field + for a defined fields - @staticmethod - def from_field_spec(field_spec: QueryFieldSpec) -> "BatchEditPack": - # don't care about which way. bad things will happen if not sorted. - # not using assert () since it can be optimised out. - if batch_edit_fields["id"][1] == 0 or batch_edit_fields["order"][1] == 0: - raise Exception("the ID field should always be sorted!") - - def extend_callback(sort_type): - def _callback(field): - return BatchEditPack._query_field( - field_spec._replace( - join_path=(*field_spec.join_path, field), date_part=None - ), - sort_type, - ) - - return _callback - - new_field_specs = { - key: BatchEditFieldPack( - idx=None, - field=Func.maybe(callback(field_spec), extend_callback(sort_type)), - value=None, - ) - for key, (callback, sort_type) in batch_edit_fields.items() - } - return BatchEditPack(**new_field_specs) - - def merge(self, other: "BatchEditPack") -> "BatchEditPack": - return BatchEditPack( - id=self.id if self.id.field is not None else other.id, - version=self.version if self.version.field is not None else other.version, - order=self.order if self.order.field is not None else other.order, - ) - - # a basic query field spec to field - @staticmethod - def _query_field(field_spec: QueryFieldSpec, sort_type: int): - return QueryField( - fieldspec=field_spec, - op_num=8, - value=None, - negate=False, - display=True, - format_name=None, - sort_type=sort_type, - strict=False, - ) - - def _index( - self, - start_idx: int, - current: tuple[dict[str, BatchEditFieldPack], list[QueryField]], - next: tuple[int, tuple[str, tuple[MaybeField, int]]], - ): - current_dict, fields = current - field_idx, (field_name, _) = next - value: BatchEditFieldPack = getattr(self, field_name) - new_dict = { - **current_dict, - field_name: value._replace( - field=None, idx=((field_idx + start_idx) if value.field else None) - ), - } - new_fields = fields if value.field is None else [*fields, value.field] - return new_dict, new_fields - - def index_plan(self, start_index=0) -> tuple["BatchEditPack", list[QueryField]]: - init: tuple[dict[str, BatchEditFieldPack], list[QueryField]] = ( - {}, - [], - ) - _dict, fields = reduce( - lambda accum, next: self._index( - start_idx=start_index, current=accum, next=next - ), - enumerate(batch_edit_fields.items()), - init, - ) - return BatchEditPack(**_dict), fields - - def bind(self, row: tuple[Any]): - return BatchEditPack( - id=self.id._replace( - value=row[self.id.idx] if self.id.idx is not None else None, - ), - order=self.order._replace( - value=row[self.order.idx] if self.order.idx is not None else None - ), - version=self.version._replace( - value=row[self.version.idx] if self.version.idx is not None else None - ), - ) - - def to_json(self) -> dict[str, Any]: - return { - "id": self.id.value, - "ordernumber": self.order.value, - "version": self.version.value, - } - - # we not only care that it is part of tree, but also care that there is rank to tree - def is_part_of_tree(self, query_fields: list[QueryField]) -> bool: - if self.id.idx is None: - return False - id_field = self.id.idx - field = query_fields[id_field - 1] - join_path = field.fieldspec.join_path - if len(join_path) < 2: - return False - return isinstance(join_path[-2], TreeRankQuery) - - @staticmethod - def replace_tree_rank(fieldspec: QueryFieldSpec, tree_rank: TreeRankQuery) -> QueryFieldSpec: - return fieldspec._replace( - join_path=tuple( - [ - tree_rank if isinstance(node, TreeRankQuery) else node - for node in fieldspec.join_path - ] - ) - ) - - def readjust_tree_rank(self, tree_rank: TreeRankQuery): - id_field = self.id._replace(field=self.id.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.id.field.fieldspec, tree_rank))) if self.id.field is not None else self.id - order_field = self.order._replace(field=self.order.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.order.field.fieldspec, tree_rank))) if self.order.field is not None else self.order - version_field = self.version._replace(field=self.version.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.version.field.fieldspec, tree_rank))) if self.version.field is not None else self.version - return BatchEditPack(id=id_field, order=order_field, version=version_field) def get_tree_rank_record(key) -> TreeRankRecord: from specifyweb.backend.workbench.upload.treerecord import RANK_KEY_DELIMITER @@ -286,12 +100,6 @@ def get_tree_rank_record(key) -> TreeRankRecord: tree_name, rank_name, tree_def_id = tuple(key.split(RANK_KEY_DELIMITER)) return TreeRankRecord(RANK_KEY_DELIMITER.join([tree_name, rank_name]), int(tree_def_id)) - -# These constants are purely for memory optimization, no code depends and/or cares if this is constant. -EMPTY_FIELD = BatchEditFieldPack() -EMPTY_PACK = BatchEditPack(id=EMPTY_FIELD, order=EMPTY_FIELD, version=EMPTY_FIELD) - - # FUTURE: this already supports nested-to-many for most part # wb plan, but contains query fields along with indexes to look-up in a result row. # TODO: see if it can be moved + combined with front-end logic. I kept all parsing on backend, but there might be possible beneft in doing this @@ -606,72 +414,6 @@ def rewrite( # RowPlanMap is just a map, this stores actual data (to many is a dict of list, rather than just a dict) # maybe unify that with RowPlanMap? -class FieldLabel(NamedTuple): - field_name: str - caption: str - is_used: bool = False - -class TableFieldLabels: - table_name: str - field_labels: list[FieldLabel] - - def __init__(self, table_name: str): - self.table_name = table_name - self.field_labels = [] - - def __iter__(self): - return iter(self.field_labels) - - def add_field_label(self, field_label: FieldLabel): - self.field_labels.append(field_label) - - def get_field_label_names(self) -> list[str]: - return [field.field_name for field in self.field_labels] - - def has_field_label(self, field_name: str) -> bool: - return any(field.field_name == field_name for field in self.field_labels) - - def use_field_label(self, field_name: str) -> FieldLabel | None: - last_match_idx: int | None = None - for idx, field_label in enumerate(self.field_labels): - if field_label.field_name != field_name: - continue - last_match_idx = idx - if not field_label.is_used: - updated = field_label._replace(is_used=True) - self.field_labels[idx] = updated - return updated - if last_match_idx is not None: - updated = self.field_labels[last_match_idx]._replace(is_used=True) - self.field_labels[last_match_idx] = updated - return updated - return None - -class BatchEditMetaTables: - table_labels: dict[str, TableFieldLabels] = {} - - def __init__(self): - self.table_labels = {} - - def __init__(self, localization_dump: dict[str, list[tuple[str, str, bool]]]): - self.table_labels = {} - for table_name, field_labels in localization_dump.items(): - table_field_labels = TableFieldLabels(table_name) - for field_label in field_labels: - table_field_labels.add_field_label(FieldLabel(field_label[0], field_label[1], field_label[2])) - self.table_labels[table_name] = table_field_labels - - def add_field_label(self, table_name: str, field_label: FieldLabel): - if table_name not in self.table_labels: - self.table_labels[table_name] = TableFieldLabels(table_name) - self.table_labels[table_name].add_field_label(field_label) - - def get_table_field_labels(self, table_name: str) -> TableFieldLabels | None: - return self.table_labels.get(table_name, None) - - def get_all_table_names(self) -> list[str]: - return list(self.table_labels.keys()) - class RowPlanCanonical(NamedTuple): batch_edit_pack: BatchEditPack columns: list[BatchEditFieldPack] = [] @@ -1144,19 +886,6 @@ def run_batch_edit(collection, user, spquery, agent): visual_order=visual_order, ) - -class BatchEditProps(TypedDict): - collection: Any - user: Any - contexttableid: int - captions: Any - limit: int | None - recordsetid: int | None - session_maker: Any - fields: list[QueryField] - omit_relationships: bool | None - treedefsfilter: Any - def _get_table_and_field(field: QueryField): table_name = field.fieldspec.table.name field_name = None if field.fieldspec.get_field() is None else field.fieldspec.get_field().name diff --git a/specifyweb/backend/stored_queries/batch_edit_helper_classes.py b/specifyweb/backend/stored_queries/batch_edit_helper_classes.py new file mode 100644 index 00000000000..0879cc46df6 --- /dev/null +++ b/specifyweb/backend/stored_queries/batch_edit_helper_classes.py @@ -0,0 +1,316 @@ +# type: ignore + +# ^^ The above is because we etensively use recursive typedefs of named tuple in this file not supported on our MyPy 0.97 version. +# When typechecked in MyPy 1.11 (supports recursive typedefs), there is no type issue in the file. +# However, using 1.11 makes things slower in other files. + +from functools import reduce +from typing import ( + Any, + NamedTuple, + TypedDict, +) +from collections.abc import Callable + +from specifyweb.backend.permissions.permissions import has_target_permission +from specifyweb.specify.api.filter_by_col import CONCRETE_HIERARCHY +from specifyweb.specify.models import datamodel +from specifyweb.specify.models_utils.load_datamodel import Field, Relationship, Table +from specifyweb.backend.trees.views import TREE_INFORMATION, get_all_tree_information +from specifyweb.backend.trees.utils import SPECIFY_TREES +from specifyweb.specify.datamodel import is_tree_table +from specifyweb.backend.stored_queries.execution import execute +from specifyweb.backend.stored_queries.queryfield import QueryField, fields_from_json +from specifyweb.backend.stored_queries.queryfieldspec import ( + QueryFieldSpec, + QueryNode, + TreeRankQuery, +) +from specifyweb.backend.workbench.models import Spdataset +from specifyweb.backend.workbench.permissions import BatchEditDataSetPT +from specifyweb.backend.workbench.upload.treerecord import TreeRecord, TreeRankRecord, RANK_KEY_DELIMITER +from specifyweb.backend.workbench.upload.upload_plan_schema import parse_column_options +from specifyweb.backend.workbench.upload.upload_table import UploadTable +from specifyweb.backend.workbench.upload.uploadable import NULL_RECORD, Uploadable +from specifyweb.backend.workbench.views import regularize_rows +from specifyweb.specify.utils.func import Func +from . import models +import json +from .format import ObjectFormatterProps + +from specifyweb.backend.workbench.upload.upload_plan_schema import schema +from jsonschema import validate + +from django.db import transaction + +MaybeField = Callable[[QueryFieldSpec], Field | None] + +# TODO: +# Investigate if any/some/most of the logic for making an upload plan could be moved to frontend and reused. +# - does generation of upload plan in the backend bc upload plan is not known (we don't know count of to-many). +# - seemed complicated to merge upload plan from the frontend +# - need to place id markers at correct level, so need to follow upload plan anyways. +# REFACTOR: Break this file into smaller pieaces + +# TODO: Play-around with localizing +BATCH_EDIT_NULL_RECORD_DESCRIPTION = "" + +# TODO: add backend support for making system tables readonly +BATCH_EDIT_READONLY_TABLES = [*CONCRETE_HIERARCHY] + +BATCH_EDIT_SHARED_READONLY_FIELDS = [ + "timestampcreated", + "timestampmodified", + "version", + "nodenumber", + "highestchildnodenumber", + "rankid", + "fullname", + "age", +] + +BATCH_EDIT_SHARED_READONLY_RELATIONSHIPS = ["createdbyagent", "modifiedbyagent"] + +BATCH_EDIT_REQUIRED_TREE_FIELDS = ["name"] + +FLOAT_FIELDS = ["java.lang.Float", "java.lang.Double", "java.math.BigDecimal"] + +class BatchEditFieldPack(NamedTuple): + field: QueryField | None = None + idx: int | None = None # default value not there, for type safety + value: Any = None # stricten this? + +class BatchEditPack(NamedTuple): + id: BatchEditFieldPack + order: BatchEditFieldPack + version: BatchEditFieldPack + + # extends a path to contain the last field + for a defined fields + @staticmethod + def from_field_spec(field_spec: QueryFieldSpec) -> "BatchEditPack": + # don't care about which way. bad things will happen if not sorted. + # not using assert () since it can be optimised out. + if batch_edit_fields["id"][1] == 0 or batch_edit_fields["order"][1] == 0: + raise Exception("the ID field should always be sorted!") + + def extend_callback(sort_type): + def _callback(field): + return BatchEditPack._query_field( + field_spec._replace( + join_path=(*field_spec.join_path, field), date_part=None + ), + sort_type, + ) + + return _callback + + new_field_specs = { + key: BatchEditFieldPack( + idx=None, + field=Func.maybe(callback(field_spec), extend_callback(sort_type)), + value=None, + ) + for key, (callback, sort_type) in batch_edit_fields.items() + } + return BatchEditPack(**new_field_specs) + + def merge(self, other: "BatchEditPack") -> "BatchEditPack": + return BatchEditPack( + id=self.id if self.id.field is not None else other.id, + version=self.version if self.version.field is not None else other.version, + order=self.order if self.order.field is not None else other.order, + ) + + # a basic query field spec to field + @staticmethod + def _query_field(field_spec: QueryFieldSpec, sort_type: int): + return QueryField( + fieldspec=field_spec, + op_num=8, + value=None, + negate=False, + display=True, + format_name=None, + sort_type=sort_type, + strict=False, + ) + + def _index( + self, + start_idx: int, + current: tuple[dict[str, BatchEditFieldPack], list[QueryField]], + next: tuple[int, tuple[str, tuple[MaybeField, int]]], + ): + current_dict, fields = current + field_idx, (field_name, _) = next + value: BatchEditFieldPack = getattr(self, field_name) + new_dict = { + **current_dict, + field_name: value._replace( + field=None, idx=((field_idx + start_idx) if value.field else None) + ), + } + new_fields = fields if value.field is None else [*fields, value.field] + return new_dict, new_fields + + def index_plan(self, start_index=0) -> tuple["BatchEditPack", list[QueryField]]: + init: tuple[dict[str, BatchEditFieldPack], list[QueryField]] = ( + {}, + [], + ) + _dict, fields = reduce( + lambda accum, next: self._index( + start_idx=start_index, current=accum, next=next + ), + enumerate(batch_edit_fields.items()), + init, + ) + return BatchEditPack(**_dict), fields + + def bind(self, row: tuple[Any]): + return BatchEditPack( + id=self.id._replace( + value=row[self.id.idx] if self.id.idx is not None else None, + ), + order=self.order._replace( + value=row[self.order.idx] if self.order.idx is not None else None + ), + version=self.version._replace( + value=row[self.version.idx] if self.version.idx is not None else None + ), + ) + + def to_json(self) -> dict[str, Any]: + return { + "id": self.id.value, + "ordernumber": self.order.value, + "version": self.version.value, + } + + # we not only care that it is part of tree, but also care that there is rank to tree + def is_part_of_tree(self, query_fields: list[QueryField]) -> bool: + if self.id.idx is None: + return False + id_field = self.id.idx + field = query_fields[id_field - 1] + join_path = field.fieldspec.join_path + if len(join_path) < 2: + return False + return isinstance(join_path[-2], TreeRankQuery) + + @staticmethod + def replace_tree_rank(fieldspec: QueryFieldSpec, tree_rank: TreeRankQuery) -> QueryFieldSpec: + return fieldspec._replace( + join_path=tuple( + [ + tree_rank if isinstance(node, TreeRankQuery) else node + for node in fieldspec.join_path + ] + ) + ) + + def readjust_tree_rank(self, tree_rank: TreeRankQuery): + id_field = self.id._replace(field=self.id.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.id.field.fieldspec, tree_rank))) if self.id.field is not None else self.id + order_field = self.order._replace(field=self.order.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.order.field.fieldspec, tree_rank))) if self.order.field is not None else self.order + version_field = self.version._replace(field=self.version.field._replace(fieldspec=BatchEditPack.replace_tree_rank(self.version.field.fieldspec, tree_rank))) if self.version.field is not None else self.version + return BatchEditPack(id=id_field, order=order_field, version=version_field) + +def _get_nested_order(field_spec: QueryFieldSpec): + # don't care about ordernumber if it ain't nested + # won't affect logic, just data being saved. + if len(field_spec.join_path) == 0: + return None + return field_spec.table.get_field("ordernumber") + +batch_edit_fields: dict[str, tuple[MaybeField, int]] = { + # technically, if version updates are correct, this is useless beyond base tables + # and to-manys. TODO: Do just that. remove it. sorts asc. using sort, the optimized + # dataset construction takes place. + "id": (lambda field_spec: field_spec.table.idField, 1), + # version control gets added here. no sort. + "version": (lambda field_spec: field_spec.table.get_field("version"), 0), + # ordernumber. no sort (actually adding a sort here is useless) + "order": (_get_nested_order, 1), +} + +# These constants are purely for memory optimization, no code depends and/or cares if this is constant. +EMPTY_FIELD = BatchEditFieldPack() +EMPTY_PACK = BatchEditPack(id=EMPTY_FIELD, order=EMPTY_FIELD, version=EMPTY_FIELD) + +class FieldLabel(NamedTuple): + field_name: str + caption: str + is_used: bool = False + +class TableFieldLabels: + table_name: str + field_labels: list[FieldLabel] + + def __init__(self, table_name: str): + self.table_name = table_name + self.field_labels = [] + + def __iter__(self): + return iter(self.field_labels) + + def add_field_label(self, field_label: FieldLabel): + self.field_labels.append(field_label) + + def get_field_label_names(self) -> list[str]: + return [field.field_name for field in self.field_labels] + + def has_field_label(self, field_name: str) -> bool: + return any(field.field_name == field_name for field in self.field_labels) + + def use_field_label(self, field_name: str) -> FieldLabel | None: + last_match_idx: int | None = None + for idx, field_label in enumerate(self.field_labels): + if field_label.field_name != field_name: + continue + last_match_idx = idx + if not field_label.is_used: + updated = field_label._replace(is_used=True) + self.field_labels[idx] = updated + return updated + if last_match_idx is not None: + updated = self.field_labels[last_match_idx]._replace(is_used=True) + self.field_labels[last_match_idx] = updated + return updated + return None + +class BatchEditMetaTables: + table_labels: dict[str, TableFieldLabels] = {} + + def __init__(self): + self.table_labels = {} + + def __init__(self, localization_dump: dict[str, list[tuple[str, str, bool]]]): + self.table_labels = {} + for table_name, field_labels in localization_dump.items(): + table_field_labels = TableFieldLabels(table_name) + for field_label in field_labels: + table_field_labels.add_field_label(FieldLabel(field_label[0], field_label[1], field_label[2])) + self.table_labels[table_name] = table_field_labels + + def add_field_label(self, table_name: str, field_label: FieldLabel): + if table_name not in self.table_labels: + self.table_labels[table_name] = TableFieldLabels(table_name) + self.table_labels[table_name].add_field_label(field_label) + + def get_table_field_labels(self, table_name: str) -> TableFieldLabels | None: + return self.table_labels.get(table_name, None) + + def get_all_table_names(self) -> list[str]: + return list(self.table_labels.keys()) + +class BatchEditProps(TypedDict): + collection: Any + user: Any + contexttableid: int + captions: Any + limit: int | None + recordsetid: int | None + session_maker: Any + fields: list[QueryField] + omit_relationships: bool | None + treedefsfilter: Any From 9215b4af4827855218df9030331dab38a1824c50 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 13:08:19 -0500 Subject: [PATCH 04/12] try using query_field_caption_lookup for better matching and disambiguation --- .../backend/stored_queries/batch_edit.py | 51 +++++++++-- .../batch_edit_helper_classes.py | 89 ++++++++++++------- 2 files changed, 101 insertions(+), 39 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 6704a9d9d0a..99aea5bc905 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -696,8 +696,10 @@ def to_upload_plan( base_table: Table, batch_edit_meta_tables: BatchEditMetaTables, query_fields: list[QueryField], + query_field_caption_lookup: dict[QueryField, str], fields_added: dict[str, int], get_column_id: Callable[[str], int], + field_caption_lookup: dict[QueryField, str], omit_relationships: bool, ) -> tuple[list[tuple[tuple[int, int], str]], Uploadable]: # Yuk, finally. @@ -718,6 +720,10 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): # It could happen that the field we saw doesn't exist. # Plus, the default options get chosen in the cases of table_name, field_name = _get_table_and_field(field) + # field_caption = field_caption_lookup.get( + # field, _reconstruct_field_caption(field) + # ) + field_caption = query_field_caption_lookup[field] table_field_labels = batch_edit_meta_tables.get_table_field_labels(table_name) if ( table_field_labels is None @@ -726,7 +732,7 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): ): localized_label = naive_field_format(field.fieldspec) else: - field_label = table_field_labels.use_field_label(field_name) + field_label = table_field_labels.use_field_label(field_name, field_caption) localized_label = ( field_label.caption if field_label is not None else naive_field_format(field.fieldspec) ) @@ -775,8 +781,10 @@ def _to_upload_plan(rel_name: str, _self: "RowPlanCanonical"): related_model, batch_edit_meta_tables, query_fields, + query_field_caption_lookup, fields_added, get_column_id, + field_caption_lookup, omit_relationships, ) @@ -891,6 +899,18 @@ def _get_table_and_field(field: QueryField): field_name = None if field.fieldspec.get_field() is None else field.fieldspec.get_field().name return (table_name, field_name) +def _reconstruct_field_caption(field: QueryField): + join_path = field.fieldspec.join_path + base_field_name = join_path[0].name if join_path else field.fieldspec.table.name + field_name = ( + None + if field.fieldspec.get_field() is None + else field.fieldspec.get_field().name + ) + if field_name is None: + return base_field_name + return f"{base_field_name} - {field_name}" + def rewrite_coordinate_fields(row, _mapped_rows: dict[tuple[tuple[str, ...], ...], Any], join_paths: tuple[tuple[str, ...], ...]) -> tuple: """ In the QueryResults we want to replace any instances of the decimal @@ -951,8 +971,12 @@ def run_batch_edit_query(props: BatchEditProps): recordsetid = props["recordsetid"] fields = props["fields"] - visible_fields = [field for field in fields if field.display] + + field_with_captions = zip(fields, captions) + query_field_caption_lookup = dict(field_with_captions) + if len(fields) != len(set(fields)): + logger.info("Key Collision: query field appears more than once.") treedefsfilter = props["treedefsfilter"] @@ -960,14 +984,21 @@ def run_batch_edit_query(props: BatchEditProps): len(visible_fields) == len(captions) ), "Got misaligned captions!" - localization_dump = {} + field_caption_pairs: list[tuple[QueryField, str]] = [] if captions: - for (field, caption) in zip(visible_fields, captions): - table_name, field_name = _get_table_and_field(field) - field_labels = localization_dump.get(table_name, []) - new_field_label = (field_name, caption, False) - field_labels.append(new_field_label) - localization_dump[table_name] = field_labels + field_caption_pairs = list(zip(visible_fields, captions)) + + query_field_caption_lookup: dict[QueryField, str] = { + field: caption for field, caption in field_caption_pairs + } + + localization_dump: dict[str, list[tuple[str, str, bool]]] = {} + for field, caption in field_caption_pairs: + table_name, field_name = _get_table_and_field(field) + field_labels = localization_dump.get(table_name, []) + new_field_label = (field_name, caption, False) + field_labels.append(new_field_label) + localization_dump[table_name] = field_labels batch_edit_meta_tables = BatchEditMetaTables(localization_dump) naive_row_plan = RowPlanMap.get_row_plan(visible_fields) @@ -1065,8 +1096,10 @@ def _get_orig_column(string_id: str): base_table, batch_edit_meta_tables, query_fields, + query_field_caption_lookup, {}, _get_orig_column, + query_field_caption_lookup, omit_relationships, ) diff --git a/specifyweb/backend/stored_queries/batch_edit_helper_classes.py b/specifyweb/backend/stored_queries/batch_edit_helper_classes.py index 0879cc46df6..b7ebe05f69c 100644 --- a/specifyweb/backend/stored_queries/batch_edit_helper_classes.py +++ b/specifyweb/backend/stored_queries/batch_edit_helper_classes.py @@ -12,36 +12,11 @@ ) from collections.abc import Callable -from specifyweb.backend.permissions.permissions import has_target_permission from specifyweb.specify.api.filter_by_col import CONCRETE_HIERARCHY -from specifyweb.specify.models import datamodel -from specifyweb.specify.models_utils.load_datamodel import Field, Relationship, Table -from specifyweb.backend.trees.views import TREE_INFORMATION, get_all_tree_information -from specifyweb.backend.trees.utils import SPECIFY_TREES -from specifyweb.specify.datamodel import is_tree_table -from specifyweb.backend.stored_queries.execution import execute -from specifyweb.backend.stored_queries.queryfield import QueryField, fields_from_json -from specifyweb.backend.stored_queries.queryfieldspec import ( - QueryFieldSpec, - QueryNode, - TreeRankQuery, -) -from specifyweb.backend.workbench.models import Spdataset -from specifyweb.backend.workbench.permissions import BatchEditDataSetPT -from specifyweb.backend.workbench.upload.treerecord import TreeRecord, TreeRankRecord, RANK_KEY_DELIMITER -from specifyweb.backend.workbench.upload.upload_plan_schema import parse_column_options -from specifyweb.backend.workbench.upload.upload_table import UploadTable -from specifyweb.backend.workbench.upload.uploadable import NULL_RECORD, Uploadable -from specifyweb.backend.workbench.views import regularize_rows +from specifyweb.specify.models_utils.load_datamodel import Field +from specifyweb.backend.stored_queries.queryfield import QueryField +from specifyweb.backend.stored_queries.queryfieldspec import QueryFieldSpec, TreeRankQuery from specifyweb.specify.utils.func import Func -from . import models -import json -from .format import ObjectFormatterProps - -from specifyweb.backend.workbench.upload.upload_plan_schema import schema -from jsonschema import validate - -from django.db import transaction MaybeField = Callable[[QueryFieldSpec], Field | None] @@ -242,6 +217,17 @@ class FieldLabel(NamedTuple): caption: str is_used: bool = False + def __str__(self) -> str: + return ( + "FieldLabel(" + f"field_name={self.field_name!r}, " + f"caption={self.caption!r}, " + f"is_used={self.is_used}" + ")" + ) + + __repr__ = __str__ + class TableFieldLabels: table_name: str field_labels: list[FieldLabel] @@ -262,13 +248,21 @@ def get_field_label_names(self) -> list[str]: def has_field_label(self, field_name: str) -> bool: return any(field.field_name == field_name for field in self.field_labels) - def use_field_label(self, field_name: str) -> FieldLabel | None: + def use_field_label( + self, + field_name: str, + expected_caption: str | None = None, + ) -> FieldLabel | None: last_match_idx: int | None = None for idx, field_label in enumerate(self.field_labels): if field_label.field_name != field_name: continue + caption_matches = ( + expected_caption is None + or field_label.caption.lower() == expected_caption.lower() + ) last_match_idx = idx - if not field_label.is_used: + if not field_label.is_used and caption_matches: updated = field_label._replace(is_used=True) self.field_labels[idx] = updated return updated @@ -278,6 +272,27 @@ def use_field_label(self, field_name: str) -> FieldLabel | None: return updated return None + def __str__(self) -> str: + indent_str = " " * indent + inner_indent = " " * (indent + 2) + if not self.field_labels: + return ( + f"{indent_str}TableFieldLabels(" + f"table_name={self.table_name!r}, field_labels=[]" + ")" + ) + lines = [ + f"{indent_str}TableFieldLabels(", + f"{indent_str} table_name={self.table_name!r},", + f"{indent_str} field_labels=[", + ] + lines.extend(f"{inner_indent}{label}" for label in self.field_labels) + lines.append(f"{indent_str} ]") + lines.append(f"{indent_str})") + return "\n".join(lines) + + __repr__ = __str__ + class BatchEditMetaTables: table_labels: dict[str, TableFieldLabels] = {} @@ -303,6 +318,20 @@ def get_table_field_labels(self, table_name: str) -> TableFieldLabels | None: def get_all_table_names(self) -> list[str]: return list(self.table_labels.keys()) + def __str__(self) -> str: + indent_str = " " * indent + if not self.table_labels: + return f"{indent_str}BatchEditMetaTables(table_labels={{}})" + lines = [f"{indent_str}BatchEditMetaTables("] + for table_name in sorted(self.table_labels.keys()): + table_lines = self.table_labels[table_name].to_pretty_string(indent + 4) + lines.append(f"{indent_str} {table_name!r}:") + lines.append(table_lines) + lines.append(f"{indent_str})") + return "\n".join(lines) + + __repr__ = __str__ + class BatchEditProps(TypedDict): collection: Any user: Any From 0e90ae8fe988aa03f5569f1e72691126aad1c10e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 13:10:40 -0500 Subject: [PATCH 05/12] fix logging --- specifyweb/backend/stored_queries/batch_edit.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 99aea5bc905..27068c3668b 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -8,7 +8,6 @@ from typing import ( Any, NamedTuple, - TypedDict, ) from collections.abc import Callable @@ -25,7 +24,6 @@ BatchEditMetaTables, BatchEditPack, BatchEditProps, - MaybeField ) from specifyweb.specify.models import datamodel from specifyweb.specify.models_utils.load_datamodel import Relationship, Table @@ -56,6 +54,10 @@ from django.db import transaction +import logging + +logger = logging.getLogger(__name__) + def get_readonly_fields(table: Table): fields = [*BATCH_EDIT_SHARED_READONLY_FIELDS, table.idFieldName.lower()] From eff6bed7eff77e8e96af2f237f1b131b1346ebe4 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 13:34:03 -0500 Subject: [PATCH 06/12] remove _reconstruct_field_caption --- specifyweb/backend/stored_queries/batch_edit.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 27068c3668b..6cddf9b88cd 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -722,9 +722,6 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): # It could happen that the field we saw doesn't exist. # Plus, the default options get chosen in the cases of table_name, field_name = _get_table_and_field(field) - # field_caption = field_caption_lookup.get( - # field, _reconstruct_field_caption(field) - # ) field_caption = query_field_caption_lookup[field] table_field_labels = batch_edit_meta_tables.get_table_field_labels(table_name) if ( @@ -901,18 +898,6 @@ def _get_table_and_field(field: QueryField): field_name = None if field.fieldspec.get_field() is None else field.fieldspec.get_field().name return (table_name, field_name) -def _reconstruct_field_caption(field: QueryField): - join_path = field.fieldspec.join_path - base_field_name = join_path[0].name if join_path else field.fieldspec.table.name - field_name = ( - None - if field.fieldspec.get_field() is None - else field.fieldspec.get_field().name - ) - if field_name is None: - return base_field_name - return f"{base_field_name} - {field_name}" - def rewrite_coordinate_fields(row, _mapped_rows: dict[tuple[tuple[str, ...], ...], Any], join_paths: tuple[tuple[str, ...], ...]) -> tuple: """ In the QueryResults we want to replace any instances of the decimal From cc823512e157c58f11ca5ac4c8e5aa9664fc46c2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 14:02:31 -0500 Subject: [PATCH 07/12] handle NoneType visible_fields and captions in batch edit request --- specifyweb/backend/stored_queries/batch_edit.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 6cddf9b88cd..c911f10c802 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -960,8 +960,11 @@ def run_batch_edit_query(props: BatchEditProps): fields = props["fields"] visible_fields = [field for field in fields if field.display] - field_with_captions = zip(fields, captions) - query_field_caption_lookup = dict(field_with_captions) + field_caption_pairs: list[tuple[QueryField, str]] = [] + query_field_caption_lookup: dict[QueryField, str] = {} + if captions is not None: + field_caption_pairs = list(zip(visible_fields, captions)) + query_field_caption_lookup = dict(field_caption_pairs) if len(fields) != len(set(fields)): logger.info("Key Collision: query field appears more than once.") From 30fcf5952d9fefa5af76f3aaf43ed5059271ddde Mon Sep 17 00:00:00 2001 From: alec_dev Date: Mon, 6 Oct 2025 14:23:44 -0500 Subject: [PATCH 08/12] handle null in query_field_caption_lookup --- specifyweb/backend/stored_queries/batch_edit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index c911f10c802..d92e575aef9 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -722,7 +722,7 @@ def _lookup_in_fields(_id: int | None, readonly_fields: list[str]): # It could happen that the field we saw doesn't exist. # Plus, the default options get chosen in the cases of table_name, field_name = _get_table_and_field(field) - field_caption = query_field_caption_lookup[field] + field_caption = query_field_caption_lookup.get(field, None) table_field_labels = batch_edit_meta_tables.get_table_field_labels(table_name) if ( table_field_labels is None From 2a8d69c3bcb7a1b4620363cb03c35a03397ffd8a Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 7 Oct 2025 10:15:50 -0500 Subject: [PATCH 09/12] fix field_caption_lookup duplicate arguments --- specifyweb/backend/stored_queries/batch_edit.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index d92e575aef9..72b101b582b 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -701,7 +701,6 @@ def to_upload_plan( query_field_caption_lookup: dict[QueryField, str], fields_added: dict[str, int], get_column_id: Callable[[str], int], - field_caption_lookup: dict[QueryField, str], omit_relationships: bool, ) -> tuple[list[tuple[tuple[int, int], str]], Uploadable]: # Yuk, finally. @@ -1089,7 +1088,6 @@ def _get_orig_column(string_id: str): query_field_caption_lookup, {}, _get_orig_column, - query_field_caption_lookup, omit_relationships, ) From e54e989c71c247bfaa1a16d3373280d23194cc7f Mon Sep 17 00:00:00 2001 From: alec_dev Date: Tue, 7 Oct 2025 10:54:03 -0500 Subject: [PATCH 10/12] remove last field_caption_lookup --- specifyweb/backend/stored_queries/batch_edit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/specifyweb/backend/stored_queries/batch_edit.py b/specifyweb/backend/stored_queries/batch_edit.py index 72b101b582b..a952cbd7dd7 100644 --- a/specifyweb/backend/stored_queries/batch_edit.py +++ b/specifyweb/backend/stored_queries/batch_edit.py @@ -782,7 +782,6 @@ def _to_upload_plan(rel_name: str, _self: "RowPlanCanonical"): query_field_caption_lookup, fields_added, get_column_id, - field_caption_lookup, omit_relationships, ) From fa2288be5b1137f6a61df2d369728efff6fd51a6 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 9 Oct 2025 12:49:04 -0500 Subject: [PATCH 11/12] init test_column_key_collision --- .../stored_queries/tests/test_batch_edit.py | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/specifyweb/backend/stored_queries/tests/test_batch_edit.py b/specifyweb/backend/stored_queries/tests/test_batch_edit.py index 2b9c954ce3c..bf177899223 100644 --- a/specifyweb/backend/stored_queries/tests/test_batch_edit.py +++ b/specifyweb/backend/stored_queries/tests/test_batch_edit.py @@ -1,6 +1,8 @@ import json from unittest.mock import patch +from django.test import Client + from specifyweb.backend.stored_queries.batch_edit import ( BatchEditPack, BatchEditProps, @@ -2310,3 +2312,318 @@ def test_to_one_stalls_to_many(self): ] self.assertEqual(correct_packs, packs) + + def test_column_key_collision(self): + c = Client() + c.force_login(self.specifyuser) + + # Create test records for query + test_collection = models.Collection.objects.first() + agent_1 = models.Agent.objects.create( + agenttype=0, + firstname="agent_test_be_key_collision_1", + lastname="agent_test_be_key_collision", + guid="agent-test-be-key-collision-1", + specifyuser=self.specifyuser + ) + agent_2 = models.Agent.objects.create( + agenttype=0, + firstname="agent_test_be_key_collision_2", + lastname="agent_test_be_key_collision", + guid="agent-test-be-key-collision-2", + specifyuser=self.specifyuser + ) + test_catalognumber = '194836194083' + co_test = models.Collectionobject.objects.create( + catalognumber=test_catalognumber, + cataloger=agent_1, + collection=models.Collection.objects.first() + ) + determiner_test = models.Determination.objects.create( + collectionobject=co_test, + iscurrent=True, + determiner=agent_2 + ) + + # Create test QB query + qb_query_response = c.post( + '/stored_query/ephemeral/', + data=json.dumps( + { + "name": "New Query", + "contextname": "CollectionObject", + "contexttableid": 1, + "selectdistinct": False, + "smushed": False, + "countonly": False, + "formatauditrecids": False, + "specifyuser": f"/api/specify/specifyuser/{self.specifyuser.id}/", + "isfavorite": True, + # "ordinal": 32767, + "fields": [ + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.guid", + "fieldname": "guid", + "isrelfld": False, + "sorttype": 0, + "position": 0, + "isdisplay": True, + "operstart": 8, + "startvalue": "", + "isnot": False, + "isstrict": False + }, + { + "tablelist": "1,9-determinations,5-determiner", + "stringid": "1,9-determinations,5-determiner.agent.guid", + "fieldname": "guid", + "isrelfld": False, + "sorttype": 0, + "position": 1, + "isdisplay": True, + "operstart": 8, + "startvalue": "", + "isnot": False, + "isstrict": False + } + ], + "_tablename": "SpQuery", + "remarks": None, + "searchsynonymy": None, + "sqlstr": None, + "timestampcreated": "2025-10-07", + "timestampmodified": None, + "version": 1, + "createdbyagent": None, + "modifiedbyagent": None, + "offset": 0, + "limit": 1 + } + ), + content_type='application/json' + ) + self.assertEqual(qb_query_response.status_code, 200) + qb_query_results = qb_query_response.json().get('results') + + saved_qb_query_response = c.post( + '/api/specify/spquery/', + data = json.dumps( + { + "name": "batch_edit_key_collision_test", + "contextname": "CollectionObject", + "contexttableid": 1, + "selectdistinct": False, + "smushed": False, + "countonly": False, + "formatauditrecids": False, + "specifyuser": f"/api/specify/specifyuser/{self.specifyuser.id}/", + "isfavorite": True, + # "ordinal": 32767, + "fields": [ + { + "tablelist": "1,5-cataloger", + "stringid": "1,5-cataloger.agent.guid", + "fieldname": "guid", + "isrelfld": False, + "sorttype": 0, + "position": 0, + "isdisplay": True, + "operstart": 8, + "startvalue": "", + "isnot": False, + "isstrict": False, + "_tableName": "SpQueryField" + }, + { + "tablelist": "1,9-determinations,5-determiner", + "stringid": "1,9-determinations,5-determiner.agent.guid", + "fieldname": "guid", + "isrelfld": False, + "sorttype": 0, + "position": 1, + "isdisplay": True, + "operstart": 8, + "startvalue": "", + "isnot": False, + "isstrict": False, + "_tableName": "SpQueryField" + } + ], + "_tableName": "SpQuery" + } + ), + content_type='application/json' + ) + self.assertEqual(saved_qb_query_response.status_code, 201) + saved_qb_query = saved_qb_query_response.json() + saved_qb_query_id = saved_qb_query.get('id') + saved_qb_query_fields = saved_qb_query.get('fields') + saved_qb_query_field_ids = [field.get('id') for field in saved_qb_query_fields] + + # Run batch edit on test query + batch_edit_response = c.post( + '/stored_query/batch_edit/', + data=json.dumps( + { + "id": saved_qb_query_id, + "contextname": "CollectionObject", + "contexttableid": 1, + "countonly": False, + "formatauditrecids": False, + "isfavorite": True, + "name": "batch_edit_key_collision_test", + # "ordinal": 32767, + "remarks": None, + "searchsynonymy": None, + "selectdistinct": False, + "smushed": False, + "sqlstr": None, + "timestampcreated": "2025-10-07T15:01:52", + "timestampmodified": "2025-10-07T15:01:52", + "version": 0, + "createdbyagent": f"/api/speacify/agent/{agent_1.id}/", + "modifiedbyagent": None, + "specifyuser": f"/api/specify/specifyuser/{agent_1.id}/", + "reports": f"/api/specify/spreport/?query={saved_qb_query_id}", + "resource_uri": f"/api/specify/spquery/{saved_qb_query_id}/", + "fields": [ + { + "id": saved_qb_query_field_ids[0], + "allownulls": None, + "alwaysfilter": None, + "columnalias": None, + "contexttableident": None, + "endvalue": None, + "fieldname": "guid", + "formatname": None, + "isdisplay": True, + "isnot": False, + "isprompt": None, + "isrelfld": False, + "operend": None, + "operstart": 8, + "position": 0, + "sorttype": 0, + "startvalue": "", + "stringid": "1,5-cataloger.agent.guid", + "tablelist": "1,5-cataloger", + "timestampcreated": "2025-10-07T15:01:52", + "timestampmodified": "2025-10-07T15:01:52", + "version": 0, + "isstrict": False, + "createdbyagent": f"/api/specify/agent/{agent_1.id}/", + "modifiedbyagent": None, + "query": f"/api/specify/spquery/{saved_qb_query_id}/", + "mappings": f"/api/specify/spexportschemaitemmapping/?queryfield={saved_qb_query_field_ids[0]}", + "resource_uri": f"/api/specify/spqueryfield/{saved_qb_query_field_ids[0]}/", + "_tablename": "SpQueryField" + }, + { + "id": saved_qb_query_field_ids[1], + "allownulls": None, + "alwaysfilter": None, + "columnalias": None, + "contexttableident": None, + "endvalue": None, + "fieldname": "guid", + "formatname": None, + "isdisplay": True, + "isnot": False, + "isprompt": None, + "isrelfld": False, + "operend": None, + "operstart": 8, + "position": 1, + "sorttype": 0, + "startvalue": "", + "stringid": "1,9-determinations,5-determiner.agent.guid", + "tablelist": "1,9-determinations,5-determiner", + "timestampcreated": "2025-10-07T15:01:52", + "timestampmodified": "2025-10-07T15:01:52", + "version": 0, + "isstrict": False, + "createdbyagent": f"/api/specify/agent/{agent_1.id}/", + "modifiedbyagent": None, + "query": f"/api/specify/spquery/{saved_qb_query_id}/", + "mappings": f"/api/specify/spexportschemaitemmapping/?queryfield={saved_qb_query_field_ids[1]}", + "resource_uri": f"/api/specify/spqueryfield/{saved_qb_query_field_ids[1]}/", + "_tablename": "SpQueryField" + } + ], + "_tablename": "SpQuery", + "captions": [ + "Cataloger - GUID", + "Determiner - GUID" + ], + "limit": 1, + "treedefsfilter": {}, + "omitrelationships": False + } + ), + content_type='application/json' + ) + self.assertEqual(batch_edit_response.status_code, 201) + batch_edit_id = batch_edit_response.json().get('id') + + batch_edit_workbench_response = c.get( + f'/api/workbench/dataset/{batch_edit_id}/' + ) + self.assertEqual(batch_edit_workbench_response.status_code, 200) + batch_edit_dataset = batch_edit_workbench_response.json() + + # Asset the columns and data match the query results + pass + + @patch(OBJ_FORMATTER_PATH, new=fake_obj_formatter) + def test_column_key_collision_2(self): + c = Client() + c.force_login(self.specifyuser) + + # Create test records for query + test_collection = models.Collection.objects.first() + agent_1 = models.Agent.objects.create( + agenttype=0, + firstname="agent_test_be_key_collision_1", + lastname="agent_test_be_key_collision", + guid="agent-test-be-key-collision-1", + specifyuser=self.specifyuser + ) + agent_2 = models.Agent.objects.create( + agenttype=0, + firstname="agent_test_be_key_collision_2", + lastname="agent_test_be_key_collision", + guid="agent-test-be-key-collision-2", + specifyuser=self.specifyuser + ) + test_catalognumber = '194836194083' + co_test = models.Collectionobject.objects.create( + catalognumber=test_catalognumber, + cataloger=agent_1, + collection=models.Collection.objects.first() + ) + determiner_test = models.Determination.objects.create( + collectionobject=co_test, + iscurrent=True, + determiner=agent_2 + ) + + # Build QB Query + base_table = "collectionobject" + query_paths = [ + ["cataloger", "GUID"], + ["determinations", "Determiner", "GUID"], + ] + + added = [(base_table, *path) for path in query_paths] + + query_fields = [ + BatchEditPack._query_field(QueryFieldSpec.from_path(path), 0) + for path in added + ] + + props = self.build_props(query_fields, base_table) + + (headers, rows, packs, plan, order) = run_batch_edit_query(props) + + From fa373fb986b0db187382fb9ba6c3083bdbaf744e Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 9 Oct 2025 12:53:51 -0500 Subject: [PATCH 12/12] edit test_column_key_collision unit test --- .../stored_queries/tests/test_batch_edit.py | 312 +----------------- 1 file changed, 14 insertions(+), 298 deletions(-) diff --git a/specifyweb/backend/stored_queries/tests/test_batch_edit.py b/specifyweb/backend/stored_queries/tests/test_batch_edit.py index bf177899223..6372521c6f3 100644 --- a/specifyweb/backend/stored_queries/tests/test_batch_edit.py +++ b/specifyweb/backend/stored_queries/tests/test_batch_edit.py @@ -1,8 +1,6 @@ import json from unittest.mock import patch -from django.test import Client - from specifyweb.backend.stored_queries.batch_edit import ( BatchEditPack, BatchEditProps, @@ -2314,302 +2312,9 @@ def test_to_one_stalls_to_many(self): self.assertEqual(correct_packs, packs) def test_column_key_collision(self): - c = Client() - c.force_login(self.specifyuser) - - # Create test records for query - test_collection = models.Collection.objects.first() - agent_1 = models.Agent.objects.create( - agenttype=0, - firstname="agent_test_be_key_collision_1", - lastname="agent_test_be_key_collision", - guid="agent-test-be-key-collision-1", - specifyuser=self.specifyuser - ) - agent_2 = models.Agent.objects.create( - agenttype=0, - firstname="agent_test_be_key_collision_2", - lastname="agent_test_be_key_collision", - guid="agent-test-be-key-collision-2", - specifyuser=self.specifyuser - ) - test_catalognumber = '194836194083' - co_test = models.Collectionobject.objects.create( - catalognumber=test_catalognumber, - cataloger=agent_1, - collection=models.Collection.objects.first() - ) - determiner_test = models.Determination.objects.create( - collectionobject=co_test, - iscurrent=True, - determiner=agent_2 - ) - - # Create test QB query - qb_query_response = c.post( - '/stored_query/ephemeral/', - data=json.dumps( - { - "name": "New Query", - "contextname": "CollectionObject", - "contexttableid": 1, - "selectdistinct": False, - "smushed": False, - "countonly": False, - "formatauditrecids": False, - "specifyuser": f"/api/specify/specifyuser/{self.specifyuser.id}/", - "isfavorite": True, - # "ordinal": 32767, - "fields": [ - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.guid", - "fieldname": "guid", - "isrelfld": False, - "sorttype": 0, - "position": 0, - "isdisplay": True, - "operstart": 8, - "startvalue": "", - "isnot": False, - "isstrict": False - }, - { - "tablelist": "1,9-determinations,5-determiner", - "stringid": "1,9-determinations,5-determiner.agent.guid", - "fieldname": "guid", - "isrelfld": False, - "sorttype": 0, - "position": 1, - "isdisplay": True, - "operstart": 8, - "startvalue": "", - "isnot": False, - "isstrict": False - } - ], - "_tablename": "SpQuery", - "remarks": None, - "searchsynonymy": None, - "sqlstr": None, - "timestampcreated": "2025-10-07", - "timestampmodified": None, - "version": 1, - "createdbyagent": None, - "modifiedbyagent": None, - "offset": 0, - "limit": 1 - } - ), - content_type='application/json' - ) - self.assertEqual(qb_query_response.status_code, 200) - qb_query_results = qb_query_response.json().get('results') - - saved_qb_query_response = c.post( - '/api/specify/spquery/', - data = json.dumps( - { - "name": "batch_edit_key_collision_test", - "contextname": "CollectionObject", - "contexttableid": 1, - "selectdistinct": False, - "smushed": False, - "countonly": False, - "formatauditrecids": False, - "specifyuser": f"/api/specify/specifyuser/{self.specifyuser.id}/", - "isfavorite": True, - # "ordinal": 32767, - "fields": [ - { - "tablelist": "1,5-cataloger", - "stringid": "1,5-cataloger.agent.guid", - "fieldname": "guid", - "isrelfld": False, - "sorttype": 0, - "position": 0, - "isdisplay": True, - "operstart": 8, - "startvalue": "", - "isnot": False, - "isstrict": False, - "_tableName": "SpQueryField" - }, - { - "tablelist": "1,9-determinations,5-determiner", - "stringid": "1,9-determinations,5-determiner.agent.guid", - "fieldname": "guid", - "isrelfld": False, - "sorttype": 0, - "position": 1, - "isdisplay": True, - "operstart": 8, - "startvalue": "", - "isnot": False, - "isstrict": False, - "_tableName": "SpQueryField" - } - ], - "_tableName": "SpQuery" - } - ), - content_type='application/json' - ) - self.assertEqual(saved_qb_query_response.status_code, 201) - saved_qb_query = saved_qb_query_response.json() - saved_qb_query_id = saved_qb_query.get('id') - saved_qb_query_fields = saved_qb_query.get('fields') - saved_qb_query_field_ids = [field.get('id') for field in saved_qb_query_fields] - - # Run batch edit on test query - batch_edit_response = c.post( - '/stored_query/batch_edit/', - data=json.dumps( - { - "id": saved_qb_query_id, - "contextname": "CollectionObject", - "contexttableid": 1, - "countonly": False, - "formatauditrecids": False, - "isfavorite": True, - "name": "batch_edit_key_collision_test", - # "ordinal": 32767, - "remarks": None, - "searchsynonymy": None, - "selectdistinct": False, - "smushed": False, - "sqlstr": None, - "timestampcreated": "2025-10-07T15:01:52", - "timestampmodified": "2025-10-07T15:01:52", - "version": 0, - "createdbyagent": f"/api/speacify/agent/{agent_1.id}/", - "modifiedbyagent": None, - "specifyuser": f"/api/specify/specifyuser/{agent_1.id}/", - "reports": f"/api/specify/spreport/?query={saved_qb_query_id}", - "resource_uri": f"/api/specify/spquery/{saved_qb_query_id}/", - "fields": [ - { - "id": saved_qb_query_field_ids[0], - "allownulls": None, - "alwaysfilter": None, - "columnalias": None, - "contexttableident": None, - "endvalue": None, - "fieldname": "guid", - "formatname": None, - "isdisplay": True, - "isnot": False, - "isprompt": None, - "isrelfld": False, - "operend": None, - "operstart": 8, - "position": 0, - "sorttype": 0, - "startvalue": "", - "stringid": "1,5-cataloger.agent.guid", - "tablelist": "1,5-cataloger", - "timestampcreated": "2025-10-07T15:01:52", - "timestampmodified": "2025-10-07T15:01:52", - "version": 0, - "isstrict": False, - "createdbyagent": f"/api/specify/agent/{agent_1.id}/", - "modifiedbyagent": None, - "query": f"/api/specify/spquery/{saved_qb_query_id}/", - "mappings": f"/api/specify/spexportschemaitemmapping/?queryfield={saved_qb_query_field_ids[0]}", - "resource_uri": f"/api/specify/spqueryfield/{saved_qb_query_field_ids[0]}/", - "_tablename": "SpQueryField" - }, - { - "id": saved_qb_query_field_ids[1], - "allownulls": None, - "alwaysfilter": None, - "columnalias": None, - "contexttableident": None, - "endvalue": None, - "fieldname": "guid", - "formatname": None, - "isdisplay": True, - "isnot": False, - "isprompt": None, - "isrelfld": False, - "operend": None, - "operstart": 8, - "position": 1, - "sorttype": 0, - "startvalue": "", - "stringid": "1,9-determinations,5-determiner.agent.guid", - "tablelist": "1,9-determinations,5-determiner", - "timestampcreated": "2025-10-07T15:01:52", - "timestampmodified": "2025-10-07T15:01:52", - "version": 0, - "isstrict": False, - "createdbyagent": f"/api/specify/agent/{agent_1.id}/", - "modifiedbyagent": None, - "query": f"/api/specify/spquery/{saved_qb_query_id}/", - "mappings": f"/api/specify/spexportschemaitemmapping/?queryfield={saved_qb_query_field_ids[1]}", - "resource_uri": f"/api/specify/spqueryfield/{saved_qb_query_field_ids[1]}/", - "_tablename": "SpQueryField" - } - ], - "_tablename": "SpQuery", - "captions": [ - "Cataloger - GUID", - "Determiner - GUID" - ], - "limit": 1, - "treedefsfilter": {}, - "omitrelationships": False - } - ), - content_type='application/json' - ) - self.assertEqual(batch_edit_response.status_code, 201) - batch_edit_id = batch_edit_response.json().get('id') - - batch_edit_workbench_response = c.get( - f'/api/workbench/dataset/{batch_edit_id}/' - ) - self.assertEqual(batch_edit_workbench_response.status_code, 200) - batch_edit_dataset = batch_edit_workbench_response.json() - - # Asset the columns and data match the query results - pass - - @patch(OBJ_FORMATTER_PATH, new=fake_obj_formatter) - def test_column_key_collision_2(self): - c = Client() - c.force_login(self.specifyuser) - - # Create test records for query - test_collection = models.Collection.objects.first() - agent_1 = models.Agent.objects.create( - agenttype=0, - firstname="agent_test_be_key_collision_1", - lastname="agent_test_be_key_collision", - guid="agent-test-be-key-collision-1", - specifyuser=self.specifyuser - ) - agent_2 = models.Agent.objects.create( - agenttype=0, - firstname="agent_test_be_key_collision_2", - lastname="agent_test_be_key_collision", - guid="agent-test-be-key-collision-2", - specifyuser=self.specifyuser - ) - test_catalognumber = '194836194083' - co_test = models.Collectionobject.objects.create( - catalognumber=test_catalognumber, - cataloger=agent_1, - collection=models.Collection.objects.first() - ) - determiner_test = models.Determination.objects.create( - collectionobject=co_test, - iscurrent=True, - determiner=agent_2 - ) - # Build QB Query base_table = "collectionobject" + expected_captions = ['Cataloger - GUID', 'Determiner - GUID'] query_paths = [ ["cataloger", "GUID"], ["determinations", "Determiner", "GUID"], @@ -2622,8 +2327,19 @@ def test_column_key_collision_2(self): for path in added ] - props = self.build_props(query_fields, base_table) - + props = BatchEditProps( + collection=self.collection, + user=self.specifyuser, + contexttableid=datamodel.get_table_strict(base_table).tableId, + fields=query_fields, + session_maker=QueryConstructionTests.test_session_context, + captions=expected_captions, + limit=None, + recordsetid=None, + omit_relationships=False, + treedefsfilter=None + ) (headers, rows, packs, plan, order) = run_batch_edit_query(props) + self.assertEqual(headers, expected_captions)