diff --git a/specifyweb/backend/stored_queries/blank_nulls.py b/specifyweb/backend/stored_queries/blank_nulls.py index da492f10117..1f532a5f16f 100644 --- a/specifyweb/backend/stored_queries/blank_nulls.py +++ b/specifyweb/backend/stored_queries/blank_nulls.py @@ -16,6 +16,7 @@ """ class blank_nulls(expression.FunctionElement): name = 'blank_nulls' + type = String() inherit_cache = True """ The `@compiles` decorator tells sqlalchemy to run this function whenever diff --git a/specifyweb/backend/stored_queries/format.py b/specifyweb/backend/stored_queries/format.py index 9a06926678c..b671a3e3ef5 100644 --- a/specifyweb/backend/stored_queries/format.py +++ b/specifyweb/backend/stored_queries/format.py @@ -7,6 +7,7 @@ from xml.etree.ElementTree import Element from xml.sax.saxutils import quoteattr +from specifyweb.backend.stored_queries.utils import ensure_string_type_if_null from specifyweb.specify.api.utils import get_picklists from sqlalchemy import Table as SQLTable, inspect, case, type_coerce from sqlalchemy.orm import aliased, Query @@ -208,9 +209,14 @@ def make_expr(self, new_expr = self._fieldformat(formatter_field_spec.table, formatter_field_spec.get_field(), new_expr) if 'trimzeros' in fieldNodeAttrib: + # new_expr = case( + # [(new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), cast(new_expr, types.Numeric(65)))], + # else_=new_expr + # ) + numeric_str = cast(cast(new_expr, types.Numeric(65)), types.String()) new_expr = case( - [(new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), cast(new_expr, types.Numeric(65)))], - else_=new_expr + (new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), numeric_str), + else_=cast(new_expr, types.String()), ) if 'format' in fieldNodeAttrib: @@ -377,17 +383,25 @@ def _dateformat(self, specify_field, field): prec_fld = getattr(field.class_, specify_field.name + 'Precision', None) - format_expr = ( - case( - [ - (prec_fld == 2, self.date_format_month), - (prec_fld == 3, self.date_format_year), - ], - else_=self.date_format, + # format_expr = ( + # case( + # [ + # (prec_fld == 2, self.date_format_month), + # (prec_fld == 3, self.date_format_year), + # ], + # else_=self.date_format, + # ) + # if prec_fld is not None + # else self.date_format + # ) + if prec_fld is not None: + format_expr = case( + (prec_fld == 2, literal(self.date_format_month, type_=types.String())), + (prec_fld == 3, literal(self.date_format_year, type_=types.String())), + else_=ensure_string_type_if_null(literal(self.date_format, type_=types.String())), ) - if prec_fld is not None - else self.date_format - ) + else: + format_expr = literal(self.date_format, type_=types.String()) return func.date_format(field, format_expr) @@ -395,8 +409,10 @@ def _fieldformat(self, table: Table, specify_field: Field, field: InstrumentedAttribute | Extract): if self.format_agent_type and specify_field is Agent_model.get_field("agenttype"): - cases = [(field == _id, name) for (_id, name) in enumerate(agent_types)] - _case = case(cases) + # cases = [(field == _id, name) for (_id, name) in enumerate(agent_types)] + # _case = case(cases) + cases = [(field == _id, ensure_string_type_if_null(literal(name, type_=types.String()))) for (_id, name) in enumerate(agent_types)] + _case = case(cases, else_=ensure_string_type_if_null(literal('', type_=types.String()))) return blank_nulls(_case) if self.replace_nulls else _case if self.format_picklist: @@ -410,7 +426,7 @@ def _fieldformat(self, table: Table, specify_field: Field, return blank_nulls(expr) if self.replace_nulls else expr cases = [ - (field == item.value, literal(item.title or "", type_=types.String())) + (field == item.value, ensure_string_type_if_null(literal(item.title or ""))) for item in items ] _case = case(cases, else_=cast(field, types.String())) diff --git a/specifyweb/backend/stored_queries/group_concat.py b/specifyweb/backend/stored_queries/group_concat.py index f14bb507e5e..6b21055cb09 100644 --- a/specifyweb/backend/stored_queries/group_concat.py +++ b/specifyweb/backend/stored_queries/group_concat.py @@ -2,48 +2,45 @@ # https://stackoverflow.com/questions/19205850/how-do-i-write-a-group-concat-function-in-sqlalchemy import sqlalchemy -from sqlalchemy.sql import expression +from sqlalchemy.sql import expression, bindparam from sqlalchemy.ext import compiler +from sqlalchemy.sql.elements import ClauseElement +from sqlalchemy.types import String from specifyweb.backend.stored_queries.query_construct import QueryConstruct -# class changed from FunctionElement to ColumnElement class group_concat(expression.ColumnElement): + # MySQL GROUP_CONCAT(expr [ORDER BY ...] [SEPARATOR ...]) name = "group_concat" + type = String() inherit_cache = True - - def __init__(self, expr, separator=None, order_by=None): + + def __init__(self, expr: ClauseElement, separator=None, order_by: ClauseElement | None = None): + super().__init__() self.expr = expr self.separator = separator self.order_by = order_by @compiler.compiles(group_concat) def _group_concat_mysql(element, compiler, **kwargs): - # Old way of extracting clauses, when group_concat was a FunctionElement - # expr, separator, order_by = extract_clauses(element, compiler) - - inner_expr= compiler.process(element.expr, **kwargs) + inner_expr = compiler.process(element.expr, **kwargs) if element.order_by is not None: - order_by = compiler.process(element.order_by, **kwargs) - inner_expr+= " ORDER BY %s" % order_by + if isinstance(element.order_by, (list, tuple, set)): + ob = ", ".join(compiler.process(ob, **kwargs) for ob in element.order_by) + else: + ob = compiler.process(element.order_by, **kwargs) + inner_expr += f" ORDER BY {ob}" + if element.separator is not None: - # Resorting to text() + bindparams to avoid SQL injection and fix parameter ordering bug - separator = compiler.process(sqlalchemy.text(" SEPARATOR :sep").bindparams(sep=element.separator), **kwargs) - inner_expr+= " %s" % separator + if isinstance(element.separator, ClauseElement): + sep_sql = compiler.process(element.separator, **kwargs) + else: + sep_sql = compiler.process(bindparam("sep", element.separator, type_=String()), **kwargs) + inner_expr += f" SEPARATOR {sep_sql}" return 'GROUP_CONCAT(%s)' % inner_expr -def extract_clauses(element, compiler): - expr = compiler.process(element.clauses.clauses[0]) - def process_clause(idx): - return compiler.process(element.clauses.clauses[idx]) - - separator = process_clause(1) if len(element.clauses) > 1 else None - order_by = process_clause(2) if len(element.clauses) > 2 else None - - return expr, separator, order_by - def group_by_displayed_fields(query: QueryConstruct, fields, ignore_cat_num=False): for field in fields: if ( diff --git a/specifyweb/backend/stored_queries/utils.py b/specifyweb/backend/stored_queries/utils.py index 218423c48f0..64cd49b1944 100644 --- a/specifyweb/backend/stored_queries/utils.py +++ b/specifyweb/backend/stored_queries/utils.py @@ -1,15 +1,90 @@ import logging +from typing import Any, Optional + +from sqlalchemy import type_coerce, types +from sqlalchemy.dialects import mysql as mysql_dialect +from sqlalchemy.sql.elements import ClauseElement +from sqlalchemy.sql.selectable import Select +from sqlalchemy.sql.sqltypes import NullType logger = logging.getLogger(__name__) -def log_sqlalchemy_query(query): - # Call this function to debug the raw SQL query generated by SQLAlchemy - # TODO: verify theis import - from sqlalchemy.dialects import mysql - compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True}) - raw_sql = str(compiled_query).replace('\n', ' ') + ';' - logger.debug('='.join(['' for _ in range(80)])) - logger.debug(raw_sql) - logger.debug('='.join(['' for _ in range(80)])) - # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546 - # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) \ No newline at end of file +def _coerce_statement(obj: Any) -> ClauseElement: + """ + Accepts an ORM Query (legacy), a Select, or any ClauseElement. + Returns a ClauseElement suitable for compilation. + """ + # Legacy ORM Query has .statement + stmt = getattr(obj, "statement", None) + if stmt is not None: + return stmt + if isinstance(obj, ClauseElement): + return obj + raise TypeError(f"Unsupported query/select type: {type(obj)!r}") + +def _debug_nulltype_columns(stmt: ClauseElement) -> None: + """ + Scan for expressions with NullType before compiling, to + help explain TypeError crashes when using literal_binds=True. + """ + try: + raw_cols = getattr(stmt, "_raw_columns", None) + if not raw_cols: + return + for i, expr in enumerate(raw_cols): + t = getattr(expr, "type", None) + if t is None or isinstance(t, NullType): + logger.debug("[SA-NullType] col #%s expr=%r type=%r", i, expr, t) + except Exception: + pass + +def log_sqlalchemy_query( + query_or_stmt: Any, + *, + literal_binds: bool = True, + dialect: Optional[Any] = None, + level: int = logging.DEBUG, +) -> Optional[str]: + """ + Log the SQL for a query/statement. + Run in the stored_queries.execute file, in the execute function, right before the return statement: + from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query) + """ + if not logger.isEnabledFor(level): + return None # skip compiling and logging if we're not logging at this level + + dialect = dialect or mysql_dialect.dialect() + stmt = _coerce_statement(query_or_stmt) + + # help trace NullType roots issues + _debug_nulltype_columns(stmt) + + try: + compiled = stmt.compile(dialect=dialect, compile_kwargs={"literal_binds": literal_binds}) + sql = str(compiled).replace("\n", " ") + ";" + sep = "=" * 80 + logger.log(level, "%s\n%s\n%s", sep, sql, sep) + return sql + + except TypeError as e: + try: + compiled_fb = stmt.compile(dialect=dialect) + sql_fb = str(compiled_fb) + logger.log( + level, + "[fallback no-literal-binds]\n%s\n[compile TypeError: %s]", + sql_fb, + e, + ) + return sql_fb + except Exception as e2: + logger.log(level, "[query logging failed after TypeError: %s] (fallback err: %s)", e, e2) + return None + + except Exception as e: + logger.log(level, "[query logging failed: %s]", e, exc_info=True) + return None + +def ensure_string_type_if_null(expr): + t = getattr(expr, "type", None) + return type_coerce(expr, types.String()) if t is None or isinstance(t, NullType) else expr \ No newline at end of file diff --git a/specifyweb/frontend/js_src/lib/components/FieldFormatters/spec.ts b/specifyweb/frontend/js_src/lib/components/FieldFormatters/spec.ts index 028455f853a..ee49406c6a6 100644 --- a/specifyweb/frontend/js_src/lib/components/FieldFormatters/spec.ts +++ b/specifyweb/frontend/js_src/lib/components/FieldFormatters/spec.ts @@ -39,8 +39,8 @@ export const fieldFormattersSpec = f.store(() => ? javaClass : undefined), rawAutoNumber: formatter.parts.some(isAutoNumbering) - ? legacyAutoNumber ?? - inferLegacyAutoNumber(table, formatter.parts) + ? (legacyAutoNumber ?? + inferLegacyAutoNumber(table, formatter.parts)) : undefined, }) ), @@ -149,10 +149,10 @@ export function normalizeFieldFormatterPart( part.type === 'regex' ? localized(trimRegexString(part.placeholder)) : part.type === 'year' - ? fieldFormatterTypeMapper.year.placeholder - : part.type === 'numeric' - ? fieldFormatterTypeMapper.numeric.buildPlaceholder(part.size) - : part.placeholder; + ? fieldFormatterTypeMapper.year.placeholder + : part.type === 'numeric' + ? fieldFormatterTypeMapper.numeric.buildPlaceholder(part.size) + : part.placeholder; const size = fieldFormatterTypesWithForcedSize.has(part.type as 'constant') ? placeholder.length : part.size; diff --git a/specifyweb/frontend/js_src/lib/components/Formatters/Element.tsx b/specifyweb/frontend/js_src/lib/components/Formatters/Element.tsx index a173e132107..1cb6f1fc3be 100644 --- a/specifyweb/frontend/js_src/lib/components/Formatters/Element.tsx +++ b/specifyweb/frontend/js_src/lib/components/Formatters/Element.tsx @@ -153,7 +153,7 @@ export const makeXmlEditorShellSlot = < readonly title: string | undefined; readonly isDefault: boolean; readonly table: SpecifyTable | undefined; - } + }, >( children: (getSet: GetSet) => JSX.Element, index: string | undefined, diff --git a/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx b/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx index 63188fab1e1..fd520a77354 100644 --- a/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx +++ b/specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx @@ -114,14 +114,14 @@ function WbSpreadsheetComponent({ : label; // REFACTOR: use new table icons const tableSvg = renderToStaticMarkup( - - ); + + ); - return `