Fix type declaration in group_concat and blank_nulls

specifyweb/backend/stored_queries/blank_nulls.py

@@ -16,6 +16,7 @@
 """
 class blank_nulls(expression.FunctionElement):
     name = 'blank_nulls'
+    type = String()
     inherit_cache = True
 
 """ The `@compiles` decorator tells sqlalchemy to run this function whenever 

specifyweb/backend/stored_queries/format.py

@@ -7,6 +7,7 @@
 from xml.etree.ElementTree import Element
 from xml.sax.saxutils import quoteattr
 
+from specifyweb.backend.stored_queries.utils import ensure_string_type_if_null
 from specifyweb.specify.api.utils import get_picklists
 from sqlalchemy import Table as SQLTable, inspect, case, type_coerce
 from sqlalchemy.orm import aliased, Query
@@ -208,9 +209,14 @@ def make_expr(self,
                 new_expr = self._fieldformat(formatter_field_spec.table, formatter_field_spec.get_field(), new_expr)
 
         if 'trimzeros' in fieldNodeAttrib:
+            # new_expr = case(
+            #     [(new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), cast(new_expr, types.Numeric(65)))],
+            #     else_=new_expr
+            # )
+            numeric_str = cast(cast(new_expr, types.Numeric(65)), types.String())
             new_expr = case(
-                [(new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), cast(new_expr, types.Numeric(65)))],
-                else_=new_expr
+                (new_expr.op('REGEXP')('^-?[0-9]+(\\.[0-9]+)?$'), numeric_str),
+                else_=cast(new_expr, types.String()),
             )
 
         if 'format' in fieldNodeAttrib:
@@ -377,26 +383,36 @@ def _dateformat(self, specify_field, field):
 
         prec_fld = getattr(field.class_, specify_field.name + 'Precision', None)
 
-        format_expr = (
-            case(
-                [
-                    (prec_fld == 2, self.date_format_month),
-                    (prec_fld == 3, self.date_format_year),
-                ],
-                else_=self.date_format,
+        # format_expr = (
+        #     case(
+        #         [
+        #             (prec_fld == 2, self.date_format_month),
+        #             (prec_fld == 3, self.date_format_year),
+        #         ],
+        #         else_=self.date_format,
+        #     )
+        #     if prec_fld is not None
+        #     else self.date_format
+        # )
+        if prec_fld is not None:
+            format_expr = case(
+                (prec_fld == 2, literal(self.date_format_month, type_=types.String())),
+                (prec_fld == 3, literal(self.date_format_year,  type_=types.String())),
+                else_=ensure_string_type_if_null(literal(self.date_format, type_=types.String())),
             )
-            if prec_fld is not None
-            else self.date_format
-        )
+        else:
+            format_expr = literal(self.date_format, type_=types.String())
 
         return func.date_format(field, format_expr)
 
     def _fieldformat(self, table: Table, specify_field: Field,
                      field: InstrumentedAttribute | Extract):
 
         if self.format_agent_type and specify_field is Agent_model.get_field("agenttype"):
-            cases = [(field == _id, name) for (_id, name) in enumerate(agent_types)]
-            _case = case(cases)
+            # cases = [(field == _id, name) for (_id, name) in enumerate(agent_types)]
+            # _case = case(cases)
+            cases = [(field == _id, ensure_string_type_if_null(literal(name, type_=types.String()))) for (_id, name) in enumerate(agent_types)]
+            _case = case(cases, else_=ensure_string_type_if_null(literal('', type_=types.String())))
             return blank_nulls(_case) if self.replace_nulls else _case
 
         if self.format_picklist:
@@ -410,7 +426,7 @@ def _fieldformat(self, table: Table, specify_field: Field,
                     return blank_nulls(expr) if self.replace_nulls else expr
 
                 cases = [
-                    (field == item.value, literal(item.title or "", type_=types.String()))
+                    (field == item.value, ensure_string_type_if_null(literal(item.title or "")))
                     for item in items
                 ]
                 _case = case(cases, else_=cast(field, types.String()))

specifyweb/backend/stored_queries/group_concat.py

@@ -2,48 +2,45 @@
 # https://stackoverflow.com/questions/19205850/how-do-i-write-a-group-concat-function-in-sqlalchemy
 
 import sqlalchemy
-from sqlalchemy.sql import expression
+from sqlalchemy.sql import expression, bindparam
 from sqlalchemy.ext import compiler
+from sqlalchemy.sql.elements import ClauseElement
+from sqlalchemy.types import String
 
 from specifyweb.backend.stored_queries.query_construct import QueryConstruct
 
-# class changed from FunctionElement to ColumnElement
 class group_concat(expression.ColumnElement):
+    # MySQL GROUP_CONCAT(expr [ORDER BY ...] [SEPARATOR ...])
     name = "group_concat"
+    type = String()
     inherit_cache = True
-
-    def __init__(self, expr, separator=None, order_by=None):
+
+    def __init__(self, expr: ClauseElement, separator=None, order_by: ClauseElement | None = None):
+        super().__init__()
         self.expr = expr
         self.separator = separator
         self.order_by = order_by
 
 @compiler.compiles(group_concat)
 def _group_concat_mysql(element, compiler, **kwargs):
-    # Old way of extracting clauses, when group_concat was a FunctionElement
-    # expr, separator, order_by = extract_clauses(element, compiler)
-
-    inner_expr= compiler.process(element.expr, **kwargs)
+    inner_expr = compiler.process(element.expr, **kwargs)
 
     if element.order_by is not None:
-        order_by = compiler.process(element.order_by, **kwargs)
-        inner_expr+= " ORDER BY %s" % order_by
+        if isinstance(element.order_by, (list, tuple, set)):
+            ob = ", ".join(compiler.process(ob, **kwargs) for ob in element.order_by)
+        else:
+            ob = compiler.process(element.order_by, **kwargs)
+        inner_expr += f" ORDER BY {ob}"
+
     if element.separator is not None:
-        # Resorting to text() + bindparams to avoid SQL injection and fix parameter ordering bug
-        separator = compiler.process(sqlalchemy.text(" SEPARATOR :sep").bindparams(sep=element.separator), **kwargs)
-        inner_expr+= " %s" % separator
+        if isinstance(element.separator, ClauseElement):
+            sep_sql = compiler.process(element.separator, **kwargs)
+        else:
+            sep_sql = compiler.process(bindparam("sep", element.separator, type_=String()), **kwargs)
+        inner_expr += f" SEPARATOR {sep_sql}"
 
     return 'GROUP_CONCAT(%s)' % inner_expr
 
-def extract_clauses(element, compiler):
-    expr = compiler.process(element.clauses.clauses[0])
-    def process_clause(idx):
-        return compiler.process(element.clauses.clauses[idx])
-
-    separator = process_clause(1) if len(element.clauses) > 1 else None
-    order_by = process_clause(2) if len(element.clauses) > 2 else None
-
-    return expr, separator, order_by
-
 def group_by_displayed_fields(query: QueryConstruct, fields, ignore_cat_num=False):
     for field in fields:
         if (

specifyweb/backend/stored_queries/utils.py

@@ -1,15 +1,90 @@
 import logging
+from typing import Any, Optional
+
+from sqlalchemy import type_coerce, types
+from sqlalchemy.dialects import mysql as mysql_dialect
+from sqlalchemy.sql.elements import ClauseElement
+from sqlalchemy.sql.selectable import Select
+from sqlalchemy.sql.sqltypes import NullType
 
 logger = logging.getLogger(__name__)
 
-def log_sqlalchemy_query(query):
-    # Call this function to debug the raw SQL query generated by SQLAlchemy
-    # TODO: verify theis import
-    from sqlalchemy.dialects import mysql
-    compiled_query = query.statement.compile(dialect=mysql.dialect(), compile_kwargs={"literal_binds": True})
-    raw_sql = str(compiled_query).replace('\n', ' ') + ';'
-    logger.debug('='.join(['' for _ in range(80)]))
-    logger.debug(raw_sql)
-    logger.debug('='.join(['' for _ in range(80)]))
-    # Run in the storred_queries.execute file, in the execute function, right before the return statement, line 546
-    # from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query)
+def _coerce_statement(obj: Any) -> ClauseElement:
+    """
+    Accepts an ORM Query (legacy), a Select, or any ClauseElement.
+    Returns a ClauseElement suitable for compilation.
+    """
+    # Legacy ORM Query has .statement
+    stmt = getattr(obj, "statement", None)
+    if stmt is not None:
+        return stmt
+    if isinstance(obj, ClauseElement):
+        return obj
+    raise TypeError(f"Unsupported query/select type: {type(obj)!r}")
+
+def _debug_nulltype_columns(stmt: ClauseElement) -> None:
+    """
+    Scan for expressions with NullType before compiling, to
+    help explain TypeError crashes when using literal_binds=True.
+    """
+    try:
+        raw_cols = getattr(stmt, "_raw_columns", None)
+        if not raw_cols:
+            return
+        for i, expr in enumerate(raw_cols):
+            t = getattr(expr, "type", None)
+            if t is None or isinstance(t, NullType):
+                logger.debug("[SA-NullType] col #%s expr=%r type=%r", i, expr, t)
+    except Exception:
+        pass
+
+def log_sqlalchemy_query(
+    query_or_stmt: Any,
+    *,
+    literal_binds: bool = True,
+    dialect: Optional[Any] = None,
+    level: int = logging.DEBUG,
+) -> Optional[str]:
+    """
+    Log the SQL for a query/statement.
+    Run in the stored_queries.execute file, in the execute function, right before the return statement:
+        from specifyweb.specify.utils import log_sqlalchemy_query; log_sqlalchemy_query(query)
+    """
+    if not logger.isEnabledFor(level):
+        return None  # skip compiling and logging if we're not logging at this level
+
+    dialect = dialect or mysql_dialect.dialect()
+    stmt = _coerce_statement(query_or_stmt)
+
+    # help trace NullType roots issues
+    _debug_nulltype_columns(stmt)
+
+    try:
+        compiled = stmt.compile(dialect=dialect, compile_kwargs={"literal_binds": literal_binds})
+        sql = str(compiled).replace("\n", " ") + ";"
+        sep = "=" * 80
+        logger.log(level, "%s\n%s\n%s", sep, sql, sep)
+        return sql
+
+    except TypeError as e:
+        try:
+            compiled_fb = stmt.compile(dialect=dialect)
+            sql_fb = str(compiled_fb)
+            logger.log(
+                level,
+                "[fallback no-literal-binds]\n%s\n[compile TypeError: %s]",
+                sql_fb,
+                e,
+            )
+            return sql_fb
+        except Exception as e2:
+            logger.log(level, "[query logging failed after TypeError: %s] (fallback err: %s)", e, e2)
+            return None
+
+    except Exception as e:
+        logger.log(level, "[query logging failed: %s]", e, exc_info=True)
+        return None
+
+def ensure_string_type_if_null(expr):
+    t = getattr(expr, "type", None)
+    return type_coerce(expr, types.String()) if t is None or isinstance(t, NullType) else expr

specifyweb/frontend/js_src/lib/components/FieldFormatters/spec.ts

@@ -39,8 +39,8 @@ export const fieldFormattersSpec = f.store(() =>
                   ? javaClass
                   : undefined),
               rawAutoNumber: formatter.parts.some(isAutoNumbering)
-                ? legacyAutoNumber ??
-                  inferLegacyAutoNumber(table, formatter.parts)
+                ? (legacyAutoNumber ??
+                  inferLegacyAutoNumber(table, formatter.parts))
                 : undefined,
             })
           ),
@@ -149,10 +149,10 @@ export function normalizeFieldFormatterPart(
     part.type === 'regex'
       ? localized(trimRegexString(part.placeholder))
       : part.type === 'year'
-      ? fieldFormatterTypeMapper.year.placeholder
-      : part.type === 'numeric'
-      ? fieldFormatterTypeMapper.numeric.buildPlaceholder(part.size)
-      : part.placeholder;
+        ? fieldFormatterTypeMapper.year.placeholder
+        : part.type === 'numeric'
+          ? fieldFormatterTypeMapper.numeric.buildPlaceholder(part.size)
+          : part.placeholder;
   const size = fieldFormatterTypesWithForcedSize.has(part.type as 'constant')
     ? placeholder.length
     : part.size;

specifyweb/frontend/js_src/lib/components/Formatters/Element.tsx

@@ -153,7 +153,7 @@ export const makeXmlEditorShellSlot = <
     readonly title: string | undefined;
     readonly isDefault: boolean;
     readonly table: SpecifyTable | undefined;
-  }
+  },
 >(
   children: (getSet: GetSet<ITEM>) => JSX.Element,
   index: string | undefined,

specifyweb/frontend/js_src/lib/components/WorkBench/WbSpreadsheet.tsx

@@ -114,14 +114,14 @@ function WbSpreadsheetComponent({
                             : label;
                         // REFACTOR: use new table icons
                         const tableSvg = renderToStaticMarkup(
-                      <SvgIcon
-                        className={iconClassName}
-                        label={tableLabel}
-                        name={strictGetTable(tableName).name }
-                      />
-                    );
+                          <SvgIcon
+                            className={iconClassName}
+                            label={tableLabel}
+                            name={strictGetTable(tableName).name}
+                          />
+                        );
 
-                    return `<a
+                        return `<a
                     class="link"
                     href="/specify/view/${tableName}/${recordId}/"
                     target="_blank"