Backport PR #31748: BUG: Fixed encoding of pd.NA with to_json (#31804)

Co-authored-by: Kendall Masse <[email protected]>

Author: jorisvandenbossche

doc/source/whatsnew/v1.0.2.rst

@@ -25,8 +25,9 @@ Fixed regressions
 Bug fixes
 ~~~~~~~~~
 
--
--
+**I/O**
+
+- Using ``pd.NA`` with :meth:`DataFrame.to_json` now correctly outputs a null value instead of an empty object (:issue:`31615`)
 
 .. ---------------------------------------------------------------------------
 

pandas/_libs/src/ujson/python/objToJSON.c

@@ -54,6 +54,7 @@ static PyTypeObject *cls_dataframe;
 static PyTypeObject *cls_series;
 static PyTypeObject *cls_index;
 static PyTypeObject *cls_nat;
+static PyTypeObject *cls_na;
 PyObject *cls_timedelta;
 
 npy_int64 get_nat(void) { return NPY_MIN_INT64; }
@@ -151,6 +152,7 @@ int PdBlock_iterNext(JSOBJ, JSONTypeContext *);
 void *initObjToJSON(void) {
     PyObject *mod_pandas;
     PyObject *mod_nattype;
+    PyObject *mod_natype;
     PyObject *mod_decimal = PyImport_ImportModule("decimal");
     type_decimal =
         (PyTypeObject *)PyObject_GetAttrString(mod_decimal, "Decimal");
@@ -176,6 +178,12 @@ void *initObjToJSON(void) {
         Py_DECREF(mod_nattype);
     }
 
+    mod_natype = PyImport_ImportModule("pandas._libs.missing");
+    if (mod_natype) {
+        cls_na = (PyTypeObject *)PyObject_GetAttrString(mod_natype, "NAType");
+        Py_DECREF(mod_natype);
+    }
+
     /* Initialise numpy API */
     import_array();
     // GH 31463
@@ -1909,6 +1917,10 @@ void Object_beginTypeContext(JSOBJ _obj, JSONTypeContext *tc) {
                      "%R (0d array) is not JSON serializable at the moment",
                      obj);
         goto INVALID;
+    } else if (PyObject_TypeCheck(obj, cls_na)) {
+        PRINTMARK();
+        tc->type = JT_NULL;
+        return;
     }
 
 ISITERABLE:

pandas/tests/io/json/test_pandas.py

@@ -1640,3 +1640,13 @@ def test_deprecate_numpy_argument_read_json(self):
         with tm.assert_produces_warning(FutureWarning):
             result = read_json(expected.to_json(), numpy=True)
             tm.assert_frame_equal(result, expected)
+
+    def test_json_pandas_na(self):
+        # GH 31615
+        result = pd.DataFrame([[pd.NA]]).to_json()
+        assert result == '{"0":{"0":null}}'
+
+    def test_json_pandas_nulls(self, nulls_fixture):
+        # GH 31615
+        result = pd.DataFrame([[nulls_fixture]]).to_json()
+        assert result == '{"0":{"0":null}}'