Apache Arrow tables

Author: mbostock

src/dependencies.mjs

@@ -13,7 +13,8 @@ export const sql = dependency("sql.js", "1.7.0", "dist/sql-wasm.js");
 export const vega = dependency("vega", "5.22.1", "build/vega.min.js");
 export const vegalite = dependency("vega-lite", "5.5.0", "build/vega-lite.min.js");
 export const vegaliteApi = dependency("vega-lite-api", "5.0.0", "build/vega-lite-api.min.js");
-export const arrow = dependency("apache-arrow", "4.0.1", "Arrow.es2015.min.js");
+export const arrow4 = dependency("apache-arrow", "4.0.1", "Arrow.es2015.min.js");
+export const arrow10 = dependency("apache-arrow", "10.0.0", "Arrow.es2015.min.js");
 export const arquero = dependency("arquero", "4.8.8", "dist/arquero.min.js");
 export const topojson = dependency("topojson-client", "3.1.0", "dist/topojson-client.min.js");
 export const exceljs = dependency("exceljs", "4.3.0", "dist/exceljs.min.js");

src/fileAttachment.mjs

@@ -1,5 +1,5 @@
 import {autoType, csvParse, csvParseRows, tsvParse, tsvParseRows} from "d3-dsv";
-import {arrow, jszip, exceljs} from "./dependencies.mjs";
+import {arrow4, arrow10, jszip, exceljs} from "./dependencies.mjs";
 import {requireDefault} from "./require.mjs";
 import {SQLiteDatabaseClient} from "./sqlite.mjs";
 import {Workbook} from "./xlsx.mjs";
@@ -56,9 +56,18 @@ export class AbstractFile {
       i.src = url;
     });
   }
-  async arrow() {
-    const [Arrow, response] = await Promise.all([requireDefault(arrow.resolve()), remote_fetch(this)]);
-    return Arrow.Table.from(response);
+  async arrow({version = 4} = {}) {
+    switch (version) {
+      case 4: {
+        const [Arrow, response] = await Promise.all([requireDefault(arrow4.resolve()), remote_fetch(this)]);
+        return Arrow.Table.from(response);
+      }
+      case 10: {
+        const [Arrow, response] = await Promise.all([requireDefault(arrow10.resolve()), remote_fetch(this)]);
+        return Arrow.tableFromIPC(response);
+      }
+      default: throw new Error(`unsupported arrow version: ${version}`);
+    }
   }
   async sqlite() {
     return SQLiteDatabaseClient.open(remote_fetch(this));

src/index.mjs

@@ -1,3 +1,3 @@
 export {default as FileAttachments, AbstractFile} from "./fileAttachment.mjs";
 export {default as Library} from "./library.mjs";
-export {makeQueryTemplate, loadDataSource, arrayIsPrimitive, isDataArray, isDatabaseClient} from "./table.mjs";
+export {makeQueryTemplate, loadDataSource, arrayIsPrimitive, isArrowTable, isDataArray, isDatabaseClient} from "./table.mjs";

src/library.mjs

@@ -17,7 +17,7 @@ import svg from "./svg.mjs";
 import tex from "./tex.mjs";
 import vegalite from "./vegalite.mjs";
 import width from "./width.mjs";
-import {arquero, arrow, d3, graphviz, htl, inputs, lodash, plot, topojson} from "./dependencies.mjs";
+import {arquero, arrow4, d3, graphviz, htl, inputs, lodash, plot, topojson} from "./dependencies.mjs";
 import {__query} from "./table.mjs";
 
 export default Object.assign(Object.defineProperties(function Library(resolver) {
@@ -39,8 +39,8 @@ export default Object.assign(Object.defineProperties(function Library(resolver)
     // Recommended libraries
     // https://observablehq.com/@observablehq/recommended-libraries
     _: () => require(lodash.resolve()),
-    aq: () => require.alias({"apache-arrow": arrow.resolve()})(arquero.resolve()),
-    Arrow: () => require(arrow.resolve()),
+    aq: () => require.alias({"apache-arrow": arrow4.resolve()})(arquero.resolve()), // TODO arquero 5
+    Arrow: () => require(arrow4.resolve()), // TODO arrow10
     d3: () => require(d3.resolve()),
     Inputs: () => require(inputs.resolve()).then(Inputs => ({...Inputs, file: Inputs.fileOf(AbstractFile)})),
     L: () => leaflet(require),

src/table.mjs

@@ -23,6 +23,16 @@ export function isDatabaseClient(value, mode) {
   );
 }
 
+// Returns true if the vaue is an Apache Arrow table.
+export function isArrowTable(value) {
+  return (
+    value &&
+    value.schema &&
+    Array.isArray(value.schema.fields) &&
+    typeof value[Symbol.iterator] === "function"
+  );
+}
+
 // Returns true if the value is a typed array (for a single-column table), or if
 // it’s an array. In the latter case, the elements of the array must be
 // consistently typed: either plain objects or primitives or dates.
@@ -145,6 +155,7 @@ export const __query = Object.assign(
     source = await loadDataSource(await source, "table");
     if (isDatabaseClient(source)) return evaluateQuery(source, makeQueryTemplate(operations, source), invalidation);
     if (isDataArray(source)) return __table(source, operations);
+    if (isArrowTable(source)) return __arrow(source, operations);
     if (!source) throw new Error("missing data source");
     throw new Error("invalid data source");
   },
@@ -164,6 +175,7 @@ export async function loadDataSource(source, mode) {
         case "text/csv": return source.csv({typed: true});
         case "text/tab-separated-values": return source.tsv({typed: true});
         case "application/json": return source.json();
+        default: if (/\.arrow$/i.test(source.name)) return source.arrow({version: 10});
       }
     }
     if (mode === "table" || mode === "sql") {
@@ -390,8 +402,17 @@ function likeOperand(operand) {
   return {...operand, value: `%${operand.value}%`};
 }
 
+// This function applies table cell operations to an in-memory Apache Arrow
+// table; it should be equivalent to the corresponding SQL query.
+function __arrow(source, operations) {
+  operations;
+  return source; // TODO
+}
+
 // This function applies table cell operations to an in-memory table (array of
-// objects); it should be equivalent to the corresponding SQL query.
+// objects); it should be equivalent to the corresponding SQL query. TODO This
+// is only exported for testing, but we should be testing the public __query
+// instead of this internal method.
 export function __table(source, operations) {
   const input = source;
   let {schema, columns} = source;