From ecc2b3f124eb801975f42b5a19d2d89d6bb830de Mon Sep 17 00:00:00 2001
From: Sean Cheren <scheren@wolfram.com>
Date: Tue, 13 Feb 2024 23:28:16 -0600
Subject: [PATCH 1/4] attempt to return pandas DataFrame using ArrowIPC

---
 wolframclient/serializers/encoders/pandas.py | 13 ++++++++++++-
 wolframclient/utils/api.py                   |  6 ++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/wolframclient/serializers/encoders/pandas.py b/wolframclient/serializers/encoders/pandas.py
index 089998d0..c0b6df71 100644
--- a/wolframclient/serializers/encoders/pandas.py
+++ b/wolframclient/serializers/encoders/pandas.py
@@ -4,6 +4,7 @@
 from itertools import starmap
 
 from wolframclient.utils.api import pandas
+from wolframclient.utils.api import pyarrow
 from wolframclient.utils.dispatch import Dispatch
 
 encoder = Dispatch()
@@ -87,7 +88,7 @@ def encode_multiindex_as_dataset(serializer, o, length):
 
 PANDAS_PROPERTIES = {
     "pandas_series_head": {"dataset", "list", "association"},
-    "pandas_dataframe_head": {"dataset", "association"},
+    "pandas_dataframe_head": {"dataset", "association", "tabular"},
     "timeseries": True,
 }
 
@@ -169,11 +170,21 @@ def encode_dataframe_as_dataset(serializer, o, length):
     )
 
 
+def encode_dataframe_as_tabular(serializer, o):
+    sink = pyarrow.BufferOutputStream()
+    batch = pyarrow.record_batch(o)
+    strm = pyarrow.ipc.new_stream(sink, batch.schema)
+    strm.write_batch(batch)
+    buf = sink.getvalue()
+    return serializer.serialize_bytes(buf)
+
 @encoder.dispatch(pandas.DataFrame)
 def encoder_panda_dataframe(serializer, o):
     head = serializer.get_property("pandas_dataframe_head", d=None)
     if head is None or head == "dataset":
         return encode_dataframe_as_dataset(serializer, o, safe_pandas_length(o.index))
+    elif head == "tabular":
+        return encode_dataframe_as_tabular(serializer, o)
     elif head in PANDAS_PROPERTIES["pandas_dataframe_head"]:
         return encode_dataframe_as_assoc(serializer, o, safe_pandas_length(o.index))
     else:
diff --git a/wolframclient/utils/api.py b/wolframclient/utils/api.py
index 527d5482..5a5a1be1 100644
--- a/wolframclient/utils/api.py
+++ b/wolframclient/utils/api.py
@@ -167,6 +167,12 @@
     bdate_range="pandas.bdate_range",
 )
 
+pyarrow = API(
+    BufferOutputStream="pyarrow.BufferOutputStream",
+    record_batch="pyarrow.record_batch",
+    new_stream="pyarrow.ipc.new_stream"
+)
+
 aiohttp = API(
     Response="aiohttp.web.Response",
     ClientResponse="aiohttp.ClientResponse",

From 21c02548fbbf9ca7b5093c5d332cbe8223592acf Mon Sep 17 00:00:00 2001
From: Sean Cheren <scheren@wolfram.com>
Date: Tue, 20 Feb 2024 15:29:15 -0600
Subject: [PATCH 2/4] include pyarrow.ipc

---
 wolframclient/utils/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wolframclient/utils/api.py b/wolframclient/utils/api.py
index 5a5a1be1..ddca9b7f 100644
--- a/wolframclient/utils/api.py
+++ b/wolframclient/utils/api.py
@@ -170,7 +170,7 @@
 pyarrow = API(
     BufferOutputStream="pyarrow.BufferOutputStream",
     record_batch="pyarrow.record_batch",
-    new_stream="pyarrow.ipc.new_stream"
+    ipc="pyarrow.ipc"
 )
 
 aiohttp = API(

From 1ac31614a7753475aa7cdb3b0239fa25f568aae3 Mon Sep 17 00:00:00 2001
From: Sean Cheren <scheren@wolfram.com>
Date: Thu, 22 Feb 2024 16:34:54 -0600
Subject: [PATCH 3/4] * Add pyarrow.Table -> Tabular implementation * pandas
 DataFrame serializes ImportByteArray call now.. if we ever figure out how to
 set pandas_dataframe_head

---
 wolframclient/serializers/encoder.py          |  1 +
 wolframclient/serializers/encoders/pandas.py  |  5 ++++-
 wolframclient/serializers/encoders/pyarrow.py | 18 ++++++++++++++++++
 wolframclient/utils/api.py                    |  1 +
 4 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 wolframclient/serializers/encoders/pyarrow.py

diff --git a/wolframclient/serializers/encoder.py b/wolframclient/serializers/encoder.py
index bd6c6cc6..11c9b6d4 100644
--- a/wolframclient/serializers/encoder.py
+++ b/wolframclient/serializers/encoder.py
@@ -80,6 +80,7 @@ def update_dispatch(self):
     # wolfram language support
     numpy="wolframclient.serializers.encoders.numpy.encoder",
     pandas="wolframclient.serializers.encoders.pandas.encoder",
+    pyarrow="wolframclient.serializers.encoders.pyarrow.encoder",
     PIL=(
         "wolframclient.serializers.encoders.pil.encoder",
         "wolframclient.serializers.encoders.numpy.encoder",
diff --git a/wolframclient/serializers/encoders/pandas.py b/wolframclient/serializers/encoders/pandas.py
index c0b6df71..d5bbdb6d 100644
--- a/wolframclient/serializers/encoders/pandas.py
+++ b/wolframclient/serializers/encoders/pandas.py
@@ -176,7 +176,10 @@ def encode_dataframe_as_tabular(serializer, o):
     strm = pyarrow.ipc.new_stream(sink, batch.schema)
     strm.write_batch(batch)
     buf = sink.getvalue()
-    return serializer.serialize_bytes(buf)
+    return serializer.serialize_function(
+        serializer.serialize_symbol(b"ImportByteArray"),
+        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC"))
+    )
 
 @encoder.dispatch(pandas.DataFrame)
 def encoder_panda_dataframe(serializer, o):
diff --git a/wolframclient/serializers/encoders/pyarrow.py b/wolframclient/serializers/encoders/pyarrow.py
new file mode 100644
index 00000000..26d7d0bc
--- /dev/null
+++ b/wolframclient/serializers/encoders/pyarrow.py
@@ -0,0 +1,18 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+from wolframclient.utils.api import pyarrow
+from wolframclient.utils.dispatch import Dispatch
+
+encoder = Dispatch()
+
+@encoder.dispatch(pyarrow.Table)
+def encoder_pyarrow_table(serializer, o):
+    sink = pyarrow.BufferOutputStream()
+    batch = o.to_batches()[0]
+    strm = pyarrow.ipc.new_stream(sink, o.schema)
+    strm.write_batch(batch)
+    buf = sink.getvalue()
+    return serializer.serialize_function(
+        serializer.serialize_symbol(b"ImportByteArray"),
+        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC"))
+    )
\ No newline at end of file
diff --git a/wolframclient/utils/api.py b/wolframclient/utils/api.py
index ddca9b7f..cc5d381f 100644
--- a/wolframclient/utils/api.py
+++ b/wolframclient/utils/api.py
@@ -168,6 +168,7 @@
 )
 
 pyarrow = API(
+    Table="pyarrow.Table",
     BufferOutputStream="pyarrow.BufferOutputStream",
     record_batch="pyarrow.record_batch",
     ipc="pyarrow.ipc"

From 26c7ba0deb054826bbc1d37364d608d73cfe9430 Mon Sep 17 00:00:00 2001
From: riccardodivirgilio <riccardodivirgilio@gmail.com>
Date: Thu, 14 Mar 2024 14:24:43 +0100
Subject: [PATCH 4/4] automatic code lint

---
 wolframclient/serializers/encoders/pandas.py  | 6 +++---
 wolframclient/serializers/encoders/pyarrow.py | 5 +++--
 wolframclient/utils/api.py                    | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/wolframclient/serializers/encoders/pandas.py b/wolframclient/serializers/encoders/pandas.py
index 0557d968..5af11b3e 100644
--- a/wolframclient/serializers/encoders/pandas.py
+++ b/wolframclient/serializers/encoders/pandas.py
@@ -3,8 +3,7 @@
 from collections import OrderedDict
 from itertools import starmap
 
-from wolframclient.utils.api import pandas
-from wolframclient.utils.api import pyarrow
+from wolframclient.utils.api import pandas, pyarrow
 from wolframclient.utils.dispatch import Dispatch
 
 encoder = Dispatch()
@@ -179,9 +178,10 @@ def encode_dataframe_as_tabular(serializer, o):
     buf = sink.getvalue()
     return serializer.serialize_function(
         serializer.serialize_symbol(b"ImportByteArray"),
-        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC"))
+        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC")),
     )
 
+
 @encoder.dispatch(pandas.DataFrame)
 def encoder_panda_dataframe(serializer, o):
     head = serializer.get_property("pandas_dataframe_head", d=None)
diff --git a/wolframclient/serializers/encoders/pyarrow.py b/wolframclient/serializers/encoders/pyarrow.py
index 26d7d0bc..a7c6c469 100644
--- a/wolframclient/serializers/encoders/pyarrow.py
+++ b/wolframclient/serializers/encoders/pyarrow.py
@@ -5,6 +5,7 @@
 
 encoder = Dispatch()
 
+
 @encoder.dispatch(pyarrow.Table)
 def encoder_pyarrow_table(serializer, o):
     sink = pyarrow.BufferOutputStream()
@@ -14,5 +15,5 @@ def encoder_pyarrow_table(serializer, o):
     buf = sink.getvalue()
     return serializer.serialize_function(
         serializer.serialize_symbol(b"ImportByteArray"),
-        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC"))
-    )
\ No newline at end of file
+        (serializer.serialize_bytes(buf), serializer.serialize_string("ArrowIPC")),
+    )
diff --git a/wolframclient/utils/api.py b/wolframclient/utils/api.py
index 6020c46e..f7cc84c5 100644
--- a/wolframclient/utils/api.py
+++ b/wolframclient/utils/api.py
@@ -172,7 +172,7 @@
     Table="pyarrow.Table",
     BufferOutputStream="pyarrow.BufferOutputStream",
     record_batch="pyarrow.record_batch",
-    ipc="pyarrow.ipc"
+    ipc="pyarrow.ipc",
 )
 
 aiohttp = API(
