feat: add bigframes.pandas.options.display.precision option (#1979)

* feat: add `bigframes.pandas.options.display.precision` option feat: when using `repr_mode = "anywidget"`, numeric values align right * include traceback in anywidet warning * add tests for more complex data types * add padding * add dataframe class to table * fix unit test * fix doctest

Author: tswast

bigframes/_config/display_options.py

@@ -26,8 +26,12 @@
 class DisplayOptions:
  __doc__ = vendored_pandas_config.display_options_doc
 
+ # Options borrowed from pandas.
  max_columns: int = 20
- max_rows: int = 25
+ max_rows: int = 10
+ precision: int = 6
+
+ # Options unique to BigQuery DataFrames.
  progress_bar: Optional[str] = "auto"
  repr_mode: Literal["head", "deferred", "anywidget"] = "head"
 
@@ -52,6 +56,8 @@ def pandas_repr(display_options: DisplayOptions):
  display_options.max_columns,
  "display.max_rows",
  display_options.max_rows,
+ "display.precision",
+ display_options.precision,
  "display.show_dimensions",
  True,
  ) as pandas_context:

bigframes/dataframe.py

@@ -23,6 +23,7 @@
 import re
 import sys
 import textwrap
+import traceback
 import typing
 from typing import (
  Callable,
@@ -814,7 +815,9 @@ def _repr_html_(self) -> str:
  except (AttributeError, ValueError, ImportError):
  # Fallback if anywidget is not available
  warnings.warn(
- "Anywidget mode is not available. Please `pip install anywidget traitlets` or `pip install 'bigframes[anywidget]'` to use interactive tables. Falling back to deferred mode."
+ "Anywidget mode is not available. "
+ "Please `pip install anywidget traitlets` or `pip install 'bigframes[anywidget]'` to use interactive tables. "
+ f"Falling back to deferred mode. Error: {traceback.format_exc()}"
  )
  return formatter.repr_query_job(self._compute_dry_run())
 

bigframes/display/anywidget.py

@@ -23,6 +23,7 @@
 import pandas as pd
 
 import bigframes
+import bigframes.display.html
 
 # anywidget and traitlets are optional dependencies. We don't want the import of this
 # module to fail if they aren't installed, though. Instead, we try to limit the surface that
@@ -201,12 +202,9 @@ def _set_table_html(self):
  page_data = cached_data.iloc[start:end]
 
  # Generate HTML table
- self.table_html = page_data.to_html(
- index=False,
- max_rows=None,
+ self.table_html = bigframes.display.html.render_html(
+ dataframe=page_data,
  table_id=f"table-{self._table_id}",
- classes="table table-striped table-hover",
- escape=False,
  )
 
  @traitlets.observe("page")

bigframes/display/html.py

@@ -0,0 +1,79 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""HTML rendering for DataFrames and other objects."""
+
+from __future__ import annotations
+
+import html
+
+import pandas as pd
+import pandas.api.types
+
+from bigframes._config import options
+
+
+def _is_dtype_numeric(dtype) -> bool:
+ """Check if a dtype is numeric for alignment purposes."""
+ return pandas.api.types.is_numeric_dtype(dtype)
+
+
+def render_html(
+ *,
+ dataframe: pd.DataFrame,
+ table_id: str,
+) -> str:
+ """Render a pandas DataFrame to HTML with specific styling."""
+ classes = "dataframe table table-striped table-hover"
+ table_html = [f'<table border="1" class="{classes}" id="{table_id}">']
+ precision = options.display.precision
+
+ # Render table head
+ table_html.append(" <thead>")
+ table_html.append(' <tr style="text-align: left;">')
+ for col in dataframe.columns:
+ table_html.append(
+ f' <th style="text-align: left;"><div style="resize: horizontal; overflow: auto; box-sizing: border-box; width: 100%; height: 100%; padding: 0.5em;">{html.escape(str(col))}</div></th>'
+ )
+ table_html.append(" </tr>")
+ table_html.append(" </thead>")
+
+ # Render table body
+ table_html.append(" <tbody>")
+ for i in range(len(dataframe)):
+ table_html.append(" <tr>")
+ row = dataframe.iloc[i]
+ for col_name, value in row.items():
+ dtype = dataframe.dtypes.loc[col_name] # type: ignore
+ align = "right" if _is_dtype_numeric(dtype) else "left"
+ table_html.append(
+ ' <td style="text-align: {}; padding: 0.5em;">'.format(align)
+ )
+
+ # TODO(b/438181139): Consider semi-exploding ARRAY/STRUCT columns
+ # into multiple rows/columns like the BQ UI does.
+ if pandas.api.types.is_scalar(value) and pd.isna(value):
+ table_html.append(' <em style="color: gray;">&lt;NA&gt;</em>')
+ else:
+ if isinstance(value, float):
+ formatted_value = f"{value:.{precision}f}"
+ table_html.append(f" {html.escape(formatted_value)}")
+ else:
+ table_html.append(f" {html.escape(str(value))}")
+ table_html.append(" </td>")
+ table_html.append(" </tr>")
+ table_html.append(" </tbody>")
+ table_html.append("</table>")
+
+ return "\n".join(table_html)

notebooks/.gitignore

@@ -0,0 +1 @@
+.ipynb_checkpoints/

notebooks/dataframes/anywidget_mode.ipynb

@@ -55,6 +55,7 @@
  "metadata": {},
  "outputs": [],
  "source": [
+ "bpd.options.bigquery.ordering_mode = \"partial\"\n",
  "bpd.options.display.repr_mode = \"anywidget\""
  ]
  },
@@ -75,7 +76,7 @@
  {
  "data": {
  "text/html": [
- "Query job c5fcfd5e-1617-49c8-afa3-86ca21019de4 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:c5fcfd5e-1617-49c8-afa3-86ca21019de4&page=queryresults\">Open Job</a>"
+ "Query job a643d120-4af9-44fc-ba3c-ed461cf1092b is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:a643d120-4af9-44fc-ba3c-ed461cf1092b&page=queryresults\">Open Job</a>"
  ],
  "text/plain": [
  "<IPython.core.display.HTML object>"
@@ -115,7 +116,7 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "Computation deferred. Computation will process 171.4 MB\n"
+ "Computation deferred. Computation will process 44.4 MB\n"
  ]
  }
  ],
@@ -138,27 +139,15 @@
  "id": "ce250157",
  "metadata": {},
  "outputs": [
- {
- "data": {
- "text/html": [
- "Query job ab900a53-5011-4e80-85d5-0ef2958598db is DONE. 171.4 MB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:ab900a53-5011-4e80-85d5-0ef2958598db&page=queryresults\">Open Job</a>"
- ],
- "text/plain": [
- "<IPython.core.display.HTML object>"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "bda63ba739dc4d5f83a5e18eb27b2686",
+ "model_id": "d2d4ef22ea9f414b89ea5bd85f0e6635",
  "version_major": 2,
  "version_minor": 1
  },
  "text/plain": [
- "TableWidget(row_count=5552452, table_html='<table border=\"1\" class=\"dataframe table table-striped table-hover\"…"
+ "TableWidget(page_size=10, row_count=5552452, table_html='<table border=\"1\" class=\"table table-striped table-ho…"
  ]
  },
  "metadata": {},
@@ -185,7 +174,7 @@
  "id": "bb15bab6",
  "metadata": {},
  "source": [
- "Progarmmatic Navigation Demo"
+ "Programmatic Navigation Demo"
  ]
  },
  {
@@ -198,18 +187,18 @@
  "name": "stdout",
  "output_type": "stream",
  "text": [
- "Total pages: 222099\n"
+ "Total pages: 555246\n"
  ]
  },
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "9bffeb73549e48419c3dd895edfe30e8",
+ "model_id": "121e3d2f28004036a922e3a11a08d4b7",
  "version_major": 2,
  "version_minor": 1
  },
  "text/plain": [
- "TableWidget(row_count=5552452, table_html='<table border=\"1\" class=\"dataframe table table-striped table-hover\"…"
+ "TableWidget(page_size=10, row_count=5552452, table_html='<table border=\"1\" class=\"table table-striped table-ho…"
  ]
  },
  "execution_count": 7,
@@ -280,6 +269,14 @@
  "id": "a9d5d13a",
  "metadata": {},
  "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/usr/local/google/home/swast/src/github.com/googleapis/python-bigquery-dataframes/bigframes/core/array_value.py:230: AmbiguousWindowWarning: Window ordering may be ambiguous, this can cause unstable results.\n",
+ " warnings.warn(msg, bfe.AmbiguousWindowWarning)\n"
+ ]
+ },
  {
  "name": "stdout",
  "output_type": "stream",
@@ -290,12 +287,12 @@
  {
  "data": {
  "application/vnd.jupyter.widget-view+json": {
- "model_id": "dfd4fa3a1c6f4b3eb1877cb0e9ba7e94",
+ "model_id": "5ed335bbbc064e5391ea06a9a218642e",
  "version_major": 2,
  "version_minor": 1
  },
  "text/plain": [
- "TableWidget(row_count=5, table_html='<table border=\"1\" class=\"dataframe table table-striped table-hover\" id=\"t…"
+ "TableWidget(page_size=10, row_count=5, table_html='<table border=\"1\" class=\"table table-striped table-hover\" i…"
  ]
  },
  "execution_count": 9,
@@ -305,16 +302,24 @@
  ],
  "source": [
  "# Test with very small dataset\n",
- "small_df = df.head(5)\n",
+ "small_df = df.sort_values([\"name\", \"year\", \"state\"]).head(5)\n",
  "small_widget = TableWidget(small_df)\n",
  "print(f\"Small dataset pages: {math.ceil(small_widget.row_count / small_widget.page_size)}\")\n",
  "small_widget"
  ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c4e5836b-c872-4a9c-b9ec-14f6f338176d",
+ "metadata": {},
+ "outputs": [],
+ "source": []
  }
  ],
  "metadata": {
  "kernelspec": {
- "display_name": "venv",
+ "display_name": "Python 3 (ipykernel)",
  "language": "python",
  "name": "python3"
  },
@@ -328,7 +333,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.10.15"
+ "version": "3.10.16"
  }
  },
  "nbformat": 4,