feat!: make remote_function params keyword-only, and default service account explicit (#1537)

* feat!: make `remote_function` default service account explicit chore!: make `remote_function` params keyword only * pass cloud_function_service_account="default" in unit tests * pass default cloud_build_service_account in the doctests * pass default cloud_function_service_account in the samples * pass cloud_function_service_account="default" to the read_gbq_function doctest * use cloud_function_service_account="default" at more places * remove positional argument usage for remote_function

Author: shobsi

bigframes/functions/_function_session.py

@@ -237,6 +237,7 @@ def _try_delattr(self, func: Callable, attr: str) -> None:
  # https://github.com/ibis-project/ibis/blob/master/ibis/backends/bigquery/udf/__init__.py
  def remote_function(
  self,
+ *,
  input_types: Union[None, type, Sequence[type]] = None,
  output_type: Optional[type] = None,
  session: Optional[Session] = None,
@@ -251,7 +252,7 @@ def remote_function(
  reuse: bool = True,
  name: Optional[str] = None,
  packages: Optional[Sequence[str]] = None,
- cloud_function_service_account: Optional[str] = None,
+ cloud_function_service_account: str,
  cloud_function_kms_key_name: Optional[str] = None,
  cloud_function_docker_repository: Optional[str] = None,
  max_batching_rows: Optional[int] = 1000,
@@ -384,8 +385,8 @@ def remote_function(
  Explicit name of the external package dependencies. Each dependency
  is added to the `requirements.txt` as is, and can be of the form
  supported in https://pip.pypa.io/en/stable/reference/requirements-file-format/.
- cloud_function_service_account (str, Optional):
- Service account to use for the cloud functions. If not provided then
+ cloud_function_service_account (str):
+ Service account to use for the cloud functions. If "default" provided then
  the default service account would be used. See
  https://cloud.google.com/functions/docs/securing/function-identity
  for more details. Please make sure the service account has the
@@ -455,22 +456,12 @@ def remote_function(
  # Some defaults may be used from the session if not provided otherwise.
  session = self._resolve_session(session)
 
- # raise a UserWarning if user does not explicitly set cloud_function_service_account to a
- # user-managed cloud_function_service_account of to default
- msg = bfe.format_message(
- "You have not explicitly set a user-managed `cloud_function_service_account`. "
- "Using the default Compute Engine service account. "
- "In BigFrames 2.0 onwards, you would have to explicitly set `cloud_function_service_account` "
- 'either to a user-managed service account (preferred) or to `"default"` '
- "to use the default Compute Engine service account (discouraged). "
- "See, https://cloud.google.com/functions/docs/securing/function-identity."
- )
-
+ # If the user forces the cloud function service argument to None, throw
+ # an exception
  if cloud_function_service_account is None:
- warnings.warn(msg, stacklevel=2, category=FutureWarning)
-
- if cloud_function_service_account == "default":
- cloud_function_service_account = None
+ raise ValueError(
+ 'You must provide a user managed cloud_function_service_account, or "default" if you would like to let the default service account be used.'
+ )
 
  # A BigQuery client is required to perform BQ operations.
  bigquery_client = self._resolve_bigquery_client(session, bigquery_client)
@@ -615,7 +606,9 @@ def wrapper(func):
  bq_connection_manager,
  cloud_function_region,
  cloud_functions_client,
- cloud_function_service_account,
+ None
+ if cloud_function_service_account == "default"
+ else cloud_function_service_account,
  cloud_function_kms_key_name,
  cloud_function_docker_repository,
  session=session, # type: ignore

bigframes/pandas/__init__.py

@@ -65,14 +65,15 @@
 
 
 def remote_function(
+ *,
  input_types: Union[None, type, Sequence[type]] = None,
  output_type: Optional[type] = None,
  dataset: Optional[str] = None,
  bigquery_connection: Optional[str] = None,
  reuse: bool = True,
  name: Optional[str] = None,
  packages: Optional[Sequence[str]] = None,
- cloud_function_service_account: Optional[str] = None,
+ cloud_function_service_account: str,
  cloud_function_kms_key_name: Optional[str] = None,
  cloud_function_docker_repository: Optional[str] = None,
  max_batching_rows: Optional[int] = 1000,

bigframes/session/__init__.py

@@ -1202,14 +1202,15 @@ def _check_file_size(self, filepath: str):
 
  def remote_function(
  self,
+ *,
  input_types: Union[None, type, Sequence[type]] = None,
  output_type: Optional[type] = None,
  dataset: Optional[str] = None,
  bigquery_connection: Optional[str] = None,
  reuse: bool = True,
  name: Optional[str] = None,
  packages: Optional[Sequence[str]] = None,
- cloud_function_service_account: Optional[str] = None,
+ cloud_function_service_account: str,
  cloud_function_kms_key_name: Optional[str] = None,
  cloud_function_docker_repository: Optional[str] = None,
  max_batching_rows: Optional[int] = 1000,
@@ -1327,8 +1328,8 @@ def remote_function(
  Explicit name of the external package dependencies. Each dependency
  is added to the `requirements.txt` as is, and can be of the form
  supported in https://pip.pypa.io/en/stable/reference/requirements-file-format/.
- cloud_function_service_account (str, Optional):
- Service account to use for the cloud functions. If not provided
+ cloud_function_service_account (str):
+ Service account to use for the cloud functions. If "default" provided
  then the default service account would be used. See
  https://cloud.google.com/functions/docs/securing/function-identity
  for more details. Please make sure the service account has the
@@ -1406,8 +1407,8 @@ def remote_function(
  `bigframes_remote_function` - The bigquery remote function capable of calling into `bigframes_cloud_function`.
  """
  return self._function_session.remote_function(
- input_types,
- output_type,
+ input_types=input_types,
+ output_type=output_type,
  session=self,
  dataset=dataset,
  bigquery_connection=bigquery_connection,
@@ -1499,8 +1500,8 @@ def udf(
  deployed for the user defined code.
  """
  return self._function_session.udf(
- input_types,
- output_type,
+ input_types=input_types,
+ output_type=output_type,
  session=self,
  dataset=dataset,
  bigquery_connection=bigquery_connection,
@@ -1593,7 +1594,7 @@ def read_gbq_function(
  Another use case is to define your own remote function and use it later.
  For example, define the remote function:
 
- >>> @bpd.remote_function()
+ >>> @bpd.remote_function(cloud_function_service_account="default")
  ... def tenfold(num: int) -> float:
  ... return num * 10
 
@@ -1620,7 +1621,7 @@ def read_gbq_function(
  note, row processor implies that the function has only one input
  parameter.
 
- >>> @bpd.remote_function()
+ >>> @bpd.remote_function(cloud_function_service_account="default")
  ... def row_sum(s: bpd.Series) -> float:
  ... return s['a'] + s['b'] + s['c']
 

notebooks/apps/synthetic_data_generation.ipynb

@@ -248,8 +248,8 @@
  },
  "outputs": [],
  "source": [
- "@bpd.remote_function([int], str, packages=['faker', 'pandas'])\n",
- "def data_generator(id):\n",
+ "@bpd.remote_function(packages=['faker', 'pandas'], cloud_function_service_account=\"default\")\n",
+ "def data_generator(id: int) -> str:\n",
  " context = {}\n",
  " exec(code, context)\n",
  " result_df = context.get(\"result_df\")\n",

notebooks/generative_ai/bq_dataframes_llm_code_generation.ipynb

@@ -914,8 +914,8 @@
  },
  "outputs": [],
  "source": [
- "@bf.remote_function([str], str)\n",
- "def extract_code(text: str):\n",
+ "@bf.remote_function(cloud_function_service_account=\"default\")\n",
+ "def extract_code(text: str) -> str:\n",
  " try:\n",
  " res = text[text.find('\\n')+1:text.find('```', 3)]\n",
  " res = res.replace(\"import pandas as pd\", \"import bigframes.pandas as bf\")\n",

notebooks/getting_started/getting_started_bq_dataframes.ipynb

@@ -1485,8 +1485,8 @@
  },
  "outputs": [],
  "source": [
- "@bpd.remote_function([float], str)\n",
- "def get_bucket(num):\n",
+ "@bpd.remote_function(cloud_function_service_account=\"default\")\n",
+ "def get_bucket(num: float) -> str:\n",
  " if not num: return \"NA\"\n",
  " boundary = 4000\n",
  " return \"at_or_above_4000\" if num >= boundary else \"below_4000\""

notebooks/location/regionalized.ipynb

@@ -1475,8 +1475,8 @@
  }
  ],
  "source": [
- "@bpd.remote_function([float], str, bigquery_connection='bigframes-rf-conn')\n",
- "def get_bucket(num):\n",
+ "@bpd.remote_function(bigquery_connection='bigframes-rf-conn', cloud_function_service_account=\"default\")\n",
+ "def get_bucket(num: float) -> str:\n",
  " if not num: return \"NA\"\n",
  " boundary = 4000\n",
  " return \"at_or_above_4000\" if num >= boundary else \"below_4000\""

notebooks/remote_functions/remote_function.ipynb

@@ -174,7 +174,7 @@
  "source": [
  "# User defined function\n",
  "# https://www.codespeedy.com/find-nth-prime-number-in-python/\n",
- "def nth_prime(n):\n",
+ "def nth_prime(n: int) -> int:\n",
  " prime_numbers = [2,3]\n",
  " i=3\n",
  " if(0<n<=2):\n",
@@ -627,8 +627,8 @@
  "\n",
  "# User defined function\n",
  "# https://www.codespeedy.com/find-nth-prime-number-in-python/\n",
- "@pd.remote_function([int], int, reuse=False)\n",
- "def nth_prime(n):\n",
+ "@pd.remote_function(reuse=False, cloud_function_service_account=\"default\")\n",
+ "def nth_prime(n: int) -> int:\n",
  " prime_numbers = [2,3]\n",
  " i=3\n",
  " if(0<n<=2):\n",
@@ -1179,7 +1179,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.10.12"
+ "version": "3.11.4"
  }
  },
  "nbformat": 4,

notebooks/remote_functions/remote_function_usecases.ipynb

@@ -259,7 +259,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False)\n",
+ "@bpd.remote_function(reuse=False, cloud_function_service_account=\"default\")\n",
  "def duration_category(duration_minutes: int) -> str:\n",
  " if duration_minutes < 90:\n",
  " return \"short\"\n",
@@ -466,7 +466,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False)\n",
+ "@bpd.remote_function(reuse=False, cloud_function_service_account=\"default\")\n",
  "def duration_category(duration_minutes: int) -> str:\n",
  " if duration_minutes < 90:\n",
  " return DURATION_CATEGORY_SHORT\n",
@@ -675,7 +675,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False)\n",
+ "@bpd.remote_function(reuse=False, cloud_function_service_account=\"default\")\n",
  "def duration_category(duration_minutes: int) -> str:\n",
  " duration_hours = mymath.ceil(duration_minutes / 60)\n",
  " return f\"{duration_hours}h\"\n",
@@ -886,7 +886,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False)\n",
+ "@bpd.remote_function(reuse=False, cloud_function_service_account=\"default\")\n",
  "def duration_category(duration_minutes: int) -> str:\n",
  " duration_hours = get_hour_ceiling(duration_minutes)\n",
  " return f\"{duration_hours} hrs\"\n",
@@ -1068,7 +1068,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False, packages=[\"cryptography\"])\n",
+ "@bpd.remote_function(reuse=False, packages=[\"cryptography\"], cloud_function_service_account=\"default\")\n",
  "def get_hash(input: str) -> str:\n",
  " from cryptography.fernet import Fernet\n",
  "\n",
@@ -1271,7 +1271,7 @@
  }
  ],
  "source": [
- "@bpd.remote_function(reuse=False, packages=[\"humanize\"])\n",
+ "@bpd.remote_function(reuse=False, packages=[\"humanize\"], cloud_function_service_account=\"default\")\n",
  "def duration_category(duration_minutes: int) -> str:\n",
  " timedelta = dt.timedelta(minutes=duration_minutes)\n",
  " return humanize.naturaldelta(timedelta)\n",
@@ -1442,7 +1442,7 @@
  "name": "python",
  "nbconvert_exporter": "python",
  "pygments_lexer": "ipython3",
- "version": "3.9.19"
+ "version": "3.11.4"
  }
  },
  "nbformat": 4,

notebooks/remote_functions/remote_function_vertex_claude_model.ipynb

@@ -286,7 +286,9 @@
  "source": [
  "@bpd.remote_function(packages=[\"anthropic[vertex]\", \"google-auth[requests]\"],\n",
  " max_batching_rows=1, \n",
- " bigquery_connection=\"bigframes-dev.us-east5.bigframes-rf-conn\") # replace with your connection\n",
+ " bigquery_connection=\"bigframes-dev.us-east5.bigframes-rf-conn\", # replace with your connection\n",
+ " cloud_function_service_account=\"default\",\n",
+ ")\n",
  "def anthropic_transformer(message: str) -> str:\n",
  " from anthropic import AnthropicVertex\n",
  " client = AnthropicVertex(region=LOCATION, project_id=PROJECT)\n",