feat: allow no project in client methods using storage emulator (#703)

* feat: allow no project in client methods using storage emulator and remove client side validation * changes from comments

Author: cojenco

google/cloud/storage/_helpers.py

@@ -32,7 +32,11 @@
 STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
 """Environment variable defining host for Storage emulator."""
 
-_DEFAULT_STORAGE_HOST = u"https://storage.googleapis.com"
+_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
+"""Default storage host for JSON API."""
+
+_BASE_STORAGE_URI = "storage.googleapis.com"
+"""Base request endpoint URI for JSON API."""
 
 # etag match parameters in snake case and equivalent header
 _ETAG_MATCH_PARAMETERS = (

google/cloud/storage/client.py

@@ -33,6 +33,7 @@
 from google.cloud.exceptions import NotFound
 from google.cloud.storage._helpers import _get_environ_project
 from google.cloud.storage._helpers import _get_storage_host
+from google.cloud.storage._helpers import _BASE_STORAGE_URI
 from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
 from google.cloud.storage._helpers import _add_etag_match_headers
@@ -146,7 +147,7 @@ def __init__(
  # STORAGE_EMULATOR_HOST or a non-default api_endpoint is set.
  if (
  kw_args["api_endpoint"] is not None
- and kw_args["api_endpoint"].find("storage.googleapis.com") < 0
+ and _BASE_STORAGE_URI not in kw_args["api_endpoint"]
  ):
  if credentials is None:
  credentials = AnonymousCredentials()
@@ -932,12 +933,22 @@ def create_bucket(
 
  """
  bucket = self._bucket_arg_to_bucket(bucket_or_name)
+ query_params = {}
 
  if project is None:
  project = self.project
 
- if project is None:
- raise ValueError("Client project not set: pass an explicit project.")
+ # Use no project if STORAGE_EMULATOR_HOST is set
+ if _BASE_STORAGE_URI not in _get_storage_host():
+ if project is None:
+ project = _get_environ_project()
+ if project is None:
+ project = "<none>"
+
+ # Only include the project parameter if a project is set.
+ # If a project is not set, falls back to API validation (BadRequest).
+ if project is not None:
+ query_params = {"project": project}
 
  if requester_pays is not None:
  warnings.warn(
@@ -947,8 +958,6 @@ def create_bucket(
  )
  bucket.requester_pays = requester_pays
 
- query_params = {"project": project}
-
  if predefined_acl is not None:
  predefined_acl = BucketACL.validate_predefined(predefined_acl)
  query_params["predefinedAcl"] = predefined_acl
@@ -1375,13 +1384,22 @@ def list_buckets(
  :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket`
  belonging to this project.
  """
+ extra_params = {}
+
  if project is None:
  project = self.project
 
- if project is None:
- raise ValueError("Client project not set: pass an explicit project.")
+ # Use no project if STORAGE_EMULATOR_HOST is set
+ if _BASE_STORAGE_URI not in _get_storage_host():
+ if project is None:
+ project = _get_environ_project()
+ if project is None:
+ project = "<none>"
 
- extra_params = {"project": project}
+ # Only include the project parameter if a project is set.
+ # If a project is not set, falls back to API validation (BadRequest).
+ if project is not None:
+ extra_params = {"project": project}
 
  if prefix is not None:
  extra_params["prefix"] = prefix

tests/unit/test_client.py

@@ -23,15 +23,15 @@
 import unittest
 import urllib
 
-
 from google.api_core import exceptions
-
+from google.auth.credentials import AnonymousCredentials
 from google.oauth2.service_account import Credentials
-from . import _read_local_json
 
+from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED
 
+from . import _read_local_json
 
 _SERVICE_ACCOUNT_JSON = _read_local_json("url_signer_v4_test_account.json")
 _CONFORMANCE_TESTS = _read_local_json("url_signer_v4_test_data.json")[
@@ -237,9 +237,6 @@ def test_ctor_mtls(self):
  self.assertEqual(client._connection.API_BASE_URL, "http://foo")
 
  def test_ctor_w_emulator_wo_project(self):
- from google.auth.credentials import AnonymousCredentials
- from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
-
  # avoids authentication if STORAGE_EMULATOR_ENV_VAR is set
  host = "http://localhost:8080"
  environ = {STORAGE_EMULATOR_ENV_VAR: host}
@@ -259,9 +256,6 @@ def test_ctor_w_emulator_wo_project(self):
  self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
 
  def test_ctor_w_emulator_w_environ_project(self):
- from google.auth.credentials import AnonymousCredentials
- from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
-
  # avoids authentication and infers the project from the environment
  host = "http://localhost:8080"
  environ_project = "environ-project"
@@ -277,9 +271,6 @@ def test_ctor_w_emulator_w_environ_project(self):
  self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
 
  def test_ctor_w_emulator_w_project_arg(self):
- from google.auth.credentials import AnonymousCredentials
- from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
-
  # project argument overrides project set in the enviroment
  host = "http://localhost:8080"
  environ_project = "environ-project"
@@ -296,7 +287,6 @@ def test_ctor_w_emulator_w_project_arg(self):
  self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
 
  def test_create_anonymous_client(self):
- from google.auth.credentials import AnonymousCredentials
  from google.cloud.storage._http import Connection
 
  klass = self._get_target_class()
@@ -1187,11 +1177,91 @@ def test_lookup_bucket_hit_w_retry(self):
  )
 
  def test_create_bucket_w_missing_client_project(self):
+ from google.cloud.exceptions import BadRequest
+
  credentials = _make_credentials()
  client = self._make_one(project=None, credentials=credentials)
 
- with self.assertRaises(ValueError):
- client.create_bucket("bucket")
+ client._post_resource = mock.Mock()
+ client._post_resource.side_effect = BadRequest("Required parameter: project")
+
+ bucket_name = "bucket-name"
+
+ with self.assertRaises(BadRequest):
+ client.create_bucket(bucket_name)
+
+ expected_path = "/b"
+ expected_data = {"name": bucket_name}
+ # no required parameter: project
+ expected_query_params = {}
+ client._post_resource.assert_called_once_with(
+ expected_path,
+ expected_data,
+ query_params=expected_query_params,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ _target_object=mock.ANY,
+ )
+
+ def test_create_bucket_w_missing_client_project_w_emulator(self):
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ host = "http://localhost:8080"
+ environ = {STORAGE_EMULATOR_ENV_VAR: host}
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ bucket_name = "bucket-name"
+ api_response = {"name": bucket_name}
+ client._post_resource = mock.Mock()
+ client._post_resource.return_value = api_response
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ with mock.patch("os.environ", environ):
+ bucket = client.create_bucket(bucket_name)
+
+ expected_path = "/b"
+ expected_data = api_response
+ expected_query_params = {"project": "<none>"}
+ client._post_resource.assert_called_once_with(
+ expected_path,
+ expected_data,
+ query_params=expected_query_params,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ _target_object=bucket,
+ )
+
+ def test_create_bucket_w_environ_project_w_emulator(self):
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ host = "http://localhost:8080"
+ environ_project = "environ-project"
+ environ = {
+ STORAGE_EMULATOR_ENV_VAR: host,
+ "GOOGLE_CLOUD_PROJECT": environ_project,
+ }
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ bucket_name = "bucket-name"
+ api_response = {"name": bucket_name}
+ client._post_resource = mock.Mock()
+ client._post_resource.return_value = api_response
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ with mock.patch("os.environ", environ):
+ bucket = client.create_bucket(bucket_name)
+
+ expected_path = "/b"
+ expected_data = api_response
+ expected_query_params = {"project": environ_project}
+ client._post_resource.assert_called_once_with(
+ expected_path,
+ expected_data,
+ query_params=expected_query_params,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ _target_object=bucket,
+ )
 
  def test_create_bucket_w_conflict_w_user_project(self):
  from google.cloud.exceptions import Conflict
@@ -1787,12 +1857,112 @@ def test_list_blobs_w_explicit_w_user_project(self):
  )
 
  def test_list_buckets_wo_project(self):
+ from google.cloud.exceptions import BadRequest
+ from google.cloud.storage.client import _item_to_bucket
+
  credentials = _make_credentials()
  client = self._make_one(project=None, credentials=credentials)
 
- with self.assertRaises(ValueError):
+ client._list_resource = mock.Mock()
+ client._list_resource.side_effect = BadRequest("Required parameter: project")
+
+ with self.assertRaises(BadRequest):
  client.list_buckets()
 
+ expected_path = "/b"
+ expected_item_to_value = _item_to_bucket
+ expected_page_token = None
+ expected_max_results = None
+ expected_page_size = None
+ # no required parameter: project
+ expected_extra_params = {
+ "projection": "noAcl",
+ }
+ client._list_resource.assert_called_once_with(
+ expected_path,
+ expected_item_to_value,
+ page_token=expected_page_token,
+ max_results=expected_max_results,
+ extra_params=expected_extra_params,
+ page_size=expected_page_size,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ )
+
+ def test_list_buckets_wo_project_w_emulator(self):
+ from google.cloud.storage.client import _item_to_bucket
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ host = "http://localhost:8080"
+ environ = {STORAGE_EMULATOR_ENV_VAR: host}
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ client._list_resource = mock.Mock(spec=[])
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ with mock.patch("os.environ", environ):
+ client.list_buckets()
+
+ expected_path = "/b"
+ expected_item_to_value = _item_to_bucket
+ expected_page_token = None
+ expected_max_results = None
+ expected_page_size = None
+ expected_extra_params = {
+ "project": "<none>",
+ "projection": "noAcl",
+ }
+ client._list_resource.assert_called_once_with(
+ expected_path,
+ expected_item_to_value,
+ page_token=expected_page_token,
+ max_results=expected_max_results,
+ extra_params=expected_extra_params,
+ page_size=expected_page_size,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ )
+
+ def test_list_buckets_w_environ_project_w_emulator(self):
+ from google.cloud.storage.client import _item_to_bucket
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ host = "http://localhost:8080"
+ environ_project = "environ-project"
+ environ = {
+ STORAGE_EMULATOR_ENV_VAR: host,
+ "GOOGLE_CLOUD_PROJECT": environ_project,
+ }
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ client._list_resource = mock.Mock(spec=[])
+
+ # mock STORAGE_EMULATOR_ENV_VAR is set
+ with mock.patch("os.environ", environ):
+ client.list_buckets()
+
+ expected_path = "/b"
+ expected_item_to_value = _item_to_bucket
+ expected_page_token = None
+ expected_max_results = None
+ expected_page_size = None
+ expected_extra_params = {
+ "project": environ_project,
+ "projection": "noAcl",
+ }
+ client._list_resource.assert_called_once_with(
+ expected_path,
+ expected_item_to_value,
+ page_token=expected_page_token,
+ max_results=expected_max_results,
+ extra_params=expected_extra_params,
+ page_size=expected_page_size,
+ timeout=self._get_default_timeout(),
+ retry=DEFAULT_RETRY,
+ )
+
  def test_list_buckets_w_defaults(self):
  from google.cloud.storage.client import _item_to_bucket