feat: avoid authentication with storage emulator (#679)

* feat: avoid authentication with storage emulator * add unit tests * add more unit tests * add support for corner case project set in envvar * add _helpers unit test * allow flexibility for locational endpoints

Author: cojenco

google/cloud/storage/_helpers.py

@@ -23,6 +23,7 @@
 from urllib.parse import urlsplit
 
 from google import resumable_media
+from google.auth import environment_vars
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
 from google.cloud.storage.retry import DEFAULT_RETRY
 from google.cloud.storage.retry import DEFAULT_RETRY_IF_METAGENERATION_SPECIFIED
@@ -62,6 +63,12 @@ def _get_storage_host():
  return os.environ.get(STORAGE_EMULATOR_ENV_VAR, _DEFAULT_STORAGE_HOST)
 
 
+def _get_environ_project():
+ return os.getenv(
+ environment_vars.PROJECT, os.getenv(environment_vars.LEGACY_PROJECT),
+ )
+
+
 def _validate_name(name):
  """Pre-flight ``Bucket`` name validation.
 

google/cloud/storage/client.py

@@ -31,6 +31,7 @@
 from google.cloud._helpers import _LocalStack, _NOW
 from google.cloud.client import ClientWithProject
 from google.cloud.exceptions import NotFound
+from google.cloud.storage._helpers import _get_environ_project
 from google.cloud.storage._helpers import _get_storage_host
 from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 from google.cloud.storage._helpers import _bucket_bound_hostname_url
@@ -121,13 +122,6 @@ def __init__(
  if project is _marker:
  project = None
 
- super(Client, self).__init__(
- project=project,
- credentials=credentials,
- client_options=client_options,
- _http=_http,
- )
-
  kw_args = {"client_info": client_info}
 
  # `api_endpoint` should be only set by the user via `client_options`,
@@ -148,6 +142,27 @@ def __init__(
  api_endpoint = client_options.api_endpoint
  kw_args["api_endpoint"] = api_endpoint
 
+ # Use anonymous credentials and no project when
+ # STORAGE_EMULATOR_HOST or a non-default api_endpoint is set.
+ if (
+ kw_args["api_endpoint"] is not None
+ and kw_args["api_endpoint"].find("storage.googleapis.com") < 0
+ ):
+ if credentials is None:
+ credentials = AnonymousCredentials()
+ if project is None:
+ project = _get_environ_project()
+ if project is None:
+ no_project = True
+ project = "<none>"
+
+ super(Client, self).__init__(
+ project=project,
+ credentials=credentials,
+ client_options=client_options,
+ _http=_http,
+ )
+
  if no_project:
  self.project = None
 

tests/unit/test__helpers.py

@@ -46,6 +46,34 @@ def test_w_env_var(self):
  self.assertEqual(host, HOST)
 
 
+class Test__get_environ_project(unittest.TestCase):
+ @staticmethod
+ def _call_fut():
+ from google.cloud.storage._helpers import _get_environ_project
+
+ return _get_environ_project()
+
+ def test_wo_env_var(self):
+ with mock.patch("os.environ", {}):
+ project = self._call_fut()
+
+ self.assertEqual(project, None)
+
+ def test_w_env_var(self):
+ from google.auth import environment_vars
+
+ PROJECT = "environ-project"
+
+ with mock.patch("os.environ", {environment_vars.PROJECT: PROJECT}):
+ project = self._call_fut()
+ self.assertEqual(project, PROJECT)
+
+ with mock.patch("os.environ", {environment_vars.LEGACY_PROJECT: PROJECT}):
+ project = self._call_fut()
+
+ self.assertEqual(project, PROJECT)
+
+
 class Test_PropertyMixin(unittest.TestCase):
  @staticmethod
  def _get_default_timeout():

tests/unit/test_client.py

@@ -236,6 +236,65 @@ def test_ctor_mtls(self):
  self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, False)
  self.assertEqual(client._connection.API_BASE_URL, "http://foo")
 
+ def test_ctor_w_emulator_wo_project(self):
+ from google.auth.credentials import AnonymousCredentials
+ from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
+
+ # avoids authentication if STORAGE_EMULATOR_ENV_VAR is set
+ host = "http://localhost:8080"
+ environ = {STORAGE_EMULATOR_ENV_VAR: host}
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ self.assertIsNone(client.project)
+ self.assertEqual(client._connection.API_BASE_URL, host)
+ self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
+
+ # avoids authentication if storage emulator is set through api_endpoint
+ client = self._make_one(
+ client_options={"api_endpoint": "http://localhost:8080"}
+ )
+ self.assertIsNone(client.project)
+ self.assertEqual(client._connection.API_BASE_URL, host)
+ self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
+
+ def test_ctor_w_emulator_w_environ_project(self):
+ from google.auth.credentials import AnonymousCredentials
+ from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
+
+ # avoids authentication and infers the project from the environment
+ host = "http://localhost:8080"
+ environ_project = "environ-project"
+ environ = {
+ STORAGE_EMULATOR_ENV_VAR: host,
+ "GOOGLE_CLOUD_PROJECT": environ_project,
+ }
+ with mock.patch("os.environ", environ):
+ client = self._make_one()
+
+ self.assertEqual(client.project, environ_project)
+ self.assertEqual(client._connection.API_BASE_URL, host)
+ self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
+
+ def test_ctor_w_emulator_w_project_arg(self):
+ from google.auth.credentials import AnonymousCredentials
+ from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
+
+ # project argument overrides project set in the enviroment
+ host = "http://localhost:8080"
+ environ_project = "environ-project"
+ project = "my-test-project"
+ environ = {
+ STORAGE_EMULATOR_ENV_VAR: host,
+ "GOOGLE_CLOUD_PROJECT": environ_project,
+ }
+ with mock.patch("os.environ", environ):
+ client = self._make_one(project=project)
+
+ self.assertEqual(client.project, project)
+ self.assertEqual(client._connection.API_BASE_URL, host)
+ self.assertIsInstance(client._connection.credentials, AnonymousCredentials)
+
  def test_create_anonymous_client(self):
  from google.auth.credentials import AnonymousCredentials
  from google.cloud.storage._http import Connection