feat: make tests run against environments other than prod (#883)

Made changes that override api endpoint information while triggering builds from kokoro . All tests against prod are still successful. Some system tests are currently incompatible with nonprod environments and hence are marked to be skipped if endpoint is being overriden.

Author: rsaksham

.kokoro/build.sh

@@ -33,6 +33,16 @@ export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json
 # Setup project id.
 export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
 
+# Export variable to override api endpoint
+export API_ENDPOINT_OVERRIDE
+
+# Export variable to override api endpoint version
+export API_VERSION_OVERRIDE
+
+# Export dual region locations
+export DUAL_REGION_LOC_1
+export DUAL_REGION_LOC_2
+
 # Remove old nox
 python3 -m pip uninstall --yes --quiet nox-automation
 

google/cloud/storage/_helpers.py

@@ -33,9 +33,14 @@
 STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
 """Environment variable defining host for Storage emulator."""
 
-_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
+_DEFAULT_STORAGE_HOST = os.getenv(
+ "API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com"
+)
 """Default storage host for JSON API."""
 
+_API_VERSION = os.getenv("API_VERSION_OVERRIDE", "v1")
+"""API version of the default storage host"""
+
 _BASE_STORAGE_URI = "storage.googleapis.com"
 """Base request endpoint URI for JSON API."""
 

google/cloud/storage/_http.py

@@ -15,7 +15,6 @@
 """Create / interact with Google Cloud Storage connections."""
 
 import functools
-
 from google.cloud import _http
 from google.cloud.storage import __version__
 from google.cloud.storage import _helpers
@@ -35,7 +34,7 @@ class Connection(_http.JSONConnection):
  :param api_endpoint: (Optional) api endpoint to use.
  """
 
- DEFAULT_API_ENDPOINT = "https://storage.googleapis.com"
+ DEFAULT_API_ENDPOINT = _helpers._DEFAULT_STORAGE_HOST
  DEFAULT_API_MTLS_ENDPOINT = "https://storage.mtls.googleapis.com"
 
  def __init__(self, client, client_info=None, api_endpoint=None):
@@ -52,7 +51,7 @@ def __init__(self, client, client_info=None, api_endpoint=None):
  if agent_version not in self._client_info.user_agent:
  self._client_info.user_agent += f" {agent_version} "
 
- API_VERSION = "v1"
+ API_VERSION = _helpers._API_VERSION
  """The version of the API, used in building the API call's URL."""
 
  API_URL_TEMPLATE = "{api_base_url}/storage/{api_version}{path}"

google/cloud/storage/blob.py

@@ -60,6 +60,8 @@
 from google.cloud.storage._signing import generate_signed_url_v2
 from google.cloud.storage._signing import generate_signed_url_v4
 from google.cloud.storage._helpers import _NUM_RETRIES_MESSAGE
+from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
+from google.cloud.storage._helpers import _API_VERSION
 from google.cloud.storage.acl import ACL
 from google.cloud.storage.acl import ObjectACL
 from google.cloud.storage.constants import _DEFAULT_TIMEOUT
@@ -78,10 +80,12 @@
 from google.cloud.storage.fileio import BlobWriter
 
 
-_API_ACCESS_ENDPOINT = "https://storage.googleapis.com"
+_API_ACCESS_ENDPOINT = _DEFAULT_STORAGE_HOST
 _DEFAULT_CONTENT_TYPE = "application/octet-stream"
-_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/v1{path}?alt=media"
-_BASE_UPLOAD_TEMPLATE = "{hostname}/upload/storage/v1{bucket_path}/o?uploadType="
+_DOWNLOAD_URL_TEMPLATE = "{hostname}/download/storage/{api_version}{path}?alt=media"
+_BASE_UPLOAD_TEMPLATE = (
+ "{hostname}/upload/storage/{api_version}{bucket_path}/o?uploadType="
+)
 _MULTIPART_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "multipart"
 _RESUMABLE_URL_TEMPLATE = _BASE_UPLOAD_TEMPLATE + "resumable"
 # NOTE: "acl" is also writeable but we defer ACL management to
@@ -823,7 +827,9 @@ def _get_download_url(
  name_value_pairs = []
  if self.media_link is None:
  hostname = _get_host_name(client._connection)
- base_url = _DOWNLOAD_URL_TEMPLATE.format(hostname=hostname, path=self.path)
+ base_url = _DOWNLOAD_URL_TEMPLATE.format(
+ hostname=hostname, path=self.path, api_version=_API_VERSION
+ )
  if self.generation is not None:
  name_value_pairs.append(("generation", f"{self.generation:d}"))
  else:
@@ -1838,7 +1844,7 @@ def _do_multipart_upload(
 
  hostname = _get_host_name(client._connection)
  base_url = _MULTIPART_URL_TEMPLATE.format(
- hostname=hostname, bucket_path=self.bucket.path
+ hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
  )
  name_value_pairs = []
 
@@ -2025,7 +2031,7 @@ def _initiate_resumable_upload(
 
  hostname = _get_host_name(client._connection)
  base_url = _RESUMABLE_URL_TEMPLATE.format(
- hostname=hostname, bucket_path=self.bucket.path
+ hostname=hostname, bucket_path=self.bucket.path, api_version=_API_VERSION
  )
  name_value_pairs = []
 

noxfile.py

@@ -111,6 +111,7 @@ def system(session):
  """Run the system test suite."""
  system_test_path = os.path.join("tests", "system.py")
  system_test_folder_path = os.path.join("tests", "system")
+ rerun_count = 0
 
  # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true.
  if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false":
@@ -121,6 +122,12 @@ def system(session):
  # mTLS tests requires pyopenssl.
  if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true":
  session.install("pyopenssl")
+ # Check if endpoint is being overriden for rerun_count
+ if (
+ os.getenv("API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com")
+ != "https://storage.googleapis.com"
+ ):
+ rerun_count = 3
 
  system_test_exists = os.path.exists(system_test_path)
  system_test_folder_exists = os.path.exists(system_test_folder_path)
@@ -138,7 +145,7 @@ def system(session):
  # 2021-05-06: defer installing 'google-cloud-*' to after this package,
  # in order to work around Python 2.7 googolapis-common-protos
  # issue.
- session.install("mock", "pytest", "-c", constraints_path)
+ session.install("mock", "pytest", "pytest-rerunfailures", "-c", constraints_path)
  session.install("-e", ".", "-c", constraints_path)
  session.install(
  "google-cloud-testutils",
@@ -151,9 +158,21 @@ def system(session):
 
  # Run py.test against the system tests.
  if system_test_exists:
- session.run("py.test", "--quiet", system_test_path, *session.posargs)
+ session.run(
+ "py.test",
+ "--quiet",
+ "--reruns={}".format(rerun_count),
+ system_test_path,
+ *session.posargs,
+ )
  if system_test_folder_exists:
- session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)
+ session.run(
+ "py.test",
+ "--quiet",
+ "--reruns={}".format(rerun_count),
+ system_test_folder_path,
+ *session.posargs,
+ )
 
 
 @nox.session(python=CONFORMANCE_TEST_PYTHON_VERSIONS)

tests/system/_helpers.py

@@ -20,6 +20,7 @@
 from test_utils.retry import RetryErrors
 from test_utils.retry import RetryInstanceState
 from test_utils.system import unique_resource_id
+from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
 
 retry_429 = RetryErrors(exceptions.TooManyRequests)
 retry_429_harder = RetryErrors(exceptions.TooManyRequests, max_tries=10)
@@ -31,6 +32,7 @@
 user_project = os.environ.get("GOOGLE_CLOUD_TESTS_USER_PROJECT")
 testing_mtls = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE") == "true"
 signing_blob_content = b"This time for sure, Rocky!"
+is_api_endpoint_override = _DEFAULT_STORAGE_HOST != "https://storage.googleapis.com"
 
 
 def _bad_copy(bad_request):

tests/system/test__signing.py

@@ -17,7 +17,7 @@
 import hashlib
 import os
 import time
-
+import pytest
 import requests
 
 from google.api_core import path_template
@@ -41,7 +41,11 @@ def _create_signed_list_blobs_url_helper(
  expiration = _morph_expiration(version, expiration)
 
  signed_url = bucket.generate_signed_url(
- expiration=expiration, method=method, client=client, version=version
+ expiration=expiration,
+ method=method,
+ client=client,
+ version=version,
+ api_access_endpoint=_helpers._DEFAULT_STORAGE_HOST,
  )
 
  response = requests.get(signed_url)
@@ -371,6 +375,10 @@ def test_create_signed_resumable_upload_url_v4(storage_client, signing_bucket, n
  )
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_generate_signed_post_policy_v4(
  storage_client, buckets_to_delete, blobs_to_delete, service_account, no_mtls
 ):

tests/system/test_blob.py

@@ -97,6 +97,10 @@ def test_large_file_write_from_stream_w_failed_checksum(
  assert not blob.exists()
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_large_file_write_from_stream_w_encryption_key(
  storage_client,
  shared_bucket,

tests/system/test_bucket.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 import datetime
-
 import pytest
 
 from google.api_core import exceptions
@@ -124,6 +123,10 @@ def test_bucket_lifecycle_rules(storage_client, buckets_to_delete):
  assert list(bucket.lifecycle_rules) == []
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_bucket_update_labels(storage_client, buckets_to_delete):
  bucket_name = _helpers.unique_name("update-labels")
  bucket = _helpers.retry_429_503(storage_client.create_bucket)(bucket_name)
@@ -797,6 +800,10 @@ def test_bucket_lock_retention_policy(
  bucket.patch()
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_new_bucket_w_ubla(
  storage_client,
  buckets_to_delete,
@@ -966,6 +973,10 @@ def test_new_bucket_created_w_enforced_pap(
  assert not bucket.iam_configuration.uniform_bucket_level_access_enabled
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_new_bucket_with_rpo(
  storage_client,
  buckets_to_delete,

tests/system/test_client.py

@@ -14,6 +14,7 @@
 
 import io
 import re
+import os
 import tempfile
 
 import pytest
@@ -23,9 +24,15 @@
 from . import _helpers
 
 
+dual_data_loc_1 = os.getenv("DUAL_REGION_LOC_1", "US-EAST1")
+dual_data_loc_2 = os.getenv("DUAL_REGION_LOC_2", "US-WEST1")
 public_bucket = "gcp-public-data-landsat"
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 @vpcsc_config.skip_if_inside_vpcsc
 def test_anonymous_client_access_to_public_bucket():
  from google.cloud.storage.client import Client
@@ -40,6 +47,10 @@ def test_anonymous_client_access_to_public_bucket():
  _helpers.retry_429_503(blob.download_to_file)(stream)
 
 
+@pytest.mark.skipif(
+ _helpers.is_api_endpoint_override,
+ reason="Test does not yet support endpoint override",
+)
 def test_get_service_account_email(storage_client, service_account):
  domain = "gs-project-accounts.iam.gserviceaccount.com"
  email = storage_client.get_service_account_email()
@@ -69,7 +80,8 @@ def test_create_bucket_dual_region(storage_client, buckets_to_delete):
 
  new_bucket_name = _helpers.unique_name("dual-region-bucket")
  location = "US"
- data_locations = ["US-EAST1", "US-WEST1"]
+
+ data_locations = [dual_data_loc_1, dual_data_loc_2]
 
  with pytest.raises(exceptions.NotFound):
  storage_client.get_bucket(new_bucket_name)