fix: allow Protobuf 6.x (#1445)

* fix: allow Protobuf 6.x * add prerelease nox session * add python 3.13 * lint * add test dependencies * add test dependencies * fix(deps): require google-crc32c >= 1.1.3 * fix(deps): require requests >= 2.22.0 * add dependencies for system tests * clean up * clean up * clean up * clean up * clean up * fix cover * clean up * Install dependencies needed for system tests * add dependencies for system test * update noxfile config * add credentials * exclude .kokoro/presubmit/prerelease-deps.cfg template * remove obsolete excludes * clean up * clean up * exclude .kokoro/continuous/prerelease-deps.cfg from templates; remove obsolete replacement * migrate prerelease test from presubmit to continuous build

Author: parthea

.kokoro/continuous/prerelease-deps.cfg

@@ -5,3 +5,9 @@ env_vars: {
  key: "NOX_SESSION"
  value: "prerelease_deps"
 }
+
+# Credentials needed to test universe domain.
+env_vars: {
+ key: "SECRET_MANAGER_KEYS"
+ value: "client-library-test-universe-domain-credential"
+}

.kokoro/presubmit/prerelease-deps.cfg

@@ -22,7 +22,7 @@ In order to add a feature:
  documentation.
 
 - The feature must work fully on the following CPython versions:
- 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 on both UNIX and Windows.
+ 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 and 3.13 on both UNIX and Windows.
 
 - The feature must not add unnecessary dependencies (where
  "unnecessary" is of course subjective, but new dependencies should

CONTRIBUTING.rst

@@ -14,11 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Generated by synthtool. DO NOT EDIT!
-
 from __future__ import absolute_import
 import os
 import pathlib
+import re
 import shutil
 
 import nox
@@ -29,16 +28,27 @@
 
 DEFAULT_PYTHON_VERSION = "3.8"
 SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"]
-UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 CONFORMANCE_TEST_PYTHON_VERSIONS = ["3.8"]
 
-_DEFAULT_STORAGE_HOST = "https://storage.googleapis.com"
-
 CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute()
 
 # Error if a python version is missing
 nox.options.error_on_missing_interpreters = True
 
+nox.options.sessions = [
+ "blacken",
+ "conftest_retry",
+ "docfx",
+ "docs",
+ "lint",
+ "lint_setup_py",
+ "system",
+ "unit",
+ # cover must be last to avoid error `No data to report`
+ "cover",
+]
+
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def lint(session):
@@ -159,8 +169,8 @@ def system(session):
  session.install(
  "google-cloud-testutils",
  "google-cloud-iam",
- "google-cloud-pubsub < 2.0.0",
- "google-cloud-kms < 2.0dev",
+ "google-cloud-pubsub",
+ "google-cloud-kms",
  "brotli",
  "-c",
  constraints_path,
@@ -300,3 +310,81 @@ def docfx(session):
  os.path.join("docs", ""),
  os.path.join("docs", "_build", "html", ""),
  )
+
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS[-1])
+@nox.parametrize(
+ "protobuf_implementation",
+ ["python", "upb"],
+)
+def prerelease_deps(session, protobuf_implementation):
+ """Run all tests with prerelease versions of dependencies installed."""
+
+ # Install all test dependencies
+ session.install("mock", "pytest", "pytest-cov", "brotli")
+
+ # Install dependencies needed for system tests
+ session.install(
+ "google-cloud-pubsub",
+ "google-cloud-kms",
+ "google-cloud-testutils",
+ "google-cloud-iam",
+ )
+
+ # Install all dependencies
+ session.install("-e", ".[protobuf, tracing]")
+
+ prerel_deps = [
+ "google-api-core",
+ "google-auth",
+ "google-cloud-core",
+ "google-crc32c",
+ "google-resumable-media",
+ "opentelemetry-api",
+ "protobuf",
+ ]
+
+ package_namespaces = {
+ "google-api-core": "google.api_core",
+ "google-auth": "google.auth",
+ "google-cloud-core": "google.cloud.version",
+ "opentelemetry-api": "opentelemetry.version",
+ "protobuf": "google.protobuf",
+ }
+
+ for dep in prerel_deps:
+ session.install("--pre", "--no-deps", "--upgrade", dep)
+ print(f"Installed {dep}")
+
+ version_namespace = package_namespaces.get(dep)
+
+ if version_namespace:
+ session.run(
+ "python",
+ "-c",
+ f"import {version_namespace}; print({version_namespace}.__version__)",
+ )
+ # Remaining dependencies
+ other_deps = [
+ "requests",
+ ]
+ session.install(*other_deps)
+
+ session.run(
+ "py.test",
+ "tests/unit",
+ env={
+ "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+ },
+ )
+
+ session.run(
+ "py.test",
+ "--verbose",
+ f"--junitxml=system_{session.python}_sponge_log.xml",
+ os.path.join("tests", "system"),
+ *session.posargs,
+ env={
+ "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation,
+ },
+ )

noxfile.py

@@ -26,12 +26,6 @@
 templated_files = common.py_library(
  cov_level=100,
  split_system_tests=True,
- system_test_external_dependencies=[
- "google-cloud-iam",
- "google-cloud-pubsub < 2.0.0",
- # See: https://github.com/googleapis/python-storage/issues/226
- "google-cloud-kms < 2.0dev",
- ],
  intersphinx_dependencies={
  # python-requests url temporary change related to
  # https://github.com/psf/requests/issues/6140#issuecomment-1135071992
@@ -48,7 +42,8 @@
  "README.rst",
  ".kokoro/continuous/continuous.cfg",
  ".kokoro/presubmit/system-3.8.cfg",
- ".kokoro/samples/python3.6", # remove python 3.6 support
+ ".kokoro/presubmit/prerelease-deps.cfg",
+ ".kokoro/continuous/prerelease-deps.cfg",
  ".github/blunderbuss.yml", # blunderbuss assignment to python squad
  ".github/workflows", # exclude gh actions as credentials are needed for tests
  ".github/release-please.yml", # special support for a python2 branch in this repo
@@ -84,12 +79,6 @@
  """omit =
  .nox/*""")
 
-s.replace(
- ".kokoro/release/common.cfg",
- 'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem"',
- 'value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem, client-library-test-universe-domain-credential"'
-)
-
 python.py_samples(skip_readmes=True)
 
 s.shell.run(["nox", "-s", "blacken"], hide_output=False)

owlbot.py

@@ -73,12 +73,15 @@ def get_cloud_kms_key():
  if session == 'py-3.12':
  return ('projects/python-docs-samples-tests-312/locations/us/'
  'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
+ if session == 'py-3.13':
+ return ('projects/python-docs-samples-tests-313/locations/us/'
+ 'keyRings/gcs-kms-key-ring/cryptoKeys/gcs-kms-key')
  return os.environ['CLOUD_KMS_KEY']
 
 
 TEST_CONFIG_OVERRIDE = {
  # You can opt out from the test for specific Python versions.
- 'ignored_versions': ["2.7", "3.6", "3.7", "3.11", "3.12"],
+ 'ignored_versions': ["2.7", "3.6", "3.7", "3.11", "3.12", "3.13"],
 
  # An envvar key for determining the project id to use. Change it
  # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a

samples/snippets/noxfile_config.py

@@ -28,24 +28,24 @@
 # 'Development Status :: 5 - Production/Stable'
 release_status = "Development Status :: 5 - Production/Stable"
 dependencies = [
- "google-auth >= 2.26.1, < 3.0dev",
- "google-api-core >= 2.15.0, <3.0.0dev",
- "google-cloud-core >= 2.4.2, < 3.0dev",
+ "google-auth >= 2.26.1, < 3.0.0",
+ "google-api-core >= 2.15.0, < 3.0.0",
+ "google-cloud-core >= 2.4.2, < 3.0.0",
  # The dependency "google-resumable-media" is no longer used. However, the
  # dependency is still included here to accommodate users who may be
  # importing exception classes from the google-resumable-media without
  # installing it explicitly. See the python-storage README for details on
  # exceptions and importing. Users who are not importing
  # google-resumable-media classes in their application can safely disregard
  # this dependency.
- "google-resumable-media >= 2.7.2",
- "requests >= 2.18.0, < 3.0.0dev",
- "google-crc32c >= 1.0, < 2.0dev",
+ "google-resumable-media >= 2.7.2, < 3.0.0",
+ "requests >= 2.22.0, < 3.0.0",
+ "google-crc32c >= 1.1.3, < 2.0.0",
 ]
 extras = {
- "protobuf": ["protobuf<6.0.0dev"],
+ "protobuf": ["protobuf >= 3.20.2, < 7.0.0"],
  "tracing": [
- "opentelemetry-api >= 1.1.0",
+ "opentelemetry-api >= 1.1.0, < 2.0.0",
  ],
 }
 
@@ -93,6 +93,7 @@
  "Programming Language :: Python :: 3.10",
  "Programming Language :: Python :: 3.11",
  "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
  "Operating System :: OS Independent",
  "Topic :: Internet",
  ],

setup.py

@@ -0,0 +1,15 @@
+# This constraints file is used to check that lower bounds
+# are correct in setup.py
+# List all library dependencies and extras in this file.
+# Pin the version to the lower bound.
+# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0",
+# Then this file should have google-cloud-foo==1.14.0
+google-auth==2.26.1
+google-api-core==2.15.0
+google-cloud-core==2.4.2
+google-resumable-media==2.7.2
+requests==2.22.0
+google-crc32c==1.1.3
+protobuf==3.20.2
+opentelemetry-api==1.1.0
+

testing/constraints-3.13.txt

@@ -59,14 +59,14 @@ def topic_path(storage_client, topic_name):
 
 @pytest.fixture(scope="session")
 def notification_topic(storage_client, publisher_client, topic_path, no_mtls):
- _helpers.retry_429(publisher_client.create_topic)(topic_path)
- policy = publisher_client.get_iam_policy(topic_path)
+ _helpers.retry_429(publisher_client.create_topic)(request={"name": topic_path})
+ policy = publisher_client.get_iam_policy(request={"resource": topic_path})
  binding = policy.bindings.add()
  binding.role = "roles/pubsub.publisher"
  binding.members.append(
  f"serviceAccount:{storage_client.get_service_account_email()}"
  )
- publisher_client.set_iam_policy(topic_path, policy)
+ publisher_client.set_iam_policy(request={"resource": topic_path, "policy": policy})
 
 
 def test_notification_create_minimal(