CI for boto: fix errors; add coverage; add skip for uncatchable ResourceWarning

.travis.yml

@@ -90,6 +90,12 @@ before_install:
  - uname -a
  - git --version
  - git tag
+ # Because travis runs on Google Cloud and has a /etc/boto.cfg,
+ # it breaks moto import, see:
+ # https://github.com/spulec/moto/issues/1771
+ # https://github.com/boto/boto/issues/3741
+ # This overrides travis and tells it to look nowhere.
+ - export BOTO_CONFIG=/dev/null
 
 install:
  - echo "install start"

ci/deps/travis-36.yaml

@@ -4,6 +4,7 @@ channels:
  - conda-forge
 dependencies:
  - beautifulsoup4
+ - botocore>=1.11
  - cython>=0.28.2
  - dask
  - fastparquet
@@ -35,10 +36,10 @@ dependencies:
  - pytest
  - pytest-xdist
  - pytest-cov
- - moto
  - hypothesis>=3.58.0
  - pip:
  - brotlipy
  - coverage
+ - moto
  - pandas-datareader
  - python-dateutil

ci/deps/travis-37.yaml

@@ -5,6 +5,7 @@ channels:
  - c3i_test
 dependencies:
  - python=3.7
+ - botocore>=1.11
  - cython>=0.28.2
  - numpy
  - python-dateutil
@@ -14,3 +15,6 @@ dependencies:
  - pytest
  - pytest-xdist
  - hypothesis>=3.58.0
+ - s3fs
+ - pip:
+ - moto

environment.yml

@@ -26,6 +26,8 @@ dependencies:
  # optional
  - beautifulsoup4>=4.2.1
  - blosc
+ - botocore>=1.11
+ - boto3
  - bottleneck>=1.2.0
  - fastparquet>=0.1.2
  - html5lib
@@ -41,6 +43,7 @@ dependencies:
  - pytables>=3.4.2
  - pytest-cov
  - pytest-xdist
+ - s3fs
  - scipy>=1.1
  - seaborn
  - sqlalchemy

pandas/tests/io/conftest.py

@@ -1,4 +1,10 @@
+from distutils.version import LooseVersion
+import os
+
 import pytest
+
+import pandas.util.testing as tm
+
 from pandas.io.parsers import read_csv
 
 
@@ -37,43 +43,48 @@ def s3_resource(tips_file, jsonl_file):
  """
  pytest.importorskip('s3fs')
  boto3 = pytest.importorskip('boto3')
- # GH-24092. See if boto.plugin skips the test or fails.
- try:
- pytest.importorskip("boto.plugin")
- except AttributeError:
- raise pytest.skip("moto/moto error")
- moto = pytest.importorskip('moto')
-
- test_s3_files = [
- ('tips.csv', tips_file),
- ('tips.csv.gz', tips_file + '.gz'),
- ('tips.csv.bz2', tips_file + '.bz2'),
- ('items.jsonl', jsonl_file),
- ]
-
- def add_tips_files(bucket_name):
- for s3_key, file_name in test_s3_files:
- with open(file_name, 'rb') as f:
- conn.Bucket(bucket_name).put_object(
- Key=s3_key,
- Body=f)
-
- try:
-
- s3 = moto.mock_s3()
- s3.start()
-
- # see gh-16135
- bucket = 'pandas-test'
- conn = boto3.resource("s3", region_name="us-east-1")
-
- conn.create_bucket(Bucket=bucket)
- add_tips_files(bucket)
-
- conn.create_bucket(Bucket='cant_get_it', ACL='private')
- add_tips_files('cant_get_it')
- yield conn
- except: # noqa: flake8
- pytest.skip("failure to use s3 resource")
- finally:
- s3.stop()
+ botocore = pytest.importorskip('botocore')
+
+ if LooseVersion(botocore.__version__) < LooseVersion("1.11.0"):
+ # botocore leaks an uncatchable ResourceWarning before 1.11.0;
+ # see GH 23731 and https://github.com/boto/botocore/issues/1464
+ pytest.skip("botocore is leaking resources before 1.11.0")
+
+ with tm.ensure_safe_environment_variables():
+ # temporary workaround as moto fails for botocore >= 1.11 otherwise,
+ # see https://github.com/spulec/moto/issues/1924 & 1952
+ os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+ os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+ moto = pytest.importorskip('moto')
+
+ test_s3_files = [
+ ('tips.csv', tips_file),
+ ('tips.csv.gz', tips_file + '.gz'),
+ ('tips.csv.bz2', tips_file + '.bz2'),
+ ('items.jsonl', jsonl_file),
+ ]
+
+ def add_tips_files(bucket_name):
+ for s3_key, file_name in test_s3_files:
+ with open(file_name, 'rb') as f:
+ conn.Bucket(bucket_name).put_object(
+ Key=s3_key,
+ Body=f)
+
+ try:
+ s3 = moto.mock_s3()
+ s3.start()
+
+ # see gh-16135
+ bucket = 'pandas-test'
+ conn = boto3.resource("s3", region_name="us-east-1")
+
+ conn.create_bucket(Bucket=bucket)
+ add_tips_files(bucket)
+
+ conn.create_bucket(Bucket='cant_get_it', ACL='private')
+ add_tips_files('cant_get_it')
+ yield conn
+ finally:
+ s3.stop()

pandas/util/testing.py

@@ -782,6 +782,22 @@ def ensure_clean_dir():
  pass
 
 
+@contextmanager
+def ensure_safe_environment_variables():
+ """
+ Get a context manager to safely set environment variables
+
+ All changes will be undone on close, hence environment variables set
+ within this contextmanager will neither persist nor change global state.
+ """
+ saved_environ = dict(os.environ)
+ try:
+ yield
+ finally:
+ os.environ.clear()
+ os.environ.update(saved_environ)
+
+
 # -----------------------------------------------------------------------------
 # Comparators