Merge pull request elastic#635 from fxdgear/nick/ssl_context

Nick/ssl context

Author: fxdgear

.travis.yml

@@ -24,9 +24,9 @@ before_install:
  - java -version
 
 install:
- - curl -L -o /tmp/es-snap.zip https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0-beta2.zip
+ - curl -L -o /tmp/es-snap.zip https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0-rc1.zip
  - unzip /tmp/es-snap.zip -d /tmp/
- - /tmp/elasticsearch-*/bin/elasticsearch -E script.inline=true -E path.repo=/tmp -E repositories.url.allowed_urls='http://*' -E node.attr.testattr=test -d
+ - /tmp/elasticsearch-*/bin/elasticsearch -E path.repo=/tmp -E repositories.url.allowed_urls='http://*' -E node.attr.testattr=test -d
  - git clone https://github.com/elastic/elasticsearch.git ../elasticsearch
  - pip install .
 

docs/connection.rst

@@ -45,5 +45,15 @@ Connection Selector
 Urllib3HttpConnection (default connection_class)
 ------------------------------------------------
 
+Deprecation Notice: `use_ssl`, `verify_certs`, `ca_certs` and `ssl_version` are being
+deprecated in favor of using a `SSLContext` (https://docs.python.org/3/library/ssl.html#ssl.SSLContext) object.
+
+You can continue to use the deprecated parameters and an `SSLContext` will be created for you.
+
+If you want to create your own `SSLContext` object you can create one natively using the
+python SSL library with the `create_default_context` (https://docs.python.org/3/library/ssl.html#ssl.create_default_context) method
+or you can use the wrapper function :function:`~elasticsearch.connection.http_urllib3.create_ssl_context`.
+
+
 .. autoclass:: Urllib3HttpConnection
  :members:

elasticsearch/connection/http_urllib3.py

@@ -1,4 +1,5 @@
 import time
+import ssl
 import urllib3
 from urllib3.exceptions import ReadTimeoutError, SSLError as UrllibSSLError
 import warnings
@@ -15,6 +16,19 @@
 from ..exceptions import ConnectionError, ImproperlyConfigured, ConnectionTimeout, SSLError
 from ..compat import urlencode
 
+
+def create_ssl_context(**kwargs):
+ """
+ A helper function around creating an SSL context
+
+ https://docs.python.org/3/library/ssl.html#context-creation
+
+ Accepts kwargs in the same manner as `create_default_context`.
+ """
+ ctx = ssl.create_default_context(**kwargs)
+ return ctx
+
+
 class Urllib3HttpConnection(Connection):
  """
  Default connection class using the `urllib3` library and the http protocol.
@@ -47,7 +61,7 @@ class Urllib3HttpConnection(Connection):
  def __init__(self, host='localhost', port=9200, http_auth=None,
  use_ssl=False, verify_certs=True, ca_certs=None, client_cert=None,
  client_key=None, ssl_version=None, ssl_assert_hostname=None,
- ssl_assert_fingerprint=None, maxsize=10, headers=None, **kwargs):
+ ssl_assert_fingerprint=None, maxsize=10, headers=None, ssl_context=None, **kwargs):
 
  super(Urllib3HttpConnection, self).__init__(host=host, port=port, use_ssl=use_ssl, **kwargs)
  self.headers = urllib3.make_headers(keep_alive=True)
@@ -62,33 +76,49 @@ def __init__(self, host='localhost', port=9200, http_auth=None,
  self.headers[k.lower()] = headers[k]
 
  self.headers.setdefault('content-type', 'application/json')
- ca_certs = CA_CERTS if ca_certs is None else ca_certs
  pool_class = urllib3.HTTPConnectionPool
  kw = {}
- if use_ssl:
+
+ # if providing an SSL context, raise error if any other SSL related flag is used
+ if ssl_context and (verify_certs or ca_certs or ssl_version):
+ raise ImproperlyConfigured("When using `ssl_context`, `use_ssl`, `verify_certs`, `ca_certs` and `ssl_version` are not permitted")
+
+ # if ssl_context provided use SSL by default
+ if use_ssl or ssl_context:
+ if not ca_certs and not ssl_context and verify_certs:
+ # If no ca_certs and no sslcontext passed and asking to verify certs
+ # raise error
+ raise ImproperlyConfigured("Root certificates are missing for certificate "
+ "validation. Either pass them in using the ca_certs parameter or "
+ "install certifi to use it automatically.")
+ if verify_certs or ca_certs or ssl_version:
+ warnings.warn('Use of `verify_certs`, `ca_certs`, `ssl_version` have been deprecated in favor of using SSLContext`', DeprecationWarning)
  pool_class = urllib3.HTTPSConnectionPool
+
+ if not ssl_context:
+ # if SSLContext hasn't been passed in, create one.
+ cafile = CA_CERTS if ca_certs is None else ca_certs
+ # need to skip if sslContext isn't avail
+ try:
+ ssl_context = create_ssl_context(cafile=cafile)
+ except AttributeError:
+ ssl_context = None
+
+ if not verify_certs and ssl_context is not None:
+ ssl_context.check_hostname = False
+ ssl_context.verify_mode = ssl.CERT_NONE
+ warnings.warn(
+ 'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
+
  kw.update({
  'ssl_version': ssl_version,
  'assert_hostname': ssl_assert_hostname,
  'assert_fingerprint': ssl_assert_fingerprint,
+ 'ssl_context': ssl_context,
+ 'cert_file': client_cert,
+ 'ca_certs': ca_certs,
+ 'key_file': client_key,
  })
-
- if verify_certs:
- if not ca_certs:
- raise ImproperlyConfigured("Root certificates are missing for certificate "
- "validation. Either pass them in using the ca_certs parameter or "
- "install certifi to use it automatically.")
-
- kw.update({
- 'cert_reqs': 'CERT_REQUIRED',
- 'ca_certs': ca_certs,
- 'cert_file': client_cert,
- 'key_file': client_key,
- })
- else:
- warnings.warn(
- 'Connecting to %s using SSL with verify_certs=False is insecure.' % host)
-
  self.pool = pool_class(host, port=port, timeout=self.timeout, maxsize=maxsize, **kw)
 
  def perform_request(self, method, url, params=None, body=None, timeout=None, ignore=(), headers=None):

test_elasticsearch/test_connection.py

@@ -1,4 +1,6 @@
+import sys
 import re
+import ssl
 from mock import Mock, patch
 import urllib3
 import warnings
@@ -7,8 +9,9 @@
 from elasticsearch.exceptions import TransportError, ConflictError, RequestError, NotFoundError
 from elasticsearch.connection import RequestsHttpConnection, \
  Urllib3HttpConnection
-
-from .test_cases import TestCase
+from elasticsearch.exceptions import ImproperlyConfigured
+from elasticsearch.connection.http_urllib3 import create_ssl_context
+from .test_cases import TestCase, SkipTest
 
 
 class TestUrllib3Connection(TestCase):
@@ -42,6 +45,12 @@ def test_http_auth_list(self):
  'connection': 'keep-alive'}, con.headers)
 
  def test_uses_https_if_verify_certs_is_off(self):
+ if (
+ sys.version_info >= (3,0) and sys.version_info <= (3,4)
+ ) or (
+ sys.version_info >= (2,6) and sys.version_info <= (2,7)
+ ):
+ raise SkipTest("SSL Context not supported in this version of python")
  with warnings.catch_warnings(record=True) as w:
  con = Urllib3HttpConnection(use_ssl=True, verify_certs=False)
  self.assertEquals(1, len(w))
@@ -53,6 +62,16 @@ def test_doesnt_use_https_if_not_specified(self):
  con = Urllib3HttpConnection()
  self.assertIsInstance(con.pool, urllib3.HTTPConnectionPool)
 
+ def test_ssl_context_and_depreicated_values(self):
+ try:
+ ctx = create_ssl_context()
+ except AttributeError:
+ raise SkipTest("SSL Context not supported in this version of python")
+ self.assertRaises(ImproperlyConfigured, Urllib3HttpConnection, ssl_context=ctx, use_ssl=True)
+ self.assertRaises(ImproperlyConfigured, Urllib3HttpConnection, ssl_context=ctx, verify_certs=True)
+ self.assertRaises(ImproperlyConfigured, Urllib3HttpConnection, ssl_context=ctx, ca_certs="/some/path/to/cert.crt")
+ self.assertRaises(ImproperlyConfigured, Urllib3HttpConnection, ssl_context=ctx, ssl_version=ssl.PROTOCOL_SSLv23)
+
 class TestRequestsConnection(TestCase):
  def _get_mock_connection(self, connection_params={}, status_code=200, response_body='{}'):
  con = RequestsHttpConnection(**connection_params)

test_elasticsearch/test_server/test_common.py

@@ -40,7 +40,11 @@
  '*': set(('TestBulk10Basic', 'TestCatSnapshots10Basic',
  'TestClusterPutSettings10Basic', 'TestIndex10WithId',
  'TestClusterPutScript10Basic', 'TestIndicesPutMapping10Basic',
- 'TestIndicesPutTemplate10Basic', 'TestMsearch10Basic'))
+ 'TestIndicesPutTemplate10Basic', 'TestMsearch10Basic',
+ # skip these two till https://github.com/elastic/elasticsearch/pull/26905 has been merged
+ 'TestSearchAggregation190PercentilesHdrMetric',
+ 'TestSearchAggregation180PercentilesTdigestMetric',
+ ))
 
 }