Take skipped shards as successful in scan helper

Author: honzakral

elasticsearch/helpers/actions.py

@@ -442,17 +442,18 @@ def scan(
  yield hit
 
  # check if we have any errors
- if resp["_shards"]["successful"] < resp["_shards"]["total"]:
+ if (resp["_shards"]["successful"] + resp["_shards"]["skipped"]) < resp["_shards"]["total"]:
  logger.warning(
- "Scroll request has only succeeded on %d shards out of %d.",
+ "Scroll request has only succeeded on %d (+%d skipped) shards out of %d.",
  resp["_shards"]["successful"],
+ resp["_shards"]["skipped"],
  resp["_shards"]["total"],
  )
  if raise_on_error:
  raise ScanError(
  scroll_id,
- "Scroll request has only succeeded on %d shards out of %d."
- % (resp["_shards"]["successful"], resp["_shards"]["total"]),
+ "Scroll request has only succeeded on %d (+%d skiped) shards out of %d."
+ % (resp["_shards"]["successful"], resp["_shards"]["skipped"], resp["_shards"]["total"]),
  )
  resp = client.scroll(
  body={"scroll_id": scroll_id, "scroll": scroll}, **scroll_kwargs

test_elasticsearch/test_server/test_helpers.py

@@ -311,12 +311,12 @@ class TestScan(ElasticsearchTestCase):
  mock_scroll_responses = [
  {
  "_scroll_id": "dummy_id",
- "_shards": {"successful": 4, "total": 5},
+ "_shards": {"successful": 4, "total": 5, "skipped": 0},
  "hits": {"hits": [{"scroll_data": 42}]},
  },
  {
  "_scroll_id": "dummy_id",
- "_shards": {"successful": 4, "total": 5},
+ "_shards": {"successful": 4, "total": 5, "skipped": 0},
  "hits": {"hits": []},
  },
  ]
@@ -398,7 +398,7 @@ def test_initial_search_error(self):
  with patch.object(self, "client") as client_mock:
  client_mock.search.return_value = {
  "_scroll_id": "dummy_id",
- "_shards": {"successful": 4, "total": 5},
+ "_shards": {"successful": 4, "total": 5, "skipped": 0},
  "hits": {"hits": [{"search_data": 1}]},
  }
  client_mock.scroll.side_effect = self.mock_scroll_responses