Bug Fix - Address deprecation of open_text (BerriAI#10208)

* Update utils.py (BerriAI#10201) * fixes importlib --------- Co-authored-by: Nathan Brake <33383515+njbrake@users.noreply.github.com>

Author: ishaan-jaff

litellm/utils.py

@@ -180,10 +180,18 @@
  all_litellm_params,
 )
 
-with resources.open_text(
- "litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json"
-) as f:
- json_data = json.load(f)
+try:
+ # Python 3.9+
+ with resources.files("litellm.litellm_core_utils.tokenizers").joinpath(
+ "anthropic_tokenizer.json"
+ ).open("r") as f:
+ json_data = json.load(f)
+except (ImportError, AttributeError, TypeError):
+ with resources.open_text(
+ "litellm.litellm_core_utils.tokenizers", "anthropic_tokenizer.json"
+ ) as f:
+ json_data = json.load(f)
+
 # Convert to str (if necessary)
 claude_json_str = json.dumps(json_data)
 import importlib.metadata