**Move random provider acquisition to FirstAvailableStrategyI and clean up FirstAvailableStrategy; update default balance strategy to first_available**

Author: Paweł Kędzia

llm_router_api/base/lb/first_available.py

@@ -225,79 +225,3 @@ def put_provider(
  raise
 
  provider.pop("__chosen_field", None)
-
- def _try_acquire_random_provider(
- self, redis_key: str, providers: List[Dict]
- ) -> Optional[Dict]:
- """
- Attempt to lock a provider chosen at random.
-
- The method works in three stages:
-
- 1. **Shuffle** – a shallow copy of ``providers`` is shuffled so that each
- provider has an equal probability of being tried first. The original
- list is left untouched.
- 2. **Atomic acquisition** – each shuffled provider is passed to the
- ``_acquire_script`` Lua script which atomically sets the corresponding
- Redis hash field to ``'true'`` *only if* it is currently ``'false'`` or
- missing. The first provider for which the script returns ``1`` is
- considered successfully acquired.
- 3. **Fallback** – if none of the providers can be locked (e.g., all are
- currently in use), the method falls back to the *first* provider in the
- original ``providers`` list, marks its ``"__chosen_field"`` for
- consistency, and returns it. This fallback mirrors the behaviour of
- the non‑random acquisition path and ensures the caller always receives
- a provider dictionary (or ``None`` when ``providers`` is empty).
-
- Parameters
- ----------
- redis_key : str
- The Redis hash key associated with the model (e.g., ``model:<name>``).
- providers : List[Dict]
- A list of provider configuration dictionaries. Each dictionary must
- contain sufficient information for :meth:`_provider_field` to generate
- a unique field name within the Redis hash.
-
- Returns
- -------
- Optional[Dict]
- The selected provider dictionary with an additional ``"__chosen_field"``
- entry indicating the Redis hash field that was locked. Returns ``None``
- only when the input ``providers`` list is empty.
-
- Raises
- ------
- Exception
- Propagates any unexpected exceptions raised by the Lua script execution;
- callers may catch these to implement retry or logging logic.
-
- Notes
- -----
- * The random selection is *non‑deterministic* on each call; however, the
- fallback to the first provider ensures deterministic behaviour when
- all providers are currently busy.
- * The method does **not** block; it returns immediately after trying all
- shuffled providers.
- """
- shuffled = providers[:]
- random.shuffle(shuffled)
- for provider in shuffled:
- provider_field = self._provider_field(provider)
- try:
- ok = int(
- self._acquire_script(keys=[redis_key], args=[provider_field])
- )
- if ok == 1:
- provider["__chosen_field"] = provider_field
- return provider
- except Exception:
- continue
- return None
-
- def _get_active_providers(
- self, model_name: str, providers: List[Dict]
- ) -> List[Dict]:
- active_providers = self._monitor.get_providers(
- model_name=model_name, only_active=True
- )
- return active_providers

llm_router_api/base/lb/first_available_i.py

@@ -1,3 +1,4 @@
+import random
 import time
 import logging
 from abc import ABC
@@ -186,6 +187,85 @@ def _init_flag(self, model_name: str) -> str:
  """
  return f"{self._get_redis_key(model_name)}:initialized"
 
+ # ----------------------------------------------------------------------
+ # Helper methods
+ # ----------------------------------------------------------------------
+ def _try_acquire_random_provider(
+ self, redis_key: str, providers: List[Dict]
+ ) -> Optional[Dict]:
+ """
+ Attempt to lock a provider chosen at random.
+
+ The method works in three stages:
+
+ 1. **Shuffle** – a shallow copy of ``providers`` is shuffled so that each
+ provider has an equal probability of being tried first. The original
+ list is left untouched.
+ 2. **Atomic acquisition** – each shuffled provider is passed to the
+ ``_acquire_script`` Lua script which atomically sets the corresponding
+ Redis hash field to ``'true'`` *only if* it is currently ``'false'`` or
+ missing. The first provider for which the script returns ``1`` is
+ considered successfully acquired.
+ 3. **Fallback** – if none of the providers can be locked (e.g., all are
+ currently in use), the method falls back to the *first* provider in the
+ original ``providers`` list, marks its ``"__chosen_field"`` for
+ consistency, and returns it. This fallback mirrors the behaviour of
+ the non‑random acquisition path and ensures the caller always receives
+ a provider dictionary (or ``None`` when ``providers`` is empty).
+
+ Parameters
+ ----------
+ redis_key : str
+ The Redis hash key associated with the model (e.g., ``model:<name>``).
+ providers : List[Dict]
+ A list of provider configuration dictionaries. Each dictionary must
+ contain sufficient information for :meth:`_provider_field` to generate
+ a unique field name within the Redis hash.
+
+ Returns
+ -------
+ Optional[Dict]
+ The selected provider dictionary with an additional ``"__chosen_field"``
+ entry indicating the Redis hash field that was locked. Returns ``None``
+ only when the input ``providers`` list is empty.
+
+ Raises
+ ------
+ Exception
+ Propagates any unexpected exceptions raised by the Lua script execution;
+ callers may catch these to implement retry or logging logic.
+
+ Notes
+ -----
+ * The random selection is *non‑deterministic* on each call; however, the
+ fallback to the first provider ensures deterministic behaviour when
+ all providers are currently busy.
+ * The method does **not** block; it returns immediately after trying all
+ shuffled providers.
+ """
+ shuffled = providers[:]
+ random.shuffle(shuffled)
+ for provider in shuffled:
+ provider_field = self._provider_field(provider)
+ try:
+ ok = int(
+ self._acquire_script(keys=[redis_key], args=[provider_field])
+ )
+ if ok == 1:
+ provider["__chosen_field"] = provider_field
+ return provider
+ except Exception:
+ continue
+ return None
+
+ def _get_active_providers(
+ self, model_name: str, providers: List[Dict]
+ ) -> List[Dict]:
+ active_providers = self._monitor.get_providers(
+ model_name=model_name, only_active=True
+ )
+ return active_providers
+
  def _initialize_providers(self, model_name: str, providers: List[Dict]) -> None:
  """
  Ensure that the provider lock fields for *model_name* exist in Redis.
@@ -282,6 +362,6 @@ def _print_provider_status(self, redis_key: str, providers: List[Dict]) -> None:
  status = hash_data.get(field, "false")
  icon = "🔴" if status == "true" else "🟢"
  # Show a short identifier for the provider (fallback to field)
- provider_id = provider.get("id") or provider.get("name") or field
+ provider_id = provider.get("id") or provider.get("api_host") or field
  print(f"{icon} {provider_id:<30} [{field}]")
  print("-" * 40)

run-rest-api-gunicorn.sh

@@ -13,7 +13,7 @@
  export LLM_ROUTER_SERVER_TYPE=${LLM_ROUTER_SERVER_TYPE:-gunicorn}
  export LLM_ROUTER_SERVER_PORT=${LLM_ROUTER_SERVER_PORT:-8080}
  export LLM_ROUTER_SERVER_HOST=${LLM_ROUTER_SERVER_HOST:-"0.0.0.0"}
- export LLM_ROUTER_BALANCE_STRATEGY=${LLM_ROUTER_BALANCE_STRATEGY:-"first_available_optim"}
+ export LLM_ROUTER_BALANCE_STRATEGY=${LLM_ROUTER_BALANCE_STRATEGY:-"first_available"}
  export LLM_ROUTER_REDIS_HOST=${LLM_ROUTER_REDIS_HOST:-"192.168.100.67"}
  export LLM_ROUTER_REDIS_PORT=${LLM_ROUTER_REDIS_PORT:-6379}
  export LLM_ROUTER_SERVER_WORKERS_COUNT=${LLM_ROUTER_SERVER_WORKERS_COUNT:-4}