import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def main(): # Setup client for your preferred provider client = get_instructor_client(provider="openai", model="gpt-4o-mini") # Define the claim to evaluate claim = "Feynman was born in 1918 in Malaysia" # Define the ground truth ground_truth = "Feynman was born in 1918 in America." # Evaluate the claim result = await evaluate( claim=claim, ground_truth=ground_truth, client=client, model_name="gpt-4o-mini" ) print(result) # Run the async function asyncio.run(main())

{ 'TP': ['Feynman was born in 1918.'], 'FP': ['Feynman was born in Malaysia.'], 'FN': ['Feynman was born in America.'], 'precision': 0.5, 'recall': 0.5, 'f1_score': 0.5 }

import os # Set environment variables programmatically os.environ["OPENAI_API_KEY"] = "your_openai_api_key" os.environ["AWS_ACCESS_KEY_ID"] = "your_aws_access_key" os.environ["AWS_SECRET_ACCESS_KEY"] = "your_aws_secret_key"

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def evaluate_with_openai(): client = get_instructor_client(provider="openai", model="gpt-4o-mini") claim = "The Earth is flat and the moon landing was in 1969." ground_truth = "The Earth is round. The moon landing was in 1969." result = await evaluate(claim, ground_truth, client=client, model_name="gpt-4o-mini") print(f"True Positives: {result['TP']}") print(f"False Positives: {result['FP']}") print(f"False Negatives: {result['FN']}") print(f"Precision: {result['precision']:.2f}") print(f"Recall: {result['recall']:.2f}") print(f"F1 Score: {result['f1_score']:.2f}") asyncio.run(evaluate_with_openai())

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def evaluate_with_bedrock(): client = get_instructor_client(provider="bedrock") claim = "Python was created by Guido van Rossum in 1991." ground_truth = "Python was created by Guido van Rossum in 1991." result = await evaluate(claim, ground_truth, client=client, model_name="bedrock-claude") return result result = asyncio.run(evaluate_with_bedrock()) print(f"Perfect match! F1 Score: {result['f1_score']}")

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def evaluate_with_anthropic(): client = get_instructor_client( provider="anthropic", model="claude-sonnet-4-20250514" ) claim = "Water boils at 100°C at sea level." ground_truth = "Water boils at 100°C at sea level." result = await evaluate(claim, ground_truth, client=client, model_name="claude-sonnet-4-20250514") return result result = asyncio.run(evaluate_with_anthropic())

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def evaluate_with_gemini(): client = get_instructor_client( provider="gemini", model="gemini-2.0-flash-exp" ) claim = "The speed of light is approximately 300,000 km/s in a vacuum." ground_truth = "The speed of light is approximately 300,000 kilometers per second in a vacuum." result = await evaluate( claim=claim, ground_truth=ground_truth, client=client, model_name="gemini-2.0-flash-exp" ) print(f"True Positives: {result['TP']}") print(f"False Positives: {result['FP']}") print(f"False Negatives: {result['FN']}") print(f"F1 Score: {result['f1_score']:.2f}") return result result = asyncio.run(evaluate_with_gemini())

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def batch_evaluate(): client = get_instructor_client(provider="openai", model="gpt-4o-mini") test_cases = [ { "claim": "Einstein developed the theory of relativity.", "ground_truth": "Einstein developed the theory of relativity." }, { "claim": "The capital of France is London.", "ground_truth": "The capital of France is Paris." }, { "claim": "Water is composed of hydrogen and oxygen.", "ground_truth": "Water is composed of hydrogen and oxygen." } ] tasks = [ evaluate(tc["claim"], tc["ground_truth"], client=client, model_name="gpt-4o-mini") for tc in test_cases ] results = await asyncio.gather(*tasks) for i, result in enumerate(results, 1): print(f"\n--- Test Case {i} ---") print(f"F1 Score: {result['f1_score']:.2f}") print(f"Precision: {result['precision']:.2f}") print(f"Recall: {result['recall']:.2f}") asyncio.run(batch_evaluate())

async def evaluate( claim: str, ground_truth: str, client: instructor.AsyncInstructor, model_name: str = "gpt-4o-mini" ) -> Dict[str, Any]

def get_instructor_client( provider: Literal["bedrock", "openai", "anthropic", "gemini"] = "bedrock", model: Optional[str] = None, api_key: Optional[str] = None, mode: instructor.Mode = instructor.Mode.JSON, **kwargs ) -> instructor.AsyncInstructor

async def text_simplifier( text: str, model_name: str, client: instructor.AsyncInstructor ) -> TextSimplify

from autoevaluator import text_simplifier, get_instructor_client async def simplify_text(): client = get_instructor_client(provider="openai") complex_text = """Although the weather was bad and it was raining heavily,   we decided to go hiking because we had planned it for weeks.""" result = await text_simplifier( text=complex_text, model_name="gpt-4o-mini", client=client ) print("Simplified sentences:") for sentence in result.simplified_sentences: print(f"- {sentence}") asyncio.run(simplify_text())

from autoevaluator.client import ( get_openai_instructor_client, get_bedrock_instructor_client, get_anthropic_instructor_client, get_gemini_instructor_client ) # OpenAI openai_client = get_openai_instructor_client(model="gpt-4o") # Bedrock bedrock_client = get_bedrock_instructor_client() # Anthropic anthropic_client = get_anthropic_instructor_client() # Gemini gemini_client = get_gemini_instructor_client(model="gemini-2.0-flash")

import asyncio from dotenv import load_dotenv load_dotenv() # Load env variables BEFORE importing autoevaluator from autoevaluator import evaluate, get_instructor_client async def safe_evaluate(): try: client = get_instructor_client(provider="openai") result = await evaluate( claim="Some claim", ground_truth="Some truth", client=client, model_name="gpt-4o-mini" ) return result except ValueError as e: print(f"Configuration error: {e}") except Exception as e: print(f"Evaluation error: {e}") asyncio.run(safe_evaluate())