# Create a virtual environment python3 -m venv myproject_env # Activate it # On Windows: myproject_env\Scripts\activate # On macOS/Linux: source myproject_env/bin/activate # Install packages pip install requests beautifulsoup4 language-tool-python # Deactivate when done deactivate

# Install a package pip install requests # Install from requirements file pip install -r requirements.txt # List installed packages pip list # Generate requirements file pip freeze > requirements.txt

#!/usr/bin/env python3 """ A simple Hello World program demonstrating modern Python syntax. """ def greet(name: str) -> str: """Return a personalized greeting.""" return f"Hello, {name}! Welcome to Python programming." def main(): """Main program entry point.""" # Get user input  user_name = input("What's your name? ") # Create and display greeting  greeting = greet(user_name) print(greeting) # Show some Python features  languages = ["Python", "Java", "JavaScript", "Go"] print(f"\nHere are some popular programming languages:") for i, lang in enumerate(languages, 1): print(f"{i}. {lang}") if __name__ == "__main__": main()

python3 hello_world.py

#!/usr/bin/env python3 """ File analyzer that demonstrates modern Python features. """ import os from pathlib import Path from typing import Dict, List def analyze_file(file_path: Path) -> Dict[str, any]: """Analyze a text file and return statistics.""" try: with open(file_path, 'r', encoding='utf-8') as file: content = file.read() lines = content.split('\n') return { 'filename': file_path.name, 'size_bytes': file_path.stat().st_size, 'line_count': len(lines), 'word_count': len(content.split()), 'char_count': len(content), 'extension': file_path.suffix } except FileNotFoundError: return {'error': f'File not found: {file_path}'} except Exception as e: return {'error': f'Error reading file: {e}'} def analyze_directory(directory: str) -> List[Dict[str, any]]: """Analyze all text files in a directory.""" path = Path(directory) results = [] if not path.exists(): return [{'error': f'Directory not found: {directory}'}] # Find text files  text_extensions = {'.txt', '.py', '.md', '.adoc', '.rst'} for file_path in path.iterdir(): if file_path.is_file() and file_path.suffix in text_extensions: results.append(analyze_file(file_path)) return results def main(): """Main program demonstrating file analysis.""" print("🔍 File Analyzer - Modern Python Example") print("=" * 40) # Analyze current directory  directory = "." results = analyze_directory(directory) if not results: print("No text files found in current directory.") return # Display results  total_files = len(results) total_lines = sum(r.get('line_count', 0) for r in results if 'error' not in r) print(f"\nFound {total_files} text files:") print(f"Total lines: {total_lines:,}") print("\nFile details:") print("-" * 60) for result in results: if 'error' in result: print(f"❌ {result['error']}") else: print(f"📄 {result['filename']:20} | " f"{result['line_count']:4} lines | " f"{result['size_bytes']:6} bytes") if __name__ == "__main__": main()

# Start interactive Python python3 # Try some expressions >>> name = "Python" >>> print(f"Hello, {name}!") Hello, Python! >>> numbers = [1, 2, 3, 4, 5] >>> sum(numbers) 15 >>> exit()

my_python_project/ ├── requirements.txt # Project dependencies ├── README.md # Project documentation ├── main.py # Entry point ├── src/ # Source code │ ├── __init__.py │ └── my_module.py └── tests/ # Test files ├── __init__.py └── test_my_module.py

#!/usr/bin/env python3 """ Modern Python variables and type hints examples. """ from typing import List, Dict, Optional # Basic variables with type hints name: str = "Alice" age: int = 30 height: float = 5.6 is_student: bool = True # Collections with type hints numbers: List[int] = [1, 2, 3, 4, 5] scores: Dict[str, int] = {"math": 95, "science": 88, "history": 92} middle_name: Optional[str] = None # Can be None or string  # Dynamic typing still works dynamic_var = "starts as string" dynamic_var = 42 # now it's an integer dynamic_var = ["now", "it's", "a", "list"] # Multiple assignment x, y, z = 1, 2, 3 first, *rest = [1, 2, 3, 4, 5] # first=1, rest=[2,3,4,5]  # Constants (by convention, use UPPER_CASE) MAX_CONNECTIONS: int = 100 API_URL: str = "https://api.example.com" print(f"Hello, {name}! You are {age} years old.") print(f"Your scores: {scores}") print(f"First number: {first}, rest: {rest}")

#!/usr/bin/env python3 """ Modern Python string operations with f-strings and advanced techniques. """ # F-string formatting (preferred in Python 3.6+) name = "Python" version = 3.12 print(f"Welcome to {name} {version}!") # Multi-line f-strings user = {"name": "Alice", "age": 30, "city": "New York"} message = f""" Hello {user['name']}! You are {user['age']} years old and live in {user['city']}. """ print(message) # F-strings with expressions and formatting numbers = [1, 2, 3, 4, 5] print(f"Sum of {numbers} = {sum(numbers)}") print(f"Pi to 3 decimal places: {3.14159:.3f}") # String methods and operations text = " Hello, World! " print(f"Original: '{text}'") print(f"Stripped: '{text.strip()}'") print(f"Uppercase: '{text.upper()}'") print(f"Lowercase: '{text.lower()}'") print(f"Title Case: '{text.title()}'") # String slicing and indexing sentence = "Python programming is fun" print(f"First word: {sentence[:6]}") print(f"Last word: {sentence.split()[-1]}") print(f"Every 2nd character: {sentence[::2]}") # String checking methods email = "user@example.com" print(f"Contains @: {'@' in email}") print(f"Starts with 'user': {email.startswith('user')}") print(f"Ends with '.com': {email.endswith('.com')}") # Joining and splitting words = ["Python", "is", "awesome"] joined = " ".join(words) print(f"Joined: {joined}") print(f"Split back: {joined.split()}") # Raw strings for regex patterns import re pattern = r"\d{3}-\d{3}-\d{4}" # Phone number pattern phone = "123-456-7890" print(f"Phone match: {bool(re.match(pattern, phone))}")

#!/usr/bin/env python3 """ Working with Python collections: lists, dictionaries, sets, and tuples. """ from collections import defaultdict, Counter from typing import List, Dict, Set, Tuple # Lists - ordered, mutable collections fruits: List[str] = ["apple", "banana", "cherry", "date"] print(f"Fruits: {fruits}") # List comprehensions (Pythonic way to create lists) squares = [x**2 for x in range(1, 6)] even_numbers = [x for x in range(20) if x % 2 == 0] print(f"Squares: {squares}") print(f"Even numbers: {even_numbers}") # List operations fruits.append("elderberry") fruits.extend(["fig", "grape"]) print(f"After additions: {fruits}") # Dictionaries - key-value pairs person: Dict[str, any] = { "name": "Alice", "age": 30, "skills": ["Python", "Java", "JavaScript"], "is_employed": True } # Dictionary comprehension word_lengths = {word: len(word) for word in fruits} print(f"Word lengths: {word_lengths}") # Safe dictionary access age = person.get("age", 0) # Returns 0 if "age" not found print(f"Age: {age}") # Sets - unique, unordered collections unique_numbers: Set[int] = {1, 2, 3, 3, 4, 4, 5} print(f"Unique numbers: {unique_numbers}") # Set operations set1 = {1, 2, 3, 4} set2 = {3, 4, 5, 6} print(f"Union: {set1 | set2}") print(f"Intersection: {set1 & set2}") print(f"Difference: {set1 - set2}") # Tuples - immutable sequences coordinates: Tuple[float, float] = (10.5, 20.3) rgb_color: Tuple[int, int, int] = (255, 128, 0) print(f"Coordinates: {coordinates}") # Named tuples for better structure from collections import namedtuple Point = namedtuple('Point', ['x', 'y']) p = Point(10, 20) print(f"Point: x={p.x}, y={p.y}") # Advanced collections # defaultdict - provides default values word_count = defaultdict(int) text = "hello world hello python world" for word in text.split(): word_count[word] += 1 print(f"Word count: {dict(word_count)}") # Counter - counts occurrences counter = Counter(text.split()) print(f"Most common word: {counter.most_common(1)}") # Unpacking and packing numbers = [1, 2, 3, 4, 5] first, second, *rest = numbers print(f"First: {first}, Second: {second}, Rest: {rest}") # Zip for parallel iteration names = ["Alice", "Bob", "Charlie"] ages = [25, 30, 35] for name, age in zip(names, ages): print(f"{name} is {age} years old")

#!/usr/bin/env python3 """ Modern Python functions with type hints and advanced features. """ from typing import List, Optional, Callable, Any from functools import wraps # Basic function with type hints def greet(name: str, age: int = 25) -> str: """Return a personalized greeting.""" return f"Hello, {name}! You are {age} years old." # Function with optional parameters def create_user(name: str, email: str, age: Optional[int] = None) -> dict: """Create a user dictionary with optional age.""" user = {"name": name, "email": email} if age is not None: user["age"] = age return user # Function with variable arguments def calculate_average(*numbers: float) -> float: """Calculate average of any number of values.""" if not numbers: return 0.0 return sum(numbers) / len(numbers) # Function with keyword arguments def create_config(**kwargs: Any) -> dict: """Create configuration dictionary from keyword arguments.""" defaults = {"debug": False, "port": 8080} defaults.update(kwargs) return defaults # Lambda functions (anonymous functions) square = lambda x: x**2 add = lambda x, y: x + y # Higher-order functions def apply_operation(numbers: List[int], operation: Callable[[int], int]) -> List[int]: """Apply an operation to each number in the list.""" return [operation(n) for n in numbers] # Decorator function def timer(func): """Decorator to time function execution.""" @wraps(func) def wrapper(*args, **kwargs): import time start = time.time() result = func(*args, **kwargs) end = time.time() print(f"{func.__name__} took {end - start:.4f} seconds") return result return wrapper @timer def slow_function(): """A function that takes some time.""" import time time.sleep(0.1) return "Done!" # Generator function def fibonacci(n: int): """Generate fibonacci numbers up to n.""" a, b = 0, 1 count = 0 while count < n: yield a a, b = b, a + b count += 1 # Example usage if __name__ == "__main__": # Basic functions  print(greet("Alice")) print(greet("Bob", 30)) # User creation  user1 = create_user("Alice", "alice@example.com") user2 = create_user("Bob", "bob@example.com", 30) print(f"User 1: {user1}") print(f"User 2: {user2}") # Variable arguments  avg = calculate_average(10, 20, 30, 40, 50) print(f"Average: {avg}") # Keyword arguments  config = create_config(debug=True, host="localhost", port=3000) print(f"Config: {config}") # Lambda and higher-order functions  numbers = [1, 2, 3, 4, 5] squared = apply_operation(numbers, square) print(f"Squared: {squared}") # Decorator  slow_function() # Generator  fib_numbers = list(fibonacci(10)) print(f"Fibonacci: {fib_numbers}")

#!/usr/bin/env python3 """ Modern Python classes with type hints, dataclasses, and properties. """ from dataclasses import dataclass from typing import List, Optional, ClassVar from abc import ABC, abstractmethod # Modern class with type hints and properties class Person: """A person with name, age, and email.""" # Class variable  species: ClassVar[str] = "Homo sapiens" def __init__(self, name: str, age: int, email: str) -> None: self._name = name self._age = age self._email = email self._friends: List[str] = [] @property def name(self) -> str: """Get the person's name.""" return self._name @property def age(self) -> int: """Get the person's age.""" return self._age @age.setter def age(self, value: int) -> None: """Set the person's age with validation.""" if value < 0: raise ValueError("Age cannot be negative") self._age = value @property def email(self) -> str: """Get the person's email.""" return self._email def add_friend(self, friend_name: str) -> None: """Add a friend to the person's friend list.""" if friend_name not in self._friends: self._friends.append(friend_name) def get_friends(self) -> List[str]: """Get a copy of the friend list.""" return self._friends.copy() def __str__(self) -> str: return f"Person(name='{self.name}', age={self.age}, email='{self.email}')" def __repr__(self) -> str: return self.__str__() # Dataclass - automatically generates __init__, __str__, __eq__, etc. @dataclass class Product: """A product with name, price, and quantity.""" name: str price: float quantity: int = 0 category: Optional[str] = None def total_value(self) -> float: """Calculate total value of this product.""" return self.price * self.quantity def __post_init__(self): """Validate data after initialization.""" if self.price < 0: raise ValueError("Price cannot be negative") # Abstract base class class Animal(ABC): """Abstract animal class.""" def __init__(self, name: str, species: str): self.name = name self.species = species @abstractmethod def make_sound(self) -> str: """Make a sound - must be implemented by subclasses.""" pass def sleep(self) -> str: """All animals can sleep.""" return f"{self.name} is sleeping..." # Concrete implementation class Dog(Animal): """A dog that inherits from Animal.""" def __init__(self, name: str, breed: str): super().__init__(name, "Canis lupus") self.breed = breed def make_sound(self) -> str: return f"{self.name} says Woof!" def fetch(self, item: str) -> str: return f"{self.name} fetches the {item}!" # Class with static and class methods class MathUtils: """Utility class for mathematical operations.""" PI: ClassVar[float] = 3.14159 @staticmethod def add(a: float, b: float) -> float: """Add two numbers.""" return a + b @classmethod def circle_area(cls, radius: float) -> float: """Calculate circle area using class constant.""" return cls.PI * radius * radius # Example usage if __name__ == "__main__": # Regular class  person = Person("Alice", 30, "alice@example.com") person.add_friend("Bob") person.add_friend("Charlie") print(person) print(f"Friends: {person.get_friends()}") # Dataclass  product = Product("Laptop", 999.99, 5, "Electronics") print(f"Product: {product}") print(f"Total value: ${product.total_value():.2f}") # Inheritance and polymorphism  dog = Dog("Buddy", "Golden Retriever") print(dog.make_sound()) print(dog.fetch("ball")) print(dog.sleep()) # Static and class methods  result = MathUtils.add(5, 3) area = MathUtils.circle_area(10) print(f"5 + 3 = {result}") print(f"Circle area (r=10): {area:.2f}")

#!/usr/bin/env python3 """ Modern error handling and exception management in Python. """ import logging from typing import Optional, List, Dict from pathlib import Path # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Custom exceptions class ValidationError(Exception): """Raised when data validation fails.""" pass class NetworkError(Exception): """Raised when network operations fail.""" def __init__(self, message: str, status_code: Optional[int] = None): super().__init__(message) self.status_code = status_code # Basic exception handling def safe_divide(a: float, b: float) -> Optional[float]: """Safely divide two numbers.""" try: result = a / b return result except ZeroDivisionError: logger.error("Cannot divide by zero") return None except TypeError as e: logger.error(f"Type error: {e}") return None # Multiple exception handling def process_user_input(user_input: str) -> Optional[int]: """Process user input and return integer.""" try: # Try to convert to integer  number = int(user_input) # Validate range  if number < 0: raise ValidationError("Number must be positive") return number except ValueError: logger.error(f"'{user_input}' is not a valid number") return None except ValidationError as e: logger.error(f"Validation error: {e}") return None # File operations with exception handling def read_config_file(filename: str) -> Dict[str, any]: """Read configuration from a file with proper error handling.""" config = {} file_path = Path(filename) try: # Check if file exists  if not file_path.exists(): raise FileNotFoundError(f"Config file {filename} not found") # Read and parse file  with open(file_path, 'r', encoding='utf-8') as file: for line_num, line in enumerate(file, 1): line = line.strip() if line and not line.startswith('#'): try: key, value = line.split('=', 1) config[key.strip()] = value.strip() except ValueError: logger.warning(f"Invalid line {line_num}: {line}") return config except FileNotFoundError as e: logger.error(f"File error: {e}") return {"error": "file_not_found"} except PermissionError: logger.error(f"Permission denied reading {filename}") return {"error": "permission_denied"} except Exception as e: logger.error(f"Unexpected error reading {filename}: {e}") return {"error": "unexpected_error"} # Context manager for resource handling class DatabaseConnection: """Mock database connection with proper cleanup.""" def __init__(self, connection_string: str): self.connection_string = connection_string self.connected = False def __enter__(self): """Enter context - establish connection.""" logger.info("Connecting to database...") self.connected = True return self def __exit__(self, exc_type, exc_val, exc_tb): """Exit context - clean up connection.""" if self.connected: logger.info("Closing database connection...") self.connected = False # Handle exceptions that occurred in the context  if exc_type is not None: logger.error(f"Exception in context: {exc_type.__name__}: {exc_val}") # Return False to propagate exceptions  return False def query(self, sql: str) -> List[Dict]: """Execute a database query.""" if not self.connected: raise ConnectionError("Not connected to database") # Simulate database operation  logger.info(f"Executing query: {sql}") return [{"id": 1, "name": "example"}] # Finally block example def process_data_with_cleanup(data_file: str) -> bool: """Process data file with guaranteed cleanup.""" temp_file = None try: # Open temporary file  temp_file = open("temp_processing.txt", "w") # Process data (might raise exceptions)  with open(data_file, "r") as file: data = file.read() temp_file.write(data.upper()) logger.info("Data processed successfully") return True except FileNotFoundError: logger.error(f"Data file {data_file} not found") return False except Exception as e: logger.error(f"Error processing data: {e}") return False finally: # This always runs, even if exception occurred  if temp_file and not temp_file.closed: temp_file.close() logger.info("Temporary file closed") # Example usage if __name__ == "__main__": # Safe division  print(f"10 / 2 = {safe_divide(10, 2)}") print(f"10 / 0 = {safe_divide(10, 0)}") # User input processing  test_inputs = ["42", "-5", "not_a_number", "100"] for inp in test_inputs: result = process_user_input(inp) print(f"Input '{inp}' -> {result}") # Context manager usage  try: with DatabaseConnection("sqlite://memory") as db: results = db.query("SELECT * FROM users") print(f"Query results: {results}") except Exception as e: print(f"Database operation failed: {e}") # Configuration file reading  config = read_config_file("nonexistent_config.txt") print(f"Config: {config}") # Processing with cleanup  success = process_data_with_cleanup("nonexistent_data.txt") print(f"Processing successful: {success}")

#!/usr/bin/env python3 """ Modern file operations using context managers and pathlib. """ from pathlib import Path from typing import List, Dict, Optional import json import csv import logging # Configure logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) # Reading files with context managers def read_text_file(filename: str) -> Optional[str]: """Read a text file safely using context manager.""" try: file_path = Path(filename) with open(file_path, 'r', encoding='utf-8') as file: content = file.read() logger.info(f"Successfully read {file_path.name} ({len(content)} characters)") return content except FileNotFoundError: logger.error(f"File {filename} not found") return None except Exception as e: logger.error(f"Error reading {filename}: {e}") return None # Writing files with automatic cleanup def write_text_file(filename: str, content: str) -> bool: """Write content to a text file.""" try: file_path = Path(filename) # Create parent directories if they don't exist  file_path.parent.mkdir(parents=True, exist_ok=True) with open(file_path, 'w', encoding='utf-8') as file: file.write(content) logger.info(f"Successfully wrote to {file_path.name}") return True except Exception as e: logger.error(f"Error writing to {filename}: {e}") return False # Working with JSON files def read_json_file(filename: str) -> Optional[Dict]: """Read and parse JSON file.""" try: file_path = Path(filename) with open(file_path, 'r', encoding='utf-8') as file: data = json.load(file) logger.info(f"Successfully loaded JSON from {file_path.name}") return data except json.JSONDecodeError as e: logger.error(f"Invalid JSON in {filename}: {e}") return None except FileNotFoundError: logger.error(f"JSON file {filename} not found") return None def write_json_file(filename: str, data: Dict) -> bool: """Write data to JSON file with proper formatting.""" try: file_path = Path(filename) file_path.parent.mkdir(parents=True, exist_ok=True) with open(file_path, 'w', encoding='utf-8') as file: json.dump(data, file, indent=2, ensure_ascii=False) logger.info(f"Successfully wrote JSON to {file_path.name}") return True except Exception as e: logger.error(f"Error writing JSON to {filename}: {e}") return False # Working with CSV files def read_csv_file(filename: str) -> Optional[List[Dict]]: """Read CSV file and return list of dictionaries.""" try: file_path = Path(filename) data = [] with open(file_path, 'r', encoding='utf-8', newline='') as file: reader = csv.DictReader(file) for row in reader: data.append(row) logger.info(f"Successfully read {len(data)} rows from {file_path.name}") return data except Exception as e: logger.error(f"Error reading CSV {filename}: {e}") return None def write_csv_file(filename: str, data: List[Dict], fieldnames: List[str]) -> bool: """Write data to CSV file.""" try: file_path = Path(filename) file_path.parent.mkdir(parents=True, exist_ok=True) with open(file_path, 'w', encoding='utf-8', newline='') as file: writer = csv.DictWriter(file, fieldnames=fieldnames) writer.writeheader() writer.writerows(data) logger.info(f"Successfully wrote {len(data)} rows to {file_path.name}") return True except Exception as e: logger.error(f"Error writing CSV to {filename}: {e}") return False # Working with paths using pathlib def analyze_directory(directory: str) -> Dict[str, any]: """Analyze directory contents using pathlib.""" try: dir_path = Path(directory) if not dir_path.exists(): return {"error": f"Directory {directory} does not exist"} if not dir_path.is_dir(): return {"error": f"{directory} is not a directory"} files = [] total_size = 0 for file_path in dir_path.iterdir(): if file_path.is_file(): size = file_path.stat().st_size files.append({ "name": file_path.name, "size": size, "extension": file_path.suffix, "modified": file_path.stat().st_mtime }) total_size += size return { "directory": str(dir_path), "file_count": len(files), "total_size": total_size, "files": files } except Exception as e: return {"error": f"Error analyzing directory: {e}"} # Processing lines from large files def process_large_file(filename: str, line_processor=None) -> int: """Process a large file line by line to save memory.""" if line_processor is None: line_processor = lambda line, num: print(f"Line {num}: {line.strip()}") try: file_path = Path(filename) line_count = 0 with open(file_path, 'r', encoding='utf-8') as file: for line_num, line in enumerate(file, 1): line_processor(line, line_num) line_count += 1 logger.info(f"Processed {line_count} lines from {file_path.name}") return line_count except Exception as e: logger.error(f"Error processing file {filename}: {e}") return 0 # Example usage and demonstrations def create_sample_files(): """Create sample files for demonstration.""" # Sample text file  text_content = """This is a sample text file. It contains multiple lines. Each line demonstrates file handling capabilities.""" write_text_file("sample_data/sample.txt", text_content) # Sample JSON file  json_data = { "name": "Python Tutorial", "version": "3.12", "features": ["modern syntax", "type hints", "async support"], "author": {"name": "Alice", "email": "alice@example.com"} } write_json_file("sample_data/config.json", json_data) # Sample CSV file  csv_data = [ {"name": "Alice", "age": "30", "city": "New York"}, {"name": "Bob", "age": "25", "city": "San Francisco"}, {"name": "Charlie", "age": "35", "city": "Chicago"} ] write_csv_file("sample_data/users.csv", csv_data, ["name", "age", "city"]) if __name__ == "__main__": # Create sample files  create_sample_files() # Read and display files  text = read_text_file("sample_data/sample.txt") if text: print("Text file content:") print(text) json_data = read_json_file("sample_data/config.json") if json_data: print(f"JSON data: {json_data}") csv_data = read_csv_file("sample_data/users.csv") if csv_data: print(f"CSV data: {csv_data}") # Analyze directory  analysis = analyze_directory("sample_data") print(f"Directory analysis: {analysis}") # Process file line by line  def word_counter(line, line_num): words = len(line.split()) print(f"Line {line_num} has {words} words") process_large_file("sample_data/sample.txt", word_counter)

# main.py from fastapi import FastAPI app = FastAPI(title="Python Tutorial API") @app.get("/") def read_root(): return {"message": "Hello from Python!"} @app.get("/crawl-status") def crawl_status(): return {"status": "Web crawler ready", "version": "1.0"}

pip install fastapi uvicorn

uvicorn main:app --reload

pip install requests beautifulsoup4 lxml

#!/usr/bin/env python3 """ Modern web crawler using requests and BeautifulSoup. Demonstrates best practices for web scraping in Python. """ import requests from bs4 import BeautifulSoup, Tag from typing import Dict, List, Optional, Set from urllib.parse import urljoin, urlparse import time import logging from dataclasses import dataclass from pathlib import Path import json # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) @dataclass class CrawlResult: """Data structure for crawl results.""" url: str title: str status_code: int links: List[str] text_content: str meta_description: str word_count: int crawl_time: float class WebCrawler: """A respectful web crawler with rate limiting and error handling.""" def __init__(self, delay: float = 1.0, max_retries: int = 3): """ Initialize the web crawler. Args: delay: Delay between requests in seconds max_retries: Maximum number of retry attempts """ self.delay = delay self.max_retries = max_retries self.session = requests.Session() # Set a reasonable user agent  self.session.headers.update({ 'User-Agent': 'Python Web Crawler Tutorial Bot 1.0 (+https://example.com/bot)' }) self.crawled_urls: Set[str] = set() self.results: List[CrawlResult] = [] def crawl_page(self, url: str) -> Optional[CrawlResult]: """ Crawl a single web page and extract information. Args: url: The URL to crawl Returns: CrawlResult object or None if crawl failed """ if url in self.crawled_urls: logger.info(f"Already crawled: {url}") return None logger.info(f"Crawling: {url}") start_time = time.time() try: # Make request with retry logic  response = self._make_request(url) if not response: return None # Parse HTML content  soup = BeautifulSoup(response.content, 'html.parser') # Extract page information  title = self._extract_title(soup) links = self._extract_links(soup, url) text_content = self._extract_text(soup) meta_description = self._extract_meta_description(soup) word_count = len(text_content.split()) crawl_time = time.time() - start_time result = CrawlResult( url=url, title=title, status_code=response.status_code, links=links, text_content=text_content[:500] + "..." if len(text_content) > 500 else text_content, meta_description=meta_description, word_count=word_count, crawl_time=crawl_time ) self.crawled_urls.add(url) self.results.append(result) # Respect the website with delay  time.sleep(self.delay) return result except Exception as e: logger.error(f"Error crawling {url}: {e}") return None def _make_request(self, url: str) -> Optional[requests.Response]: """Make HTTP request with retry logic.""" for attempt in range(self.max_retries): try: response = self.session.get(url, timeout=10) response.raise_for_status() return response except requests.exceptions.RequestException as e: logger.warning(f"Attempt {attempt + 1} failed for {url}: {e}") if attempt < self.max_retries - 1: time.sleep(2 ** attempt) # Exponential backoff  else: logger.error(f"All retry attempts failed for {url}") return None def _extract_title(self, soup: BeautifulSoup) -> str: """Extract page title.""" title_tag = soup.find('title') if title_tag and isinstance(title_tag, Tag): return title_tag.get_text().strip() return "No title found" def _extract_links(self, soup: BeautifulSoup, base_url: str) -> List[str]: """Extract all links from the page.""" links = [] for link in soup.find_all('a', href=True): if isinstance(link, Tag): href = link['href'] absolute_url = urljoin(base_url, href) links.append(absolute_url) return links def _extract_text(self, soup: BeautifulSoup) -> str: """Extract visible text content from the page.""" # Remove script and style elements  for script in soup(["script", "style"]): script.decompose() # Get text and clean it up  text = soup.get_text() lines = (line.strip() for line in text.splitlines()) chunks = (phrase.strip() for line in lines for phrase in line.split(" ")) text = ' '.join(chunk for chunk in chunks if chunk) return text def _extract_meta_description(self, soup: BeautifulSoup) -> str: """Extract meta description.""" meta_desc = soup.find('meta', attrs={'name': 'description'}) if meta_desc and isinstance(meta_desc, Tag): return meta_desc.get('content', '') return "" def crawl_multiple_pages(self, urls: List[str]) -> List[CrawlResult]: """Crawl multiple pages and return results.""" logger.info(f"Starting crawl of {len(urls)} pages") results = [] for url in urls: result = self.crawl_page(url) if result: results.append(result) logger.info(f"Crawling completed. Successfully crawled {len(results)} pages") return results def save_results(self, filename: str = "crawl_results.json") -> bool: """Save crawl results to JSON file.""" try: # Convert dataclass objects to dictionaries  results_data = [ { 'url': result.url, 'title': result.title, 'status_code': result.status_code, 'links_count': len(result.links), 'first_10_links': result.links[:10], # Save only first 10 links  'text_preview': result.text_content, 'meta_description': result.meta_description, 'word_count': result.word_count, 'crawl_time': result.crawl_time } for result in self.results ] with open(filename, 'w', encoding='utf-8') as f: json.dump(results_data, f, indent=2, ensure_ascii=False) logger.info(f"Results saved to {filename}") return True except Exception as e: logger.error(f"Error saving results: {e}") return False def get_statistics(self) -> Dict[str, any]: """Get crawling statistics.""" if not self.results: return {"error": "No crawling results available"} total_words = sum(result.word_count for result in self.results) avg_crawl_time = sum(result.crawl_time for result in self.results) / len(self.results) total_links = sum(len(result.links) for result in self.results) return { "pages_crawled": len(self.results), "total_words": total_words, "average_words_per_page": total_words // len(self.results), "total_links_found": total_links, "average_crawl_time": round(avg_crawl_time, 2), "successful_crawls": len(self.results), "domains_crawled": len(set(urlparse(result.url).netloc for result in self.results)) } if __name__ == "__main__": # Example usage  crawler = WebCrawler(delay=1.0) # Example URLs (using HTTP examples that are safe to crawl)  test_urls = [ "http://httpbin.org/html", "http://httpbin.org/robots.txt", ] # Crawl the pages  results = crawler.crawl_multiple_pages(test_urls) # Display results  for result in results: print(f"\n{'='*60}") print(f"URL: {result.url}") print(f"Title: {result.title}") print(f"Status: {result.status_code}") print(f"Word Count: {result.word_count}") print(f"Links Found: {len(result.links)}") print(f"Crawl Time: {result.crawl_time:.2f}s") print(f"Text Preview: {result.text_content[:100]}...") # Show statistics  stats = crawler.get_statistics() print(f"\n{'='*60}") print("CRAWLING STATISTICS") print(f"{'='*60}") for key, value in stats.items(): print(f"{key.replace('_', ' ').title()}: {value}") # Save results  crawler.save_results("web_crawl_results.json")

#!/usr/bin/env python3 """ Example usage of the web crawler. """ from web_crawler import WebCrawler import logging # Configure logging to see crawler activity logging.basicConfig(level=logging.INFO) def main(): """Demonstrate web crawler usage.""" print("🕷️ Web Crawler Example") print("=" * 50) # Create crawler with 2-second delay between requests  crawler = WebCrawler(delay=2.0, max_retries=2) # URLs to crawl (using safe test websites)  urls_to_crawl = [ "http://httpbin.org/html", "http://httpbin.org/robots.txt", "https://jsonplaceholder.typicode.com/", # API service with HTML  ] print(f"Crawling {len(urls_to_crawl)} URLs...") # Perform the crawl  results = crawler.crawl_multiple_pages(urls_to_crawl) # Display detailed results  print(f"\n✅ Successfully crawled {len(results)} pages\n") for i, result in enumerate(results, 1): print(f"📄 Page {i}: {result.url}") print(f" Title: {result.title}") print(f" Status: {result.status_code}") print(f" Words: {result.word_count}") print(f" Links: {len(result.links)}") print(f" Time: {result.crawl_time:.2f}s") if result.meta_description: print(f" Description: {result.meta_description}") print(f" Preview: {result.text_content[:100]}...\n") # Show overall statistics  stats = crawler.get_statistics() print("📊 Crawling Statistics:") print("-" * 30) for key, value in stats.items(): print(f"{key.replace('_', ' ').title()}: {value}") # Save results to file  saved = crawler.save_results("example_crawl_results.json") if saved: print("\n💾 Results saved to example_crawl_results.json") if __name__ == "__main__": main()

pip install language-tool-python

#!/usr/bin/env python3 """ AsciiDoc validator using LanguageTools to check grammar and style. Demonstrates file processing and external API integration. """ import language_tool_python from pathlib import Path from typing import List, Dict, Optional, NamedTuple import re import logging import json from dataclasses import dataclass import argparse # Configure logging logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) class ValidationIssue(NamedTuple): """Structure for validation issues.""" line_number: int column: int message: str rule_id: str suggestions: List[str] context: str @dataclass class FileReport: """Report for a single file.""" file_path: str total_lines: int issues_found: int issues: List[ValidationIssue] processing_time: float class AsciiDocValidator: """Validator for AsciiDoc files using LanguageTools.""" def __init__(self, language: str = 'en-US'): """ Initialize the validator. Args: language: Language code for LanguageTools (default: en-US) """ self.language = language self.tool: Optional[language_tool_python.LanguageTool] = None self.reports: List[FileReport] = [] # Patterns to ignore in AsciiDoc files  self.ignore_patterns = [ r'include::', # Include directives  r'image::', # Image directives  r'\[source,', # Source code blocks  r'----', # Code block delimiters  r'====', # Example block delimiters  r'^\|', # Table rows  r'^\+', # Table continuation  r'^\*\*\*', # Section breaks  r'^\=\=\=', # Headers  r'^\#', # Comments  r'^\[', # Attribute definitions  r'^\:', # Attribute assignments  ] def _initialize_language_tool(self): """Initialize LanguageTools (lazy loading).""" if self.tool is None: logger.info("Initializing LanguageTools... (this may take a moment)") try: self.tool = language_tool_python.LanguageTool(self.language) logger.info("LanguageTools initialized successfully") except Exception as e: logger.error(f"Failed to initialize LanguageTools: {e}") raise def _should_check_line(self, line: str) -> bool: """Determine if a line should be checked for language issues.""" line_stripped = line.strip() # Skip empty lines  if not line_stripped: return False # Check against ignore patterns  for pattern in self.ignore_patterns: if re.match(pattern, line_stripped): return False return True def _extract_text_content(self, file_path: Path) -> List[str]: """Extract text content from AsciiDoc file.""" try: with open(file_path, 'r', encoding='utf-8') as file: lines = file.readlines() # Filter lines that should be checked  text_lines = [] for line_num, line in enumerate(lines, 1): if self._should_check_line(line): text_lines.append((line_num, line.strip())) return text_lines except Exception as e: logger.error(f"Error reading file {file_path}: {e}") return [] def validate_file(self, file_path: Path) -> FileReport: """ Validate a single AsciiDoc file. Args: file_path: Path to the AsciiDoc file Returns: FileReport with validation results """ import time start_time = time.time() logger.info(f"Validating: {file_path}") # Initialize LanguageTools if needed  self._initialize_language_tool() # Extract text content  text_lines = self._extract_text_content(file_path) if not text_lines: processing_time = time.time() - start_time return FileReport( file_path=str(file_path), total_lines=0, issues_found=0, issues=[], processing_time=processing_time ) # Check each line for issues  all_issues = [] for line_number, text in text_lines: try: matches = self.tool.check(text) for match in matches: issue = ValidationIssue( line_number=line_number, column=match.offset, message=match.message, rule_id=match.ruleId, suggestions=[s for s in match.replacements[:3]], # First 3 suggestions  context=text[max(0, match.offset-10):match.offset+match.errorLength+10] ) all_issues.append(issue) except Exception as e: logger.warning(f"Error checking line {line_number}: {e}") processing_time = time.time() - start_time report = FileReport( file_path=str(file_path), total_lines=len(text_lines), issues_found=len(all_issues), issues=all_issues, processing_time=processing_time ) self.reports.append(report) return report def validate_directory(self, directory: Path, pattern: str = "*.adoc") -> List[FileReport]: """ Validate all AsciiDoc files in a directory. Args: directory: Directory to scan pattern: File pattern to match (default: *.adoc) Returns: List of FileReport objects """ logger.info(f"Scanning directory: {directory}") if not directory.exists(): logger.error(f"Directory not found: {directory}") return [] # Find all matching files  adoc_files = list(directory.glob(pattern)) if not adoc_files: logger.warning(f"No {pattern} files found in {directory}") return [] logger.info(f"Found {len(adoc_files)} files to validate") # Validate each file  reports = [] for file_path in adoc_files: report = self.validate_file(file_path) reports.append(report) return reports def generate_report(self, output_file: Optional[str] = None) -> Dict: """Generate summary report of all validations.""" if not self.reports: return {"error": "No validation reports available"} total_files = len(self.reports) total_issues = sum(report.issues_found for report in self.reports) files_with_issues = sum(1 for report in self.reports if report.issues_found > 0) # Group issues by rule ID  issue_types = {} for report in self.reports: for issue in report.issues: rule_id = issue.rule_id if rule_id not in issue_types: issue_types[rule_id] = {"count": 0, "message": issue.message} issue_types[rule_id]["count"] += 1 # Create summary report  summary = { "validation_summary": { "total_files": total_files, "files_with_issues": files_with_issues, "total_issues": total_issues, "average_issues_per_file": round(total_issues / total_files, 2), }, "issue_breakdown": issue_types, "file_reports": [ { "file": report.file_path, "lines_checked": report.total_lines, "issues": report.issues_found, "processing_time": round(report.processing_time, 2) } for report in self.reports ] } # Save to file if requested  if output_file: try: with open(output_file, 'w', encoding='utf-8') as f: json.dump(summary, f, indent=2, ensure_ascii=False) logger.info(f"Report saved to {output_file}") except Exception as e: logger.error(f"Error saving report: {e}") return summary def print_detailed_report(self): """Print detailed validation report to console.""" if not self.reports: print("No validation reports available.") return print("\n" + "="*60) print("ASCIIDOC VALIDATION REPORT") print("="*60) total_issues = sum(report.issues_found for report in self.reports) files_with_issues = [r for r in self.reports if r.issues_found > 0] print(f"Files scanned: {len(self.reports)}") print(f"Files with issues: {len(files_with_issues)}") print(f"Total issues found: {total_issues}") print("\n" + "-"*60) # Show issues by file  for report in self.reports: if report.issues_found > 0: print(f"\n📄 {Path(report.file_path).name}") print(f" Issues: {report.issues_found}") for issue in report.issues[:5]: # Show first 5 issues  print(f" Line {issue.line_number}: {issue.message}") if issue.suggestions: suggestions = ", ".join(issue.suggestions) print(f" Suggestions: {suggestions}") if len(report.issues) > 5: print(f" ... and {len(report.issues) - 5} more issues") def main(): """Command-line interface for the validator.""" parser = argparse.ArgumentParser(description="Validate AsciiDoc files using LanguageTools") parser.add_argument("path", help="File or directory path to validate") parser.add_argument("--language", default="en-US", help="Language code (default: en-US)") parser.add_argument("--output", help="Output file for JSON report") parser.add_argument("--pattern", default="*.adoc", help="File pattern for directory scanning") args = parser.parse_args() validator = AsciiDocValidator(language=args.language) path = Path(args.path) if path.is_file(): # Validate single file  validator.validate_file(path) elif path.is_dir(): # Validate directory  validator.validate_directory(path, args.pattern) else: print(f"Error: Path {path} does not exist") return # Generate and display report  validator.print_detailed_report() if args.output: validator.generate_report(args.output) if __name__ == "__main__": main()