# Python Performance Tips & Hidden Optimizations

> Deep dive into Python internals and performance optimization.

---

## 1. Local vs Global Variable Lookup

### The Bytecode Difference

```python
import dis

global_var = 10

def use_global():
    return global_var + 1  # LOAD_GLOBAL — slower

def use_local(x):
    return x + 1           # LOAD_FAST — faster

dis.dis(use_global)
#   2           0 LOAD_GLOBAL              0 (global_var)
#               2 LOAD_CONST               1 (1)
#               4 BINARY_OP                0 (+)
#               6 RETURN_VALUE

dis.dis(use_local)
#   2           0 LOAD_FAST                0 (x)
#               2 LOAD_CONST               1 (1)
#               4 BINARY_OP                0 (+)
#               6 RETURN_VALUE
```

### Performance Impact

```python
import timeit

# Global lookup: ~100ns
timeit.timeit('global_var + 1', setup='global_var = 10', number=10_000_000)

# Local lookup: ~50ns
timeit.timeit('x + 1', setup='x = 10', number=10_000_000)

# Optimization: cache globals in locals
def optimized():
    local_var = global_var  # LOAD_GLOBAL once
    total = 0
    for _ in range(1_000_000):
        total += local_var  # LOAD_FAST every iteration
    return total
```

### Best Practices

```python
# BAD: Repeated global lookups in loops
PI = 3.14159
def bad_circle_areas(radii):
    areas = []
    for r in radii:
        areas.append(PI * r * r)  # LOAD_GLOBAL every iteration
    return areas

# GOOD: Cache globals in local
def good_circle_areas(radii):
    local_pi = PI  # LOAD_GLOBAL once
    areas = []
    for r in radii:
        areas.append(local_pi * r * r)  # LOAD_FAST
    return areas

# BETTER: Use list comprehension (faster than .append())
def best_circle_areas(radii):
    local_pi = PI
    return [local_pi * r * r for r in radii]
```

---

## 2. `__slots__` — Memory Optimization

```python
import sys

class PointNormal:
    def __init__(self, x, y):
        self.x = x
        self.y = y

class PointSlotted:
    __slots__ = ('x', 'y')
    def __init__(self, x, y):
        self.x = x
        self.y = y

normal = PointNormal(1, 2)
slotted = PointSlotted(1, 2)

print(sys.getsizeof(normal))   # ~152 bytes
print(sys.getsizeof(slotted))  # ~56 bytes
print(f"Savings: {(152-56)/152*100:.0f}%")  # ~63%

# At scale: 10 million instances
# Normal: ~1.5 GB
# Slotted: ~0.5 GB
# Savings: ~1 GB

# Inheritance: all classes need __slots__
class Animal:
    __slots__ = ('name',)
    def __init__(self, name):
        self.name = name

class Dog(Animal):
    __slots__ = ('breed',)
    def __init__(self, name, breed):
        super().__init__(name)
        self.breed = breed

# Without __slots__ in Dog, instances get __dict__
```

---

## 3. List Comprehension vs Map/Filter

```python
import timeit

data = list(range(100_000))

# List comprehension (fastest for most cases)
timeit.timeit('[x*2 for x in data]', setup='data = list(range(100_000))', number=1000)
# ~0.5s

# Map with lambda (slower)
timeit.timeit('list(map(lambda x: x*2, data))', setup='data = list(range(100_000))', number=1000)
# ~0.8s

# Map with builtin (can be faster for simple cases)
def double(x):
    return x * 2

timeit.timeit('list(map(double, data))', setup='data = list(range(100_000)); def double(x): return x*2', number=1000)
# ~0.6s

# For filtering: list comprehension is clearer and comparable speed
timeit.timeit('[x for x in data if x % 2 == 0]', setup='data = list(range(100_000))', number=1000)
# ~0.6s

timeit.timeit('list(filter(lambda x: x % 2 == 0, data))', setup='data = list(range(100_000))', number=1000)
# ~0.9s
```

---

## 4. String Concatenation

```python
import timeit

# BAD: String concatenation in loop (creates new string each iteration)
def bad_concat(n):
    result = ""
    for i in range(n):
        result += str(i)  # O(n²) — creates new string each time
    return result

# GOOD: Join (creates string once at the end)
def good_concat(n):
    return "".join(str(i) for i in range(n))  # O(n)

# BEST: Pre-allocated list then join
def best_concat(n):
    parts = []
    for i in range(n):
        parts.append(str(i))
    return "".join(parts)  # O(n)

timeit.timeit('good_concat(1000)', setup='def good_concat(n): return "".join(str(i) for i in range(n))', number=1000)
# ~0.2s

timeit.timeit('bad_concat(1000)', setup='def bad_concat(n): r=""; [r:=r+str(i) for i in range(n)]; return r', number=1000)
# ~1.5s (7x slower!)
```

---

## 5. Dict/Set Lookups — O(1) Average

```python
import timeit

data = list(range(1_000_000))

# Linear search (O(n))
timeit.timeit('999999 in data', setup='data = list(range(1_000_000))', number=10_000)
# ~5s

# Set lookup (O(1))
timeit.timeit('999999 in data_set', setup='data_set = set(range(1_000_000))', number=10_000)
# ~0.01s (500x faster!)

# Dict lookup (O(1))
data_dict = {i: i*2 for i in range(1_000_000)}
timeit.timeit('999999 in data_dict', setup='data_dict = {i: i*2 for i in range(1_000_000)}', number=10_000)
# ~0.01s

# Use sets for membership testing, deduplication
def deduplicate(items):
    return list(dict.fromkeys(items))  # preserves order

# Or simply
def deduplicate_set(items):
    return list(set(items))  # loses order
```

---

## 6. Generator Memory Savings

```python
import sys
import tracemalloc

# List comprehension loads everything into memory
tracemalloc.start()
big_list = [i * i for i in range(1_000_000)]
current, peak = tracemalloc.get_traced_memory()
print(f"List: {peak / 1024 / 1024:.1f} MB peak")  # ~32 MB
tracemalloc.stop()

# Generator expression: lazy evaluation, constant memory
tracemalloc.start()
big_gen = (i * i for i in range(1_000_000))
current, peak = tracemalloc.get_traced_memory()
print(f"Generator: {peak / 1024 / 1024:.1f} MB peak")  # ~0.1 MB
tracemalloc.stop()

# When to use generators:
# ✅ Processing large files
# ✅ Infinite sequences
# ✅ Pipeline of transformations
# ❌ Need to iterate multiple times
# ❌ Need random access

# Pipeline example (memory efficient)
def process_large_file(filename):
    with open(filename) as f:
        lines = (line.strip() for line in f)           # Step 1: strip
        non_empty = (line for line in lines if line)   # Step 2: filter
        upper = (line.upper() for line in non_empty)   # Step 3: transform
        count = sum(1 for line in upper)               # Step 4: count
    return count
# Memory usage: O(1) regardless of file size
```

---

## 7. String Interning

```python
# Python automatically interns:
# 1. String literals in code
# 2. Small strings (implementation-dependent)
# 3. Strings that look like identifiers

# Automatic interning
a = "hello"
b = "hello"
print(a is b)  # True — same object (interned)

# Not automatically interned
a = "hello world"  # Contains space → may not be interned
b = "hello world"
print(a is b)  # May be False

# Manual interning (Python 3.12+)
import sys
a = "dynamic" + " string"
b = sys.intern("dynamic" + " string")
c = sys.intern("dynamic" + " string")
print(b is c)  # True

# Integer interning (-5 to 256)
a = 256
b = 256
print(a is b)  # True

c = 257
d = 257
print(c is d)  # True (in same code block, CPython optimization)
# Note: implementation-dependent, don't rely on it
```

---

## 8. `collections` Performance

```python
from collections import Counter, defaultdict, deque, namedtuple
import timeit

# Counter — O(n) counting
words = ["apple", "banana", "apple", "cherry", "banana", "apple"]
counts = Counter(words)
print(counts.most_common(2))  # [('apple', 3), ('banana', 2)]

# defaultdict — avoid KeyError
from collections import defaultdict
groups = defaultdict(list)
for name, group in [("alice", "A"), ("bob", "B"), ("charlie", "A")]:
    groups[group].append(name)
print(dict(groups))  # {'A': ['alice', 'charlie'], 'B': ['bob']}

# deque — O(1) append/pop from both ends
queue = deque(maxlen=100)  # Fixed-size queue
queue.append("new item")
old = queue.popleft()

# namedtuple — memory efficient, attribute access
from collections import namedtuple
Point = namedtuple('Point', ['x', 'y'])
p = Point(1, 2)
print(p.x, p.y)  # 1 2
# Uses ~40% less memory than regular class with __slots__
```

---

## 9. Function Call Overhead

```python
import timeit

# Global function call
def add(a, b):
    return a + b

timeit.timeit('add(1, 2)', setup='def add(a, b): return a + b', number=10_000_000)
# ~0.8s

# Lambda (slightly slower due to extra indirection)
add_lambda = lambda a, b: a + b
timeit.timeit('add_lambda(1, 2)', setup='add_lambda = lambda a, b: a + b', number=10_000_000)
# ~1.0s

# operator.add (fastest for simple operations)
from operator import add as op_add
timeit.timeit('op_add(1, 2)', setup='from operator import add as op_add', number=10_000_000)
# ~0.4s

# Method calls have overhead too
class MyClass:
    def method(self):
        pass

obj = MyClass()
timeit.timeit('obj.method()', setup='obj = MyClass()', number=10_000_000)
# ~1.2s
```

---

## 10. Memory Profiling with `tracemalloc`

```python
import tracemalloc

tracemalloc.start()

# Take snapshots at different points
snapshot1 = tracemalloc.take_snapshot()

# Allocate memory
big_list = [i for i in range(1_000_000)]
snapshot2 = tracemalloc.take_snapshot()

big_dict = {i: i*2 for i in range(500_000)}
snapshot3 = tracemalloc.take_snapshot()

# Compare snapshots
stats2 = snapshot2.compare_to(snapshot1, 'lineno')
print("After creating list:")
for stat in stats2[:3]:
    print(f"  {stat}")

stats3 = snapshot3.compare_to(snapshot2, 'lineno')
print("After creating dict:")
for stat in stats3[:3]:
    print(f"  {stat}")

# Statistics by filename
stats = snapshot3.statistics('filename')
for stat in stats[:5]:
    print(f"  {stat}")

tracemalloc.stop()
```

---

## 11. Caching and Memoization

```python
from functools import lru_cache, cached_property

# @lru_cache — automatic memoization
@lru_cache(maxsize=128)
def fibonacci(n: int) -> int:
    if n < 2:
        return n
    return fibonacci(n - 1) + fibonacci(n - 2)

print(fibonacci(100))  # Fast — cached results

# Inspect cache
print(fibonacci.cache_info())
# CacheInfo(hits=98, misses=100, maxsize=128, currsize=100)

# @cached_property — per-instance caching
class Circle:
    def __init__(self, radius):
        self.radius = radius

    @cached_property
    def area(self):
        print("Computing area...")  # Only once
        return 3.14159 * self.radius ** 2

circle = Circle(10)
print(circle.area)  # Computing area... 314.159
print(circle.area)  # 314.159 (cached)

# Custom cache with TTL (time-to-live)
import time
from functools import wraps

def ttl_cache(ttl_seconds=60):
    def decorator(func):
        cache = {}
        @wraps(func)
        def wrapper(*args, **kwargs):
            now = time.time()
            key = (args, tuple(sorted(kwargs.items())))
            if key in cache:
                value, timestamp = cache[key]
                if now - timestamp < ttl_seconds:
                    return value
            result = func(*args, **kwargs)
            cache[key] = (result, now)
            # Evict old entries
            cache = {k: v for k, v in cache.items() if now - v[1] < ttl_seconds}
            return result
        return wrapper
    return decorator

@ttl_cache(ttl_seconds=60)
def fetch_data(url):
    print(f"Fetching {url}...")
    return {"data": "result"}
```

---

## 12. I/O Performance

```python
import io
import os

# Buffered I/O (default)
with open('large_file.txt', 'r', buffering=8192) as f:
    content = f.read()  # Reads in 8KB chunks

# Line-by-line (memory efficient)
with open('large_file.txt', 'r') as f:
    for line in f:  # Iterates line by line
        process(line)

# mmap for large files (fastest for random access)
import mmap
with open('large_file.txt', 'r+b') as f:
    mm = mmap.mmap(f.fileno(), 0)
    # Access like a string/array
    first_100 = mm[:100]
    mm.close()

# Async I/O for network operations
import asyncio
import aiohttp

async def fetch_urls(urls):
    async with aiohttp.ClientSession() as session:
        tasks = [session.get(url) for url in urls]
        responses = await asyncio.gather(*tasks)
        return [await r.text() for r in responses]

# Multiple file operations
async def process_files(file_paths):
    loop = asyncio.get_event_loop()
    tasks = [loop.run_in_executor(None, read_file, path) for path in file_paths]
    return await asyncio.gather(*tasks)
```

---

## Performance Checklist

| Optimization | Impact | Effort | When |
|-------------|--------|--------|------|
| Use local vars | High | Low | Always |
| List comprehensions | Medium | Low | Always |
| `__slots__` | High (memory) | Medium | Many instances |
| Sets for membership | High | Low | Lookups |
| `lru_cache` | High | Low | Repeated calls |
| `itertools` | Medium | Medium | Complex loops |
| `deque` | Medium | Low | Queue operations |
| `mmap` | High | High | Large files |
| Multiprocessing | High | High | CPU-bound tasks |
| `tracemalloc` | Diagnostic | Low | Debugging |
| Cython/C extensions | Very High | High | Hot loops |