# Python Performance Tips & Hidden Optimizations
> Deep dive into Python internals and performance optimization.
---
## 1. Local vs Global Variable Lookup
### The Bytecode Difference
```python
import dis
global_var = 10
def use_global():
return global_var + 1 # LOAD_GLOBAL ā slower
def use_local(x):
return x + 1 # LOAD_FAST ā faster
dis.dis(use_global)
# 2 0 LOAD_GLOBAL 0 (global_var)
# 2 LOAD_CONST 1 (1)
# 4 BINARY_OP 0 (+)
# 6 RETURN_VALUE
dis.dis(use_local)
# 2 0 LOAD_FAST 0 (x)
# 2 LOAD_CONST 1 (1)
# 4 BINARY_OP 0 (+)
# 6 RETURN_VALUE
```
### Performance Impact
```python
import timeit
# Global lookup: ~100ns
timeit.timeit('global_var + 1', setup='global_var = 10', number=10_000_000)
# Local lookup: ~50ns
timeit.timeit('x + 1', setup='x = 10', number=10_000_000)
# Optimization: cache globals in locals
def optimized():
local_var = global_var # LOAD_GLOBAL once
total = 0
for _ in range(1_000_000):
total += local_var # LOAD_FAST every iteration
return total
```
### Best Practices
```python
# BAD: Repeated global lookups in loops
PI = 3.14159
def bad_circle_areas(radii):
areas = []
for r in radii:
areas.append(PI * r * r) # LOAD_GLOBAL every iteration
return areas
# GOOD: Cache globals in local
def good_circle_areas(radii):
local_pi = PI # LOAD_GLOBAL once
areas = []
for r in radii:
areas.append(local_pi * r * r) # LOAD_FAST
return areas
# BETTER: Use list comprehension (faster than .append())
def best_circle_areas(radii):
local_pi = PI
return [local_pi * r * r for r in radii]
```
---
## 2. `__slots__` ā Memory Optimization
```python
import sys
class PointNormal:
def __init__(self, x, y):
self.x = x
self.y = y
class PointSlotted:
__slots__ = ('x', 'y')
def __init__(self, x, y):
self.x = x
self.y = y
normal = PointNormal(1, 2)
slotted = PointSlotted(1, 2)
print(sys.getsizeof(normal)) # ~152 bytes
print(sys.getsizeof(slotted)) # ~56 bytes
print(f"Savings: {(152-56)/152*100:.0f}%") # ~63%
# At scale: 10 million instances
# Normal: ~1.5 GB
# Slotted: ~0.5 GB
# Savings: ~1 GB
# Inheritance: all classes need __slots__
class Animal:
__slots__ = ('name',)
def __init__(self, name):
self.name = name
class Dog(Animal):
__slots__ = ('breed',)
def __init__(self, name, breed):
super().__init__(name)
self.breed = breed
# Without __slots__ in Dog, instances get __dict__
```
---
## 3. List Comprehension vs Map/Filter
```python
import timeit
data = list(range(100_000))
# List comprehension (fastest for most cases)
timeit.timeit('[x*2 for x in data]', setup='data = list(range(100_000))', number=1000)
# ~0.5s
# Map with lambda (slower)
timeit.timeit('list(map(lambda x: x*2, data))', setup='data = list(range(100_000))', number=1000)
# ~0.8s
# Map with builtin (can be faster for simple cases)
def double(x):
return x * 2
timeit.timeit('list(map(double, data))', setup='data = list(range(100_000)); def double(x): return x*2', number=1000)
# ~0.6s
# For filtering: list comprehension is clearer and comparable speed
timeit.timeit('[x for x in data if x % 2 == 0]', setup='data = list(range(100_000))', number=1000)
# ~0.6s
timeit.timeit('list(filter(lambda x: x % 2 == 0, data))', setup='data = list(range(100_000))', number=1000)
# ~0.9s
```
---
## 4. String Concatenation
```python
import timeit
# BAD: String concatenation in loop (creates new string each iteration)
def bad_concat(n):
result = ""
for i in range(n):
result += str(i) # O(nĀ²) ā creates new string each time
return result
# GOOD: Join (creates string once at the end)
def good_concat(n):
return "".join(str(i) for i in range(n)) # O(n)
# BEST: Pre-allocated list then join
def best_concat(n):
parts = []
for i in range(n):
parts.append(str(i))
return "".join(parts) # O(n)
timeit.timeit('good_concat(1000)', setup='def good_concat(n): return "".join(str(i) for i in range(n))', number=1000)
# ~0.2s
timeit.timeit('bad_concat(1000)', setup='def bad_concat(n): r=""; [r:=r+str(i) for i in range(n)]; return r', number=1000)
# ~1.5s (7x slower!)
```
---
## 5. Dict/Set Lookups ā O(1) Average
```python
import timeit
data = list(range(1_000_000))
# Linear search (O(n))
timeit.timeit('999999 in data', setup='data = list(range(1_000_000))', number=10_000)
# ~5s
# Set lookup (O(1))
timeit.timeit('999999 in data_set', setup='data_set = set(range(1_000_000))', number=10_000)
# ~0.01s (500x faster!)
# Dict lookup (O(1))
data_dict = {i: i*2 for i in range(1_000_000)}
timeit.timeit('999999 in data_dict', setup='data_dict = {i: i*2 for i in range(1_000_000)}', number=10_000)
# ~0.01s
# Use sets for membership testing, deduplication
def deduplicate(items):
return list(dict.fromkeys(items)) # preserves order
# Or simply
def deduplicate_set(items):
return list(set(items)) # loses order
```
---
## 6. Generator Memory Savings
```python
import sys
import tracemalloc
# List comprehension loads everything into memory
tracemalloc.start()
big_list = [i * i for i in range(1_000_000)]
current, peak = tracemalloc.get_traced_memory()
print(f"List: {peak / 1024 / 1024:.1f} MB peak") # ~32 MB
tracemalloc.stop()
# Generator expression: lazy evaluation, constant memory
tracemalloc.start()
big_gen = (i * i for i in range(1_000_000))
current, peak = tracemalloc.get_traced_memory()
print(f"Generator: {peak / 1024 / 1024:.1f} MB peak") # ~0.1 MB
tracemalloc.stop()
# When to use generators:
# ā
Processing large files
# ā
Infinite sequences
# ā
Pipeline of transformations
# ā Need to iterate multiple times
# ā Need random access
# Pipeline example (memory efficient)
def process_large_file(filename):
with open(filename) as f:
lines = (line.strip() for line in f) # Step 1: strip
non_empty = (line for line in lines if line) # Step 2: filter
upper = (line.upper() for line in non_empty) # Step 3: transform
count = sum(1 for line in upper) # Step 4: count
return count
# Memory usage: O(1) regardless of file size
```
---
## 7. String Interning
```python
# Python automatically interns:
# 1. String literals in code
# 2. Small strings (implementation-dependent)
# 3. Strings that look like identifiers
# Automatic interning
a = "hello"
b = "hello"
print(a is b) # True ā same object (interned)
# Not automatically interned
a = "hello world" # Contains space ā may not be interned
b = "hello world"
print(a is b) # May be False
# Manual interning (Python 3.12+)
import sys
a = "dynamic" + " string"
b = sys.intern("dynamic" + " string")
c = sys.intern("dynamic" + " string")
print(b is c) # True
# Integer interning (-5 to 256)
a = 256
b = 256
print(a is b) # True
c = 257
d = 257
print(c is d) # True (in same code block, CPython optimization)
# Note: implementation-dependent, don't rely on it
```
---
## 8. `collections` Performance
```python
from collections import Counter, defaultdict, deque, namedtuple
import timeit
# Counter ā O(n) counting
words = ["apple", "banana", "apple", "cherry", "banana", "apple"]
counts = Counter(words)
print(counts.most_common(2)) # [('apple', 3), ('banana', 2)]
# defaultdict ā avoid KeyError
from collections import defaultdict
groups = defaultdict(list)
for name, group in [("alice", "A"), ("bob", "B"), ("charlie", "A")]:
groups[group].append(name)
print(dict(groups)) # {'A': ['alice', 'charlie'], 'B': ['bob']}
# deque ā O(1) append/pop from both ends
queue = deque(maxlen=100) # Fixed-size queue
queue.append("new item")
old = queue.popleft()
# namedtuple ā memory efficient, attribute access
from collections import namedtuple
Point = namedtuple('Point', ['x', 'y'])
p = Point(1, 2)
print(p.x, p.y) # 1 2
# Uses ~40% less memory than regular class with __slots__
```
---
## 9. Function Call Overhead
```python
import timeit
# Global function call
def add(a, b):
return a + b
timeit.timeit('add(1, 2)', setup='def add(a, b): return a + b', number=10_000_000)
# ~0.8s
# Lambda (slightly slower due to extra indirection)
add_lambda = lambda a, b: a + b
timeit.timeit('add_lambda(1, 2)', setup='add_lambda = lambda a, b: a + b', number=10_000_000)
# ~1.0s
# operator.add (fastest for simple operations)
from operator import add as op_add
timeit.timeit('op_add(1, 2)', setup='from operator import add as op_add', number=10_000_000)
# ~0.4s
# Method calls have overhead too
class MyClass:
def method(self):
pass
obj = MyClass()
timeit.timeit('obj.method()', setup='obj = MyClass()', number=10_000_000)
# ~1.2s
```
---
## 10. Memory Profiling with `tracemalloc`
```python
import tracemalloc
tracemalloc.start()
# Take snapshots at different points
snapshot1 = tracemalloc.take_snapshot()
# Allocate memory
big_list = [i for i in range(1_000_000)]
snapshot2 = tracemalloc.take_snapshot()
big_dict = {i: i*2 for i in range(500_000)}
snapshot3 = tracemalloc.take_snapshot()
# Compare snapshots
stats2 = snapshot2.compare_to(snapshot1, 'lineno')
print("After creating list:")
for stat in stats2[:3]:
print(f" {stat}")
stats3 = snapshot3.compare_to(snapshot2, 'lineno')
print("After creating dict:")
for stat in stats3[:3]:
print(f" {stat}")
# Statistics by filename
stats = snapshot3.statistics('filename')
for stat in stats[:5]:
print(f" {stat}")
tracemalloc.stop()
```
---
## 11. Caching and Memoization
```python
from functools import lru_cache, cached_property
# @lru_cache ā automatic memoization
@lru_cache(maxsize=128)
def fibonacci(n: int) -> int:
if n < 2:
return n
return fibonacci(n - 1) + fibonacci(n - 2)
print(fibonacci(100)) # Fast ā cached results
# Inspect cache
print(fibonacci.cache_info())
# CacheInfo(hits=98, misses=100, maxsize=128, currsize=100)
# @cached_property ā per-instance caching
class Circle:
def __init__(self, radius):
self.radius = radius
@cached_property
def area(self):
print("Computing area...") # Only once
return 3.14159 * self.radius ** 2
circle = Circle(10)
print(circle.area) # Computing area... 314.159
print(circle.area) # 314.159 (cached)
# Custom cache with TTL (time-to-live)
import time
from functools import wraps
def ttl_cache(ttl_seconds=60):
def decorator(func):
cache = {}
@wraps(func)
def wrapper(*args, **kwargs):
now = time.time()
key = (args, tuple(sorted(kwargs.items())))
if key in cache:
value, timestamp = cache[key]
if now - timestamp < ttl_seconds:
return value
result = func(*args, **kwargs)
cache[key] = (result, now)
# Evict old entries
cache = {k: v for k, v in cache.items() if now - v[1] < ttl_seconds}
return result
return wrapper
return decorator
@ttl_cache(ttl_seconds=60)
def fetch_data(url):
print(f"Fetching {url}...")
return {"data": "result"}
```
---
## 12. I/O Performance
```python
import io
import os
# Buffered I/O (default)
with open('large_file.txt', 'r', buffering=8192) as f:
content = f.read() # Reads in 8KB chunks
# Line-by-line (memory efficient)
with open('large_file.txt', 'r') as f:
for line in f: # Iterates line by line
process(line)
# mmap for large files (fastest for random access)
import mmap
with open('large_file.txt', 'r+b') as f:
mm = mmap.mmap(f.fileno(), 0)
# Access like a string/array
first_100 = mm[:100]
mm.close()
# Async I/O for network operations
import asyncio
import aiohttp
async def fetch_urls(urls):
async with aiohttp.ClientSession() as session:
tasks = [session.get(url) for url in urls]
responses = await asyncio.gather(*tasks)
return [await r.text() for r in responses]
# Multiple file operations
async def process_files(file_paths):
loop = asyncio.get_event_loop()
tasks = [loop.run_in_executor(None, read_file, path) for path in file_paths]
return await asyncio.gather(*tasks)
```
---
## Performance Checklist
| Optimization | Impact | Effort | When |
|-------------|--------|--------|------|
| Use local vars | High | Low | Always |
| List comprehensions | Medium | Low | Always |
| `__slots__` | High (memory) | Medium | Many instances |
| Sets for membership | High | Low | Lookups |
| `lru_cache` | High | Low | Repeated calls |
| `itertools` | Medium | Medium | Complex loops |
| `deque` | Medium | Low | Queue operations |
| `mmap` | High | High | Large files |
| Multiprocessing | High | High | CPU-bound tasks |
| `tracemalloc` | Diagnostic | Low | Debugging |
| Cython/C extensions | Very High | High | Hot loops |