# Python Advanced Features & Dark Patterns
> Comprehensive reference with practical examples.
---
## 1. Metaclasses — Classes That Create Classes
Metaclasses control class creation. `type` is the default metaclass.
### How They Work
```python
class MyClass:
pass
# Behind the scenes:
# MyClass = type.__call__('MyClass', (object,), {})
# Which calls: type.__new__() then type.__init__()
# You can verify:
print(type(MyClass)) # <class 'type'>
print(type(MyClass()) == MyClass) # True
```
### Practical Example: Auto-Registering Plugin System
```python
class PluginRegistryMeta(type):
"""Metaclass that automatically registers subclasses."""
_registry = {}
def __new__(mcs, name, bases, namespace):
cls = super().__new__(mcs, name, bases, namespace)
if bases != (object,):
mcs._registry[cls.__name__] = cls
return cls
@classmethod
def get_plugin(mcs, name):
return mcs._registry.get(name)
@classmethod
def list_plugins(mcs):
return list(mcs._registry.keys())
class BasePlugin(metaclass=PluginRegistryMeta):
def execute(self):
raise NotImplementedError
class LoggerPlugin(BasePlugin):
def execute(self):
print("[Logger] Logging initialized")
class CachePlugin(BasePlugin):
def execute(self):
print("[Cache] Cache warmed up")
# Usage:
print(BasePlugin.list_plugins()) # ['LoggerPlugin', 'CachePlugin']
plugin = BasePlugin.get_plugin('CachePlugin')()
plugin.execute() # [Cache] Cache warmed up
```
### Practical Example: Enforcing Method Signatures
```python
class EnforceAPI(type):
"""Metaclass that validates all subclasses implement required methods."""
REQUIRED_METHODS = None
def __new__(mcs, name, bases, namespace):
cls = super().__new__(mcs, name, bases, namespace)
if mcs.REQUIRED_METHODS and bases != (object,):
missing = [m for m in mcs.REQUIRED_METHODS if m not in namespace]
if missing:
raise TypeError(f"{name} is missing required methods: {missing}")
return cls
class Serializable(metaclass=EnforceAPI):
REQUIRED_METHODS = ['to_dict', 'from_dict']
# class BadSerializable(Serializable):
# def to_dict(self): return {}
# # Missing from_dict → TypeError at class definition time!
```
### When to Use Metaclasses
- Framework design (Django ORM, SQLAlchemy)
- Plugin systems
- Code generation
- Validation at class creation time
> **Rule of thumb:** If you need to understand metaclasses, you probably don't need them yet. (Tim Peters)
---
## 2. Descriptors — Controlling Attribute Access
Descriptors implement `__get__`, `__set__`, or `__delete__`. Properties use them.
### Data Descriptor (has `__set__`)
```python
class Validated:
"""Descriptor that enforces a type and optional constraints."""
def __init__(self, expected_type, min_val=None, max_val=None):
self.expected_type = expected_type
self.min_val = min_val
self.max_val = max_val
def __set_name__(self, owner, name):
self.storage_name = f'_val_{name}'
def __get__(self, obj, objtype=None):
if obj is None:
return self
return getattr(obj, self.storage_name, None)
def __set__(self, obj, value):
if not isinstance(value, self.expected_type):
raise TypeError(f"Expected {self.expected_type.__name__}, got {type(value).__name__}")
if self.min_val is not None and value < self.min_val:
raise ValueError(f"Value {value} < min {self.min_val}")
if self.max_val is not None and value > self.max_val:
raise ValueError(f"Value {value} > max {self.max_val}")
object.__setattr__(obj, self.storage_name, value)
def __delete__(self, obj):
delattr(obj, self.storage_name)
class Person:
age = Validated(int, min_val=0, max_val=150)
score = Validated(float, min_val=0.0, max_val=1.0)
name = Validated(str)
p = Person()
p.age = 30 # OK
p.age = -1 # ValueError
p.age = "thirty" # TypeError
```
### Non-Data Descriptor (no `__set__`) — Cached Property
```python
class CachedResult:
"""Non-data descriptor that caches the result of a method."""
def __init__(self, func):
self.func = func
self.attrname = None
def __set_name__(self, owner, name):
self.attrname = f'_cached_{name}'
def __get__(self, obj, objtype=None):
if obj is None:
return self
cache = getattr(obj, self.attrname, None)
if cache is None:
cache = self.func(obj)
object.__setattr__(obj, self.attrname, cache)
return cache
class ExpensiveComputation:
@CachedResult
def heavy_calc(self):
print(" [Computing...]")
return sum(i * i for i in range(10000))
obj = ExpensiveComputation()
print(obj.heavy_calc) # [Computing...] → cached
print(obj.heavy_calc) # returns cached value, no recomputation
```
### Descriptor Precedence Order
```
1. Data descriptors (have __set__)
2. Instance __dict__
3. Non-data descriptors (no __set__)
4. __getattr__ fallback
```
---
## 3. `__slots__` — Memory Optimization
Eliminates per-instance `__dict__`, saving memory.
```python
import sys
class PointNormal:
def __init__(self, x, y):
self.x = x
self.y = y
class PointSlotted:
__slots__ = ('x', 'y')
def __init__(self, x, y):
self.x = x
self.y = y
normal = PointNormal(1, 2)
slotted = PointSlotted(1, 2)
print(sys.getsizeof(normal)) # ~152 bytes (includes __dict__)
print(sys.getsizeof(slotted)) # ~56 bytes (fixed layout)
# At scale: 10M instances saves ~1 GB
# Inheritance: each class needs __slots__
class BaseAnimal:
__slots__ = ('name',)
def __init__(self, name):
self.name = name
class Dog(BaseAnimal):
__slots__ = ('breed',)
def __init__(self, name, breed):
super().__init__(name)
self.breed = breed
# Caveat: __slots__ prevents dynamic attributes
# Hybrid: __slots__ = ('x', '__dict__') # x is fast, rest is dynamic
```
---
## 4. MRO — Method Resolution Order
Python uses C3 linearization for multiple inheritance.
```python
class A:
def process(self):
return "Base"
class MixinA:
def process(self):
return "A(" + super().process() + ")"
class MixinB:
def process(self):
return "B(" + super().process() + ")"
class Impl(MixinA, MixinB, A):
pass
print(Impl.__mro__)
# (Impl, MixinA, MixinB, A, object)
print(Impl().process())
# 'A(B(Base))' — calls flow: Impl → MixinA → MixinB → A
# Diamond problem:
class Base: pass
class Left(Base): pass
class Right(Base): pass
class Diamond(Left, Right): pass
print(Diamond.__mro__)
# (Diamond, Left, Right, Base, object)
# Conflicting order → TypeError:
class X: pass
class Y: pass
class Z(X, Y): pass
class P(Y, X): pass
# class Q(Z, P): pass # TypeError!
```
---
## 5. `__new__` vs `__init__`
`__new__` creates the instance; `__init__` initializes it.
### Singleton via `__new__`
```python
class Singleton:
_instance = None
def __new__(cls, *args, **kwargs):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self, value=0):
if not hasattr(self, '_initialized'):
self.value = value
self._initialized = True
a = Singleton(10)
b = Singleton(20)
print(a is b) # True
print(a.value) # 10 (not 20 — __init__ guard protects it)
```
### Factory via `__new__`
```python
class ShapeFactory:
def __new__(cls, shape_type, *args, **kwargs):
if shape_type == 'circle':
return Circle(*args, **kwargs)
elif shape_type == 'rect':
return Rectangle(*args, **kwargs)
class Circle:
def __init__(self, radius):
self.radius = radius
def area(self): return 3.14159 * self.radius ** 2
class Rectangle:
def __init__(self, w, h):
self.w, self.h = w, h
def area(self): return self.w * self.h
s = ShapeFactory('circle', 5)
print(type(s).__name__) # Circle
print(s.area()) # 78.539...
```
### Customizing Immutable Types
```python
class FixedLengthTuple(tuple):
def __new__(cls, items, max_len=3):
if len(items) > max_len:
raise ValueError(f"Too many items: {len(items)} > {max_len}")
return super().__new__(cls, items)
t = FixedLengthTuple([1, 2, 3])
# FixedLengthTuple([1,2,3,4]) # ValueError
```
---
## 6. Generator Tricks
### `.send()` — Two-Way Communication
```python
def accumulator():
total = 0
while True:
received = yield total
total += received
acc = accumulator()
next(acc) # Prime: 0
print(acc.send(10)) # 10
print(acc.send(25)) # 35
print(acc.send(-5)) # 30
```
### `.throw()` — Injecting Exceptions
```python
def safe_divider():
total = 0
count = 0
while True:
try:
value = yield total / max(count, 1)
total += value
count += 1
except ValueError as e:
print(f" Caught ValueError: {e}")
yield -1
div = safe_divider()
next(div) # 0.0
div.send(10) # 10.0
div.send(20) # 15.0
div.throw(ValueError, "bad input")
# Caught ValueError: bad input
# -1
```
### `yield from` — Delegation
```python
def sub_gen(n):
for i in range(n):
yield i * i
def main_gen():
yield from sub_gen(3) # yields 0, 1, 4
yield '---'
yield from sub_gen(5) # yields 0, 1, 4, 9, 16
print(list(main_gen()))
# [0, 1, 4, '---', 0, 1, 4, 9, 16]
```
---
## 7. Python Gotchas (Dark Patterns)
### Mutable Default Arguments
```python
# BAD: mutable default is SHARED across ALL calls
def append_to(item, target=[]):
target.append(item)
return target
print(append_to(1)) # [1]
print(append_to(2)) # [1, 2] ← BUG!
# FIXED: use None sentinel
def append_to_fixed(item, target=None):
if target is None:
target = []
target.append(item)
return target
```
### Late Binding in Closures
```python
# BAD: closures capture the variable, not the value
bad_closures = [(lambda: i) for i in range(5)]
print([f() for f in bad_closures])
# [4, 4, 4, 4, 4] ← all capture the final i
# FIXED: capture the value using default arg
fixed_closures = [(lambda i=i: i) for i in range(5)]
print([f() for f in fixed_closures])
# [0, 1, 2, 3, 4] ← correct
```
### Integer Interning
```python
# Python caches small integers (-5 to 256)
a, b = 256, 256
print(a is b) # True — same cached object
c, d = 257, 257
print(c is d) # True (in same code block, CPython optimization)
e = 200 + 57
f = 257
print(e is f) # False — different objects (computed at runtime)
# NEVER use `is` for value comparison. Always use `==`.
```
### Truthiness Rules
```python
# These are ALL falsy:
# None, False, 0, 0.0, 0j, '', [], {}, set(), range(0)
# DANGER: 0 is falsy but a valid value
def get_score(player):
score = player.get('score')
if score: # FAILS when score == 0
return score
return 'no score'
# FIXED: check for None explicitly
def get_score_fixed(player):
score = player.get('score')
if score is not None:
return score
return 'no score'
```
### Chained Assignment with Mutable Objects
```python
x = y = []
x.append(1)
print(y) # [1] — x and y point to THE SAME list
# FIXED: x = []; y = []
```
### Modifying Dict During Iteration
```python
# BAD: RuntimeError
# d = {'a': 1, 'b': 0, 'c': 3}
# for k in d:
# if d[k] == 0:
# del d[k] # RuntimeError!
# FIXED: iterate over a snapshot
d = {'a': 1, 'b': 0, 'c': 3, 'd': 0}
for k in list(d.keys()):
if d[k] == 0:
del d[k]
```
---
## 8. The GIL & Multiprocessing
The Global Interpreter Lock allows only one thread to execute Python bytecode at a time.
### CPU-Bound: Use Multiprocessing
```python
import time
import multiprocessing
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
def cpu_bound_task(n):
return sum(i * i for i in range(n))
# ProcessPoolExecutor — TRUE parallelism
with ProcessPoolExecutor(max_workers=4) as pool:
results = list(pool.map(cpu_bound_task, [10_000_000] * 4))
print(len(results)) # 4 results computed in parallel
# ThreadPoolExecutor — NOT parallel for CPU work (GIL)
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(cpu_bound_task, [10_000_000] * 4))
# Sequential due to GIL
```
### I/O-Bound: Threading IS Fine
```python
import urllib.request
from concurrent.futures import ThreadPoolExecutor
def fetch_url(url):
with urllib.request.urlopen(url, timeout=5) as resp:
return len(resp.read())
# GIL released during network calls → threads run concurrently
urls = ['http://example.com', 'http://example.org', 'http://example.net']
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(pool.map(fetch_url, urls))
```
### Python 3.13: Per-Interpreter GIL
```bash
# Python 3.13+ supports separate interpreters without GIL contention
python -X preparser -c "import _xxinterpchannels; print('Subinterpreters available')"
# Free-threading mode (3.13+)
python -X novalgrind -c "import sys; print(sys._is_gil_enabled())"
# Run with: python -X novalgrind script.py
```
---
## 9. Memory Management
### Reference Counting + Garbage Collection
```python
import sys
import gc
import tracemalloc
# Reference counting
x = [1, 2, 3]
print(sys.getrefcount(x)) # 2 (x + the argument to getrefcount)
y = x
print(sys.getrefcount(x)) # 3
del y
print(sys.getrefcount(x)) # 2
del x # refcount hits 0 → object destroyed immediately
# Reference cycles (GC's job)
a = []
b = []
a.append(b)
b.append(a)
# Neither's refcount reaches 0
gc.collect() # Finds and destroys cycles
# Memory profiling
tracemalloc.start()
big_list = [i for i in range(1_000_000)]
snapshot1 = tracemalloc.take_snapshot()
big_dict = {i: i*2 for i in range(500_000)}
snapshot2 = tracemalloc.take_snapshot()
stats = snapshot2.compare_to(snapshot1, 'lineno')
for stat in stats[:5]:
print(stat)
tracemalloc.stop()
```
---
## 10. Common Anti-Patterns
### Bare Except
```python
# BAD: catches KeyboardInterrupt, SystemExit, everything
def bad_handler():
try:
do_something()
except:
pass # silent failure
# GOOD: catch specific exceptions
def good_handler():
try:
do_something()
except (ValueError, TypeError) as e:
print(f"Expected error: {e}")
except Exception as e:
print(f"Unexpected: {e}")
raise
```
### Mutating List While Iterating
```python
# BAD
nums = [1, 2, 3, 4, 5, 6]
for n in nums:
if n % 2 == 0:
nums.remove(n) # skips elements!
print(nums) # [1, 3, 5, 6] — 4 was missed!
# GOOD: list comprehension
nums = [1, 2, 3, 4, 5, 6]
nums = [n for n in nums if n % 2 != 0]
print(nums) # [1, 3, 5]
```
### Using `==` for None Check
```python
# BAD: slow + invokes __eq__, can fail
if value == None: ...
# GOOD: identity check
if value is None: ...
```
### eval() with User Input
```python
# BAD: arbitrary code execution
# result = eval(user_input) # __import__('os').system('rm -rf /')
# GOOD: ast.literal_eval for data
import ast
safe = ast.literal_eval(user_input) # only literals, no code
```
### Using `time.sleep()` for Synchronization
```python
# BAD: polling
import time
def bad_wait(shared_state):
while not shared_state.ready:
time.sleep(0.1) # wastes CPU
# GOOD: threading.Event
import threading
def good_wait(event, shared_state):
event.wait() # blocks efficiently
return shared_state.result
```