# Python Lesser-Known Standard Library Modules
> Research compiled 2025 — modules most Python developers don't know about or underutilize.
---
## 1. `itertools` — The Iterator Toolkit
Beyond the basics, itertools has powerful combinatorial tools.
```python
from itertools import chain, groupby, permutations, combinations, islice, tee, cycle, repeat, starmap
# chain: flatten multiple iterables
flat = list(chain([1,2], [3,4], [5,6])) # [1,2,3,4,5,6]
# groupby: group consecutive items (must be sorted first!)
data = sorted([('a',1), ('a',2), ('b',1)], key=lambda x: x[0])
groups = {k: list(v) for k, v in groupby(data, key=lambda x: x[0])}
# {'a': [('a',1),('a',2)], 'b': [('b',1)]}
# islice: efficient slicing of iterators (no loading into memory)
first_10 = list(islice(huge_iterator, 10))
# tee: create N independent iterators from one
a, b = tee(single_iterator)
# starmap: like map but unpacks arguments
import operator
pairs = [(3,4), (5,12), (8,15)]
results = list(starmap(operator.add, pairs)) # [7, 17, 23]
```
**Use case:** Processing large datasets without loading everything into memory.
---
## 2. `contextlib` — Beyond Basic Context Managers
Most know `@contextmanager`, but `ExitStack` and `suppress` are powerful.
```python
from contextlib import contextmanager, ExitStack, suppress, chdir, redirect_stdout
import io
# ExitStack: dynamically manage multiple context managers
with ExitStack() as stack:
files = [stack.open(f'/tmp/file_{i}.txt', 'w') for i in range(10)]
# All files guaranteed closed, even if some fail to open
for f in files:
f.write('data')
# suppress: clean way to ignore specific exceptions
with suppress(FileNotFoundError):
os.remove('nonexistent.txt') # No error raised
# redirect_stdout: capture print output
buf = io.StringIO()
with redirect_stdout(buf):
print("hello")
print(buf.getvalue()) # "hello\n"
# chdir: temporary directory change
import pathlib
with chdir(pathlib.Path('/tmp')):
os.listdir('.') # lists /tmp contents
```
**Use case:** Resource management in complex scenarios, testing, cleanup.
---
## 3. `dataclasses` — Beyond `@dataclass`
Hidden features: `kw_only`, `slots=True`, `frozen`, field defaults, and custom init.
```python
from dataclasses import dataclass, field, asdict, astuple
from typing import ClassVar
@dataclass(frozen=True, slots=True, kw_only=True)
class ImmutablePoint:
"""Memory-efficient, immutable, keyword-only."""
x: float
y: float
_cache: dict = field(default_factory=dict, repr=False, compare=False)
# slots=True: no __dict__ per instance (like manual __slots__)
# frozen=True: immutable (like namedtuple but with mutability control)
# kw_only=True: all fields must be passed as kwargs
p = ImmutablePoint(x=1.0, y=2.0)
# Advanced: custom __post_init__
@dataclass
class ValidatedData:
value: int
validated: bool = False
def __post_init__(self):
if self.value < 0:
raise ValueError("Must be non-negative")
self.validated = True
# field() with factory
@dataclass
class User:
name: str
tags: list = field(default_factory=list)
metadata: dict = field(default_factory=dict)
```
**Use case:** Clean data models with memory optimization.
---
## 4. `functools` — Function Toolkit
Beyond `@lru_cache`: `singledispatch`, `partialmethod`, `cached_property`, `wraps`.
```python
from functools import singledispatch, partialmethod, cached_property, wraps, reduce
import time
# @singledispatch: type-based dispatch (like polymorphism without inheritance)
@singledispatch
def serialize(obj):
raise NotImplementedError(f"Cannot serialize {type(obj)}")
@serialize.register
def _(obj: str):
return f'" {obj}"'
@serialize.register
def _(obj: int):
return str(obj)
@serialize.register(list)
def _(obj: list):
return '[' + ', '.join(serialize(item) for item in obj) + ']'
serialize("hello") # '"hello"'
serialize([1, "two", 3]) # '[1, "two", 3]'
# @cached_property: compute once, cache for life of instance
class Circle:
def __init__(self, radius):
self.radius = radius
@cached_property
def area(self):
print("Computing area...") # Only prints once
return 3.14159 * self.radius ** 2
# partialmethod: bind arguments to class methods
class Button:
def __init__(self, label, color="blue", size="medium"):
self.label = label
self.color = color
self.size = size
@partialmethod
def click(self, action="default"):
print(f"Clicked {self.label} with {action}")
```
**Use case:** Polymorphism without inheritance, memoization, partial application.
---
## 5. `weakref` — References That Don't Prevent GC
Critical for caching, observers, and parent-child relationships.
```python
import weakref
# Basic weak reference
class ExpensiveObject:
def __init__(self, name):
self.name = name
def __del__(self):
print(f" [{self.name}] destroyed")
obj = ExpensiveObject("cached_item")
weak = weakref.ref(obj)
print(weak()) # <ExpensiveObject object>
del obj # Object destroyed!
print(weak()) # None
# WeakValueDictionary: cache that doesn't prevent GC
class Cache:
def __init__(self):
self._cache = weakref.WeakValueDictionary()
def get_or_create(self, name):
if name not in self._cache:
self._cache[name] = ExpensiveObject(name)
return self._cache[name]
# WeakSet: track objects without owning them
class EventBus:
def __init__(self):
self._listeners = weakref.WeakSet()
def subscribe(self, listener):
self._listeners.add(listener)
def notify(self, event):
for listener in list(self._listeners): # list() to avoid modification during iteration
listener.handle(event)
# Callback on object death
class Tracker:
def __init__(self):
self.destroyed = []
def on_destroy(self, obj, name):
def callback(ref):
self.destroyed.append(name)
weakref.finalize(obj, callback)
tracker = Tracker()
tracker.on_destroy(obj, "my_object")
# When obj is GC'd, tracker.destroyed gets "my_object"
```
**Use case:** Caching, observer patterns, avoiding reference cycles.
---
## 6. `inspect` — Runtime Introspection
Examine live objects: get source, signatures, frames, module contents.
```python
import inspect
def my_func(a: int, b: str = "hello", *args, **kwargs) -> str:
"""Example function."""
return b
# Get function signature
sig = inspect.signature(my_func)
print(sig) # (a: int, b: str = 'hello', *args, **kwargs) -> str
# Get source code
source = inspect.getsource(my_func)
# Examine a class
class MyClass:
def method(self): pass
@classmethod
def class_method(cls): pass
members = inspect.getmembers(MyClass, predicate=inspect.ismethod)
print([name for name, _ in members]) # ['class_method']
# Walk the call stack
def deep_function():
frame = inspect.currentframe()
caller = frame.f_back
print(f"Called from: {caller.f_code.co_name}")
# Clean up reference cycles
del frame, caller
# Get all methods of a class (including inherited)
for name, method in inspect.getmembers(MyClass, predicate=inspect.isfunction):
print(f"{name}: {inspect.signature(method)}")
```
**Use case:** Debugging tools, documentation generators, frameworks.
---
## 7. `dis` — Bytecode Disassembler
See what Python actually executes under the hood.
```python
import dis
def add(a, b):
return a + b
dis.dis(add)
# Output:
# 2 0 LOAD_FAST 0 (a)
# 2 LOAD_FAST 1 (b)
# 4 BINARY_ADD
# 6 RETURN_VALUE
# Compare: local vs global lookup speed
global_var = 10
def use_global():
return global_var + 1 # LOAD_GLOBAL (slower)
def use_local(x):
return x + 1 # LOAD_FAST (faster)
dis.dis(use_global)
dis.dis(use_local)
# Understand list comprehension optimization
dis.dis(compile('[x for x in range(10)]', '<string>', 'eval'))
# Uses BUILD_LIST and FOR_ITER — much faster than .append() loop
```
**Use case:** Performance analysis, understanding Python internals, education.
---
## 8. `enum` — Advanced Enumeration
Beyond `Enum`: `Flag`, `IntEnum`, `auto()`, `unique`, and methods.
```python
from enum import Enum, IntEnum, Flag, auto, IntFlag
# Basic Enum with auto
class Status(Enum):
PENDING = auto()
ACTIVE = auto()
DONE = auto()
# IntEnum: enum + int (can compare with integers)
class Priority(IntEnum):
LOW = 1
MEDIUM = 2
HIGH = 3
print(Priority.HIGH > Priority.LOW) # True
print(Priority.HIGH == 3) # True
# Flag: bitmask enums
class Permission(Flag):
READ = auto() # 1
WRITE = auto() # 2
EXECUTE = auto() # 4
perms = Permission.READ | Permission.WRITE
print(Permission.READ in perms) # True
print(~perms) # Permission.EXECUTE
# IntFlag: same as Flag but also an int
class Mode(IntFlag):
R = 4
W = 2
X = 1
mode = Mode.R | Mode.W
print(int(mode)) # 6
print(oct(mode)) # '0o6'
# Enum with methods
class Color(Enum):
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)
@property
def hex(self):
return f"#{self.value[0]:02X}{self.value[1]:02X}{self.value[2]:02X}"
print(Color.RED.hex) # #FF0000
```
**Use case:** Type-safe constants, state machines, permissions.
---
## 9. `mmap` — Memory-Mapped Files
Treat files as arrays — extremely fast for large files.
```python
import mmap
import os
# Write a file
with open('/tmp/large_file.bin', 'wb') as f:
f.write(b'Hello, World! This is a large file. ' * 1000)
# Memory-map it
with open('/tmp/large_file.bin', 'r+b') as f:
mm = mmap.mmap(f.fileno(), 0) # 0 = entire file
# Search (like str methods)
pos = mm.find(b'World')
print(pos) # Position of 'World'
# Slice
print(mm[:13]) # b'Hello, World!'
# Modify in-place (changes the file!)
mm[0:5] = b'Bye'
mm.flush() # Write back
mm.close()
# Read-only mapping
with open('/tmp/large_file.bin', 'rb') as f:
mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
print(mm[:8]) # b'Bye, Worl'
```
**Use case:** Processing large files, database engines, search engines.
---
## 10. `sched` — Event Scheduler
Built-in scheduling without external libraries.
```python
import sched
import time
scheduler = sched.scheduler(time.time, time.sleep)
def task(name, duration):
print(f"[{time.strftime('%H:%M:%S')}] Running {name}")
# Schedule events
scheduler.enter(2, 1, task, ("short task", 1))
scheduler.enter(5, 1, task, ("medium task", 2))
scheduler.enter(3, 1, task, ("in between", 2))
# Events are sorted by time
print("Running scheduler...")
scheduler.run()
# Output order: short task (2s), in between (3s), medium task (5s)
# Cancel an event
event = scheduler.enter(10, 1, task, ("will be cancelled", 1))
scheduler.cancel(event)
# Priority queue behavior
scheduler.enter(1, 3, task, ("low priority", 3))
scheduler.enter(1, 1, task, ("high priority", 1)) # Same time, lower priority = runs first
```
**Use case:** Simple task scheduling, delayed execution, event-driven programs.
---
## 11. `difflib` — Text Diff and Similarity
Built-in diff, patch, and similarity tools.
```python
from difflib import unified_diff, ndiff, SequenceMatcher, context_diff
text1 = """The quick brown fox
jumps over the lazy dog
in the morning sun"""
text2 = """The quick red fox
jumps over the sleepy dog
in the evening shade"""
# Unified diff
diff = list(unified_diff(text1.splitlines(keepends=True),
text2.splitlines(keepends=True),
fromfile='a.txt', tofile='b.txt'))
print(''.join(diff))
# Similarity ratio
ratio = SequenceMatcher(None, text1, text2).ratio()
print(f"Similarity: {ratio:.2%}") # ~70%
# Get matching blocks
blocks = SequenceMatcher(None, text1, text2).get_matching_blocks()
for i, j, n in blocks:
print(f"Match: text1[{i}:{i+n}] == text2[{j}:{j+n}] = {text1[i:i+n]!r}")
# Word-level diff
diff_words = list(ndiff('the quick brown fox'.split(), 'the quick red fox'.split()))
print(diff_words) # ['the ', 'quick ', '- brown ', '+ red ', 'fox']
```
**Use case:** Version control, change detection, plagiarism detection, testing.
---
## 12. `struct` — Binary Data Packing
Pack/unpack binary data (network protocols, file formats).
```python
import struct
# Pack integers and floats into bytes
data = struct.pack('>IHf', 42, 100, 3.14)
# '>' = big-endian, 'I' = unsigned int (4 bytes), 'H' = unsigned short (2 bytes), 'f' = float (4 bytes)
print(data.hex()) # '0000002a006440490fdb'
# Unpack
i, h, f = struct.unpack('>IHf', data)
print(i, h, f) # 42 100 3.140000104904175
# Calculate size
size = struct.calcsize('>IHf') # 10 bytes
# Network protocol example: parse a simple packet
def parse_packet(data):
# 4-byte length, 2-byte type, rest is payload
length, ptype = struct.unpack_from('>IH', data)
payload = data[6:6+length-6]
return ptype, payload
packet = struct.pack('>IH', 16, 1) + b'Hello, World!'
print(parse_packet(packet)) # (1, b'Hello, World!')
# Endianness matters!
little = struct.pack('<I', 0x12345678) # 78 56 34 12
big = struct.pack('>I', 0x12345678) # 12 34 56 78
```
**Use case:** Network protocols, file parsing, serialization, cryptography.
---
## 13. `shlex` — Shell Lexing
Parse shell-like syntax safely.
```python
import shlex
# Split a command respecting quotes
command = 'echo "hello world" --verbose -n 42'
tokens = shlex.split(command)
# ['echo', 'hello world', '--verbose', '-n', '42']
# Join back (safe quoting)
result = shlex.join(tokens)
# 'echo "hello world" --verbose -n 42'
# Custom lexer
class CustomLexer(shlex.shlex):
def __init__(self, *args, **kwargs):
super().__init__(*args, posix=True, **kwargs)
self.whitespace_split = True
self.commenters = '#'
self.quotes = '"\''
lexer = CustomLexer('name="John Doe" age=30 # comment')
print(list(lexer)) # ['name=John Doe', 'age=30']
```
**Use case:** Config file parsing, CLI argument preprocessing.
---
## 14. `tracemalloc` — Memory Tracking
Built-in memory profiler.
```python
import tracemalloc
tracemalloc.start()
# Take a snapshot
snapshot1 = tracemalloc.take_snapshot()
# Allocate some memory
big_list = [i for i in range(1_000_000)]
big_dict = {i: i*2 for i in range(500_000)}
snapshot2 = tracemalloc.take_snapshot()
# Compare snapshots
stats = snapshot2.compare_to(snapshot1, 'lineno')
print("Top 5 memory allocations:")
for stat in stats[:5]:
print(stat)
# Filter and sort
stats = snapshot2.statistics('filename')
for stat in stats[:3]:
print(f"{stat[0]}: {stat[1]} bytes")
tracemalloc.stop()
```
**Use case:** Memory leak detection, performance optimization.
---
## 15. `atexit` — Cleanup on Exit
Register functions to run on program exit.
```python
import atexit
class DatabaseConnection:
def __init__(self):
print("Connected to database")
atexit.register(self.close)
def close(self):
print("Closing database connection")
def query(self, sql):
print(f"Executing: {sql}")
db = DatabaseConnection()
db.query("SELECT * FROM users")
# On exit: "Closing database connection" is called automatically
# Multiple handlers (called in LIFO order)
@atexit.register
def cleanup_temp():
print("Cleaning up temp files...")
@atexit.register
def save_state():
print("Saving application state...")
# With arguments
@atexit.register
def goodbye(name="world"):
print(f"Goodbye, {name}!")
```
**Use case:** Cleanup, logging, saving state, resource release.
---
## 16. `lzma` / `zlib` — Compression
Built-in compression without external dependencies.
```python
import lzma
import zlib
# LZMA (better compression, slower)
data = b"Hello, World!" * 1000
compressed = lzma.compress(data)
print(f"Original: {len(data)} bytes, Compressed: {len(compressed)} bytes")
decompressed = lzma.decompress(compressed)
assert decompressed == data
# Streaming compression
import io
with lzma.LZMAFile('/tmp/data.lzma', 'wb') as f:
f.write(data)
with lzma.LZMAFile('/tmp/data.lzma', 'rb') as f:
result = f.read()
# ZLIB (faster, less compression)
compressed = zlib.compress(data, level=9) # level 1-9
decompressed = zlib.decompress(compressed)
# Checksums
crc = zlib.crc32(data)
adler = zlib.adler32(data)
```
**Use case:** Data storage, network transfer, backup.
---
## Summary Table
| Module | Use Case | Complexity |
|--------|----------|------------|
| `itertools` | Iterator combinatorics | Medium |
| `contextlib` | Resource management | Medium |
| `weakref` | Non-owning references | Medium |
| `inspect` | Runtime introspection | Medium |
| `dis` | Bytecode analysis | Advanced |
| `enum` | Type-safe constants | Easy |
| `functools` | Function utilities | Medium |
| `mmap` | Large file I/O | Advanced |
| `sched` | Event scheduling | Easy |
| `difflib` | Text comparison | Easy |
| `struct` | Binary data | Advanced |
| `shlex` | Shell parsing | Easy |
| `tracemalloc` | Memory profiling | Easy |
| `atexit` | Cleanup handlers | Easy |
| `lzma` | Compression | Easy |