Python 性能优化完全指南：从性能分析到内存优化与 JIT 加速实战

1
import cProfile
2
import pstats
3

4
def slow_function():
5
    total = 0
6
    for i in range(1_000_000):
7
        total += i * i
8
    return total
9

10
def fast_function():
11
    return sum(i * i for i in range(1_000_000))
12

13
def main():
14
    slow_function()
15
    fast_function()
16

17
# 方法1：代码内调用
18
cProfile.run('main()', 'profile.prof')
19

20
# 查看结果
21
stats = pstats.Stats('profile.prof')
22
stats.sort_stats('cumulative')
23
stats.print_stats(10)
24

25
# 方法2：命令行
26
# python -m cProfile -o profile.prof script.py
27
# python -m pstats profile.prof

1
   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
2
        1    0.234    0.234    0.234    0.234 script.py:5(slow_function)
3
        1    0.156    0.156    0.156    0.156 script.py:9(fast_function)

1
from memory_profiler import profile
2

3
@profile
4
def memory_heavy():
5
    # 逐行内存变化
6
    a = [1] * 1_000_000        # +8 MB
7
    b = [2] * 1_000_000        # +8 MB
8
    c = a + b                  # +16 MB
9
    del a                      # -8 MB
10
    return c
11

12
memory_heavy()

1
import time
2

3
data = list(range(1_000_000))
4

5
# ❌ 慢：手动循环
6
start = time.time()
7
total = 0
8
for x in data:
9
    total += x
10
print(f"手动循环: {time.time() - start:.4f}s")
11

12
# ✅ 快：内置 sum()
13
start = time.time()
14
total = sum(data)
15
print(f"sum(): {time.time() - start:.4f}s")
16

17
# ✅ 更快：数学公式
18
start = time.time()
19
n = len(data)
20
total = n * (n - 1) // 2
21
print(f"数学公式: {time.time() - start:.4f}s")
22

23
# 结果：
24
# 手动循环: 0.0234s
25
# sum(): 0.0045s  (5倍加速)
26
# 数学公式: 0.0000s (1000倍加速)

1
# ❌ 慢
2
result = []
3
for i in range(1_000_000):
4
    result.append(i * 2)
5

6
# ✅ 快（快约 2 倍）
7
result = [i * 2 for i in range(1_000_000)]
8

9
# ✅ 更快（生成器表达式，省内存）
10
result = (i * 2 for i in range(1_000_000))
11

12
# ✅ 最快（map）
13
result = list(map(lambda x: x * 2, range(1_000_000)))

1
# ❌ 慢：字符串拼接
2
result = ""
3
for i in range(10_000):
4
    result += str(i)
5

6
# ✅ 快：join
7
result = "".join(str(i) for i in range(10_000))
8

9
# ✅ 更快：列表 + join
10
parts = [str(i) for i in range(10_000)]
11
result = "".join(parts)

1
data_list = list(range(100_000))
2
data_set = set(range(100_000))
3

4
# ❌ 慢：列表查找 O(n)
5
if 99_999 in data_list:
6
    pass
7

8
# ✅ 快：集合查找 O(1)
9
if 99_999 in data_set:
10
    pass
11

12
# 集合查找比列表快 10000+ 倍（大数据集）

1
import math
2

3
# ❌ 慢：全局查找
4
def compute_global(n):
5
    result = 0
6
    for i in range(n):
7
        result += math.sqrt(i)  # 每次都查找 math.sqrt
8
    return result
9

10
# ✅ 快：局部变量
11
def compute_local(n):
12
    sqrt = math.sqrt  # 局部缓存
13
    result = 0
14
    for i in range(n):
15
        result += sqrt(i)
16
    return result
17

18
# 局部变量查找比全局快约 15-30%

1
class DataProcessor:
2
    def __init__(self):
3
        self.data = list(range(100_000))
4

5
    # ❌ 慢：重复属性访问
6
    def process_slow(self):
7
        total = 0
8
        for i in range(len(self.data)):
9
            total += self.data[i]  # 每次都查找 self.data
10
        return total
11

12
    # ✅ 快：缓存属性
13
    def process_fast(self):
14
        data = self.data  # 一次查找
15
        total = 0
16
        for i in range(len(data)):
17
            total += data[i]
18
        return total
19

20
    # ✅ 最快：直接迭代
21
    def process_best(self):
22
        return sum(self.data)

1
import sys
2

3
# ❌ 列表：一次性加载所有数据
4
def get_numbers_list(n):
5
    return [i for i in range(n)]
6

7
# ✅ 生成器：惰性求值
8
def get_numbers_gen(n):
9
    for i in range(n):
10
        yield i
11

12
# 内存对比
13
list_numbers = get_numbers_list(1_000_000)
14
gen_numbers = get_numbers_gen(1_000_000)
15

16
print(f"列表内存: {sys.getsizeof(list_numbers)} bytes")  # ~8 MB
17
print(f"生成器内存: {sys.getsizeof(gen_numbers)} bytes")  # ~120 bytes
18

19
# 生成器节省了 99.99% 的内存

1
import sys
2

3
# ❌ 普通类：使用 __dict__ 存储属性
4
class PointDict:
5
    def __init__(self, x, y, z):
6
        self.x = x
7
        self.y = y
8
        self.z = z
9

10
# ✅ slots 类：固定属性，省内存
11
class PointSlots:
12
    __slots__ = ('x', 'y', 'z')
13

14
    def __init__(self, x, y, z):
15
        self.x = x
16
        self.y = y
17
        self.z = z
18

19
# 内存对比
20
p1 = PointDict(1, 2, 3)
21
p2 = PointSlots(1, 2, 3)
22

23
print(f"普通类: {sys.getsizeof(p1) + sys.getsizeof(p1.__dict__)} bytes")
24
print(f"slots类: {sys.getsizeof(p2)} bytes")
25

26
# 创建 100 万个对象
27
points_dict = [PointDict(1, 2, 3) for _ in range(1_000_000)]
28
points_slots = [PointSlots(1, 2, 3) for _ in range(1_000_000)]
29

30
# slots 类节省约 40-50% 内存

1
import array
2
import sys
3

4
# ❌ 列表存储整数
5
list_data = [0] * 1_000_000
6
print(f"列表: {sys.getsizeof(list_data)} bytes")  # ~8 MB
7

8
# ✅ array 存储整数
9
array_data = array.array('i', [0] * 1_000_000)
10
print(f"array: {sys.getsizeof(array_data)} bytes")  # ~4 MB
11

12
# array 比列表节省约 50% 内存

1
# ❌ 切片创建副本
2
data = b'x' * 1_000_000
3
chunk = data[100:200]  # 创建新的 bytes 对象
4

5
# ✅ memoryview 零拷贝
6
mv = memoryview(data)
7
chunk = mv[100:200]  # 不创建副本
8

9
# 适合处理大二进制数据（图像、视频）

1
import sys
2

3
# 大量重复字符串
4
strings = [f"user_{i % 100}" for i in range(1_000_000)]
5

6
# ❌ 不使用 intern
7
total_before = sum(sys.getsizeof(s) for s in strings)
8

9
# ✅ 使用 intern
10
strings_interned = [sys.intern(s) for s in strings]
11
total_after = sum(sys.getsizeof(s) for s in strings_interned)
12

13
# intern 后相同字符串共享内存

1
import numpy as np
2
import time
3

4
# 创建大数据
5
data = list(range(1_000_000))
6
np_data = np.array(data)
7

8
# ❌ Python 循环
9
start = time.time()
10
result = [x * 2 + 1 for x in data]
11
print(f"Python 列表推导: {time.time() - start:.4f}s")
12

13
# ✅ NumPy 向量化
14
start = time.time()
15
result = np_data * 2 + 1
16
print(f"NumPy 向量化: {time.time() - start:.4f}s")
17

18
# NumPy 快 10-50 倍

1
import numpy as np
2

3
# 条件筛选
4
data = np.random.randn(1_000_000)
5

6
# ❌ 循环筛选
7
result = [x for x in data if x > 0]
8

9
# ✅ 布尔索引
10
result = data[data > 0]
11

12
# 数学运算
13
# ❌ 循环
14
result = [math.sqrt(abs(x)) for x in data]
15

16
# ✅ 向量化
17
result = np.sqrt(np.abs(data))
18

19
# 聚合运算
20
# ❌ 循环
21
total = sum(data)
22
mean = sum(data) / len(data)
23

24
# ✅ NumPy
25
total = np.sum(data)
26
mean = np.mean(data)
27
std = np.std(data)

1
import numpy as np
2

3
# 矩阵运算
4
matrix = np.random.rand(1000, 1000)
5
vector = np.random.rand(1000)
6

7
# ❌ 循环逐行计算
8
result = np.zeros_like(matrix)
9
for i in range(1000):
10
    result[i] = matrix[i] + vector
11

12
# ✅ 广播自动扩展
13
result = matrix + vector  # 自动广播
14

15
# 广播比循环快 100+ 倍

1
from collections import deque, defaultdict, Counter
2

3
# ❌ 列表头部插入 O(n)
4
data = list(range(100_000))
5
data.insert(0, -1)  # 慢
6

7
# ✅ deque 头部插入 O(1)
8
data = deque(range(100_000))
9
data.appendleft(-1)  # 快
10

11
# ❌ 字典分组
12
groups = {}
13
for item in items:
14
    key = item.category
15
    if key not in groups:
16
        groups[key] = []
17
    groups[key].append(item)
18

19
# ✅ defaultdict
20
groups = defaultdict(list)
21
for item in items:
22
    groups[item.category].append(item)
23

24
# ❌ 手动计数
25
counts = {}
26
for word in words:
27
    if word in counts:
28
        counts[word] += 1
29
    else:
30
        counts[word] = 1
31

32
# ✅ Counter
33
counts = Counter(words)

1
from functools import lru_cache
2
import time
3

4
# ❌ 重复计算
5
def fib_slow(n):
6
    if n <= 1:
7
        return n
8
    return fib_slow(n - 1) + fib_slow(n - 2)
9

10
# ✅ 缓存
11
@lru_cache(maxsize=None)
12
def fib_fast(n):
13
    if n <= 1:
14
        return n
15
    return fib_fast(n - 1) + fib_fast(n - 2)
16

17
# 测试
18
start = time.time()
19
print(fib_slow(35))  # 约 3s
20
print(f"无缓存: {time.time() - start:.2f}s")
21

22
start = time.time()
23
print(fib_fast(35))  # < 0.001s
24
print(f"有缓存: {time.time() - start:.2f}s")

1
# ❌ 遍历整个列表
2
def has_negative(numbers):
3
    found = False
4
    for n in numbers:
5
        if n < 0:
6
            found = True
7
    return found
8

9
# ✅ 找到就退出
10
def has_negative(numbers):
11
    for n in numbers:
12
        if n < 0:
13
            return True
14
    return False
15

16
# ✅ 更简洁
17
def has_negative(numbers):
18
    return any(n < 0 for n in numbers)  # 生成器，找到即停

1
性能问题？
2
│
3
├── 第1步：性能分析
4
│   ├── cProfile 找热点函数
5
│   ├── line_profiler 逐行分析
6
│   └── memory_profiler 内存分析
7
│
8
├── 第2步：低 hanging fruit（快速见效）
9
│   ├── 使用内置函数（sum, max, min）
10
│   ├── 列表推导式替代循环
11
│   ├── 集合替代列表查找
12
│   └── 局部变量缓存
13
│
14
├── 第3步：算法优化
15
│   ├── 选择正确数据结构
16
│   ├── 减少时间复杂度
17
│   ├── 使用缓存
18
│   └── 提前退出
19
│
20
├── 第4步：内存优化
21
│   ├── 生成器替代列表
22
│   ├── __slots__
23
│   ├── array / memoryview
24
│   └── 及时释放大对象
25
│
26
├── 第5步：高级优化
27
│   ├── NumPy 向量化
28
│   ├── Cython 编译
29
│   ├── PyPy JIT
30
│   └── multiprocessing 并行
31
│
32
└── 第6步：架构优化
33
    ├── 异步 IO（asyncio）
34
    ├── 批处理
35
    └── 分布式计算

1
✅ 先分析，后优化（不要盲目优化）
2
✅ 使用 cProfile 找出真正的瓶颈
3
✅ 优先使用内置函数和标准库
4
✅ 选择正确的数据结构
5
✅ 数值计算用 NumPy
6
✅ 重复计算用缓存
7
✅ 大数据用生成器
8
✅ CPU 密集用 Cython 或 multiprocessing
9
✅ 纯 Python 用 PyPy
10
✅ IO 密集用 asyncio
11
✅ 优化后重新测试验证效果