Repository navigation
Expand file tree
/
Copy pathallocation_profiler.py
More file actions
124 lines (90 loc) · 3.91 KB
/
Copy pathallocation_profiler.py
File metadata and controls
124 lines (90 loc) · 3.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
from __future__ import division
import sys
import time
import re
import threading
from ..runtime import min_version, runtime_info, read_vm_size
from ..utils import timestamp
from ..metric import Metric
from ..metric import Breakdown
if min_version(3, 4):
import tracemalloc
class AllocationProfiler(object):
MAX_TRACEBACK_SIZE = 25 # number of frames
MAX_MEMORY_OVERHEAD = 10 * 1e6 # 10MB
MAX_PROFILED_ALLOCATIONS = 25
def __init__(self, agent):
self.agent = agent
self.ready = False
self.profile = None
self.profile_lock = threading.Lock()
self.overhead_monitor = None
self.start_ts = None
def setup(self):
if self.agent.get_option('allocation_profiler_disabled'):
return
if not runtime_info.OS_LINUX and not runtime_info.OS_DARWIN:
self.agent.log('CPU profiler is only supported on Linux and OS X.')
return
if not min_version(3, 4):
self.agent.log('Memory allocation profiling is available for Python 3.4 or higher')
return
self.ready = True
def reset(self):
self.profile = Breakdown('Allocation call graph', Breakdown.TYPE_CALLGRAPH)
def start_profiler(self):
self.agent.log('Activating memory allocation profiler.')
def start():
tracemalloc.start(self.MAX_TRACEBACK_SIZE)
self.agent.run_in_main_thread(start)
self.start_ts = time.time()
def monitor_overhead():
if tracemalloc.is_tracing() and tracemalloc.get_tracemalloc_memory() > self.MAX_MEMORY_OVERHEAD:
self.agent.log('Allocation profiler memory overhead limit exceeded: {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
self.stop_profiler()
self.overhead_monitor = self.agent.schedule(0.5, 0.5, monitor_overhead)
def stop_profiler(self):
self.agent.log('Deactivating memory allocation profiler.')
with self.profile_lock:
if self.overhead_monitor:
self.overhead_monitor.cancel()
self.overhead_monitor = None
if tracemalloc.is_tracing():
snapshot = tracemalloc.take_snapshot()
self.agent.log('Allocation profiler memory overhead {0} bytes'.format(tracemalloc.get_tracemalloc_memory()))
tracemalloc.stop()
self.process_snapshot(snapshot, time.time() - self.start_ts)
def build_profile(self, duration):
with self.profile_lock:
self.profile.normalize(duration)
self.profile.propagate()
self.profile.floor()
self.profile.filter(2, 1000, float("inf"))
return [{
'category': Metric.CATEGORY_MEMORY_PROFILE,
'name': Metric.NAME_UNCOLLECTED_ALLOCATIONS,
'unit': Metric.UNIT_BYTE,
'unit_interval': 1,
'profile': self.profile
}]
def destroy(self):
pass
def process_snapshot(self, snapshot, duration):
stats = snapshot.statistics('traceback')
for stat in stats[:self.MAX_PROFILED_ALLOCATIONS]:
if stat.traceback:
skip_stack = False
for frame in stat.traceback:
if self.agent.frame_cache.is_agent_frame(frame.filename):
skip_stack = True
break
if skip_stack:
continue
current_node = self.profile
for frame in reversed(stat.traceback):
if frame.filename == '<unknown>':
continue
frame_name = '{0}:{1}'.format(frame.filename, frame.lineno)
current_node = current_node.find_or_add_child(frame_name)
current_node.set_type(Breakdown.TYPE_CALLSITE)
current_node.increment(stat.size, stat.count)