Merging PR_218 openai_rev package with new streamlit chat app
This commit is contained in:
noptuno
238
venv/lib/python3.9/site-packages/pympler/process.py
Normal file
238
venv/lib/python3.9/site-packages/pympler/process.py
Normal file
@@ -0,0 +1,238 @@
|
||||
"""
|
||||
This module queries process memory allocation metrics from the operating
|
||||
system. It provides a platform independent layer to get the amount of virtual
|
||||
and physical memory allocated to the Python process.
|
||||
|
||||
Different mechanisms are implemented: Either the process stat file is read
|
||||
(Linux), the `ps` command is executed (BSD/OSX/Solaris) or the resource module
|
||||
is queried (Unix fallback). On Windows try to use the win32 module if
|
||||
available. If all fails, return 0 for each attribute.
|
||||
|
||||
Windows without the win32 module is not supported.
|
||||
|
||||
>>> from pympler.process import ProcessMemoryInfo
|
||||
>>> pmi = ProcessMemoryInfo()
|
||||
>>> print ("Virtual size [Byte]: " + str(pmi.vsz)) # doctest: +ELLIPSIS
|
||||
Virtual size [Byte]: ...
|
||||
"""
|
||||
|
||||
from typing import Iterable, List, Tuple
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from mmap import PAGESIZE # type: ignore
|
||||
from os import getpid
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
from pympler.util.stringutils import pp
|
||||
|
||||
|
||||
class _ProcessMemoryInfo(object):
|
||||
"""Stores information about various process-level memory metrics. The
|
||||
virtual size is stored in attribute `vsz`, the physical memory allocated to
|
||||
the process in `rss`, and the number of (major) pagefaults in `pagefaults`.
|
||||
On Linux, `data_segment`, `code_segment`, `shared_segment` and
|
||||
`stack_segment` contain the number of Bytes allocated for the respective
|
||||
segments. This is an abstract base class which needs to be overridden by
|
||||
operating system specific implementations. This is done when importing the
|
||||
module.
|
||||
"""
|
||||
|
||||
pagesize = PAGESIZE
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.pid = getpid()
|
||||
self.rss = 0
|
||||
self.vsz = 0
|
||||
self.pagefaults = 0
|
||||
self.os_specific = [] # type: List[Tuple[str, str]]
|
||||
|
||||
self.data_segment = 0
|
||||
self.code_segment = 0
|
||||
self.shared_segment = 0
|
||||
self.stack_segment = 0
|
||||
|
||||
self.available = self.update()
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return "<%s vsz=%d rss=%d>" % (self.__class__.__name__,
|
||||
self.vsz, self.rss)
|
||||
|
||||
def update(self) -> bool:
|
||||
"""
|
||||
Refresh the information using platform instruments. Returns true if
|
||||
this operation yields useful values on the current platform.
|
||||
"""
|
||||
return False # pragma: no cover
|
||||
|
||||
def __sub__(self, other: '_ProcessMemoryInfo') -> Iterable[Tuple[str, int]]:
|
||||
diff = [('Resident set size (delta)', self.rss - other.rss),
|
||||
('Virtual size (delta)', self.vsz - other.vsz),
|
||||
]
|
||||
return diff
|
||||
|
||||
|
||||
ProcessMemoryInfo = _ProcessMemoryInfo # type: type
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""
|
||||
Convenience function to check if the current platform is supported by this
|
||||
module.
|
||||
"""
|
||||
return ProcessMemoryInfo().update()
|
||||
|
||||
|
||||
class _ProcessMemoryInfoPS(_ProcessMemoryInfo):
|
||||
|
||||
def update(self) -> bool:
|
||||
"""
|
||||
Get virtual and resident size of current process via 'ps'.
|
||||
This should work for MacOS X, Solaris, Linux. Returns true if it was
|
||||
successful.
|
||||
"""
|
||||
try:
|
||||
p = Popen(['/bin/ps', '-p%s' % self.pid, '-o', 'rss,vsz'],
|
||||
stdout=PIPE, stderr=PIPE)
|
||||
except OSError: # pragma: no cover
|
||||
pass
|
||||
else:
|
||||
s = p.communicate()[0].split()
|
||||
if p.returncode == 0 and len(s) >= 2: # pragma: no branch
|
||||
self.vsz = int(s[-1]) * 1024
|
||||
self.rss = int(s[-2]) * 1024
|
||||
return True
|
||||
return False # pragma: no cover
|
||||
|
||||
|
||||
class _ProcessMemoryInfoProc(_ProcessMemoryInfo):
|
||||
|
||||
key_map = {
|
||||
'VmPeak': 'Peak virtual memory size',
|
||||
'VmSize': 'Virtual memory size',
|
||||
'VmLck': 'Locked memory size',
|
||||
'VmHWM': 'Peak resident set size',
|
||||
'VmRSS': 'Resident set size',
|
||||
'VmStk': 'Size of stack segment',
|
||||
'VmData': 'Size of data segment',
|
||||
'VmExe': 'Size of code segment',
|
||||
'VmLib': 'Shared library code size',
|
||||
'VmPTE': 'Page table entries size',
|
||||
}
|
||||
|
||||
def update(self) -> bool:
|
||||
"""
|
||||
Get virtual size of current process by reading the process' stat file.
|
||||
This should work for Linux.
|
||||
"""
|
||||
try:
|
||||
stat = open('/proc/self/stat')
|
||||
status = open('/proc/self/status')
|
||||
except IOError: # pragma: no cover
|
||||
return False
|
||||
else:
|
||||
stats = stat.read().split()
|
||||
self.vsz = int(stats[22])
|
||||
self.rss = int(stats[23]) * self.pagesize
|
||||
self.pagefaults = int(stats[11])
|
||||
|
||||
for entry in status.readlines():
|
||||
try:
|
||||
key, value = entry.split(':', 1)
|
||||
except ValueError:
|
||||
continue
|
||||
value = value.strip()
|
||||
|
||||
def size_in_bytes(x: str) -> int:
|
||||
return int(x.split()[0]) * 1024
|
||||
|
||||
if key == 'VmData':
|
||||
self.data_segment = size_in_bytes(value)
|
||||
elif key == 'VmExe':
|
||||
self.code_segment = size_in_bytes(value)
|
||||
elif key == 'VmLib':
|
||||
self.shared_segment = size_in_bytes(value)
|
||||
elif key == 'VmStk':
|
||||
self.stack_segment = size_in_bytes(value)
|
||||
key = self.key_map.get(key, '')
|
||||
if key:
|
||||
self.os_specific.append((key, pp(size_in_bytes(value))))
|
||||
|
||||
stat.close()
|
||||
status.close()
|
||||
return True
|
||||
|
||||
|
||||
try:
|
||||
from resource import getrusage, RUSAGE_SELF
|
||||
|
||||
class _ProcessMemoryInfoResource(_ProcessMemoryInfo):
|
||||
def update(self) -> bool:
|
||||
"""
|
||||
Get memory metrics of current process through `getrusage`. Only
|
||||
available on Unix, on Linux most of the fields are not set,
|
||||
and on BSD units are used that are not very helpful, see:
|
||||
|
||||
http://www.perlmonks.org/?node_id=626693
|
||||
|
||||
Furthermore, getrusage only provides accumulated statistics (e.g.
|
||||
max rss vs current rss).
|
||||
"""
|
||||
usage = getrusage(RUSAGE_SELF)
|
||||
self.rss = usage.ru_maxrss * 1024
|
||||
self.data_segment = usage.ru_idrss * 1024 # TODO: ticks?
|
||||
self.shared_segment = usage.ru_ixrss * 1024 # TODO: ticks?
|
||||
self.stack_segment = usage.ru_isrss * 1024 # TODO: ticks?
|
||||
self.vsz = (self.data_segment + self.shared_segment +
|
||||
self.stack_segment)
|
||||
|
||||
self.pagefaults = usage.ru_majflt
|
||||
return self.rss != 0
|
||||
|
||||
if _ProcessMemoryInfoProc().update(): # pragma: no branch
|
||||
ProcessMemoryInfo = _ProcessMemoryInfoProc
|
||||
elif _ProcessMemoryInfoPS().update(): # pragma: no cover
|
||||
ProcessMemoryInfo = _ProcessMemoryInfoPS
|
||||
elif _ProcessMemoryInfoResource().update(): # pragma: no cover
|
||||
ProcessMemoryInfo = _ProcessMemoryInfoResource
|
||||
|
||||
except ImportError:
|
||||
try:
|
||||
# Requires pywin32
|
||||
from win32process import GetProcessMemoryInfo
|
||||
from win32api import GetCurrentProcess, GlobalMemoryStatusEx
|
||||
except ImportError:
|
||||
logging.warn("Please install pywin32 when using pympler on Windows.")
|
||||
else:
|
||||
class _ProcessMemoryInfoWin32(_ProcessMemoryInfo):
|
||||
def update(self) -> bool:
|
||||
process_handle = GetCurrentProcess()
|
||||
meminfo = GetProcessMemoryInfo(process_handle)
|
||||
memstatus = GlobalMemoryStatusEx()
|
||||
self.vsz = (memstatus['TotalVirtual'] -
|
||||
memstatus['AvailVirtual'])
|
||||
self.rss = meminfo['WorkingSetSize']
|
||||
self.pagefaults = meminfo['PageFaultCount']
|
||||
return True
|
||||
|
||||
ProcessMemoryInfo = _ProcessMemoryInfoWin32
|
||||
|
||||
|
||||
class ThreadInfo(object):
|
||||
"""Collect information about an active thread."""
|
||||
|
||||
def __init__(self, thread: threading.Thread):
|
||||
self.ident = thread.ident
|
||||
self.name = thread.name
|
||||
self.daemon = thread.daemon
|
||||
|
||||
|
||||
def get_current_threads() -> Iterable[ThreadInfo]:
|
||||
"""Get a list of `ThreadInfo` objects."""
|
||||
return [ThreadInfo(thread) for thread in threading.enumerate()]
|
||||
|
||||
|
||||
def get_current_thread_id() -> int:
|
||||
"""Get the ID of the current thread."""
|
||||
return threading.get_ident()
|
||||
Reference in New Issue
Block a user