Source code for topo.misc.memuse

"""
Functions for measuring memory usage, to allow usage to be optimized.

Examples::

  bin/python -c 'execfile("topo/misc/memuse.py") ; print topsize_mb()'

  ./topographica -c 'from topo.misc import memuse' -c 'print memuse.topsize_mb()'

  ./topographica -c 'from topo.misc import memuse, asizeof' -c 'print memuse.topsize_mb()'

  ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'print memuse.topsize_mb()'

  ./topographica -a -c 'from topo.misc import memuse, asizeof' examples/tiny.ty -c 'print memuse.allsizes_mb()'

  ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'memuse.memuse_batch("examples/tiny.ty",cortex_density=20)'

  ./topographica -a -c 'from topo.misc import memuse, asizeof' -c 'memuse.memuse_batch("examples/tiny.ty",times=[0,100],analysis_fn=memuse.plotting_and_saving_analysis_fn,cortex_density=20)'
"""

# If functions in this file need anything other than the very basic
# imports declared here at the top, they must do so using import
# statements *within* the function definition to avoid polluting
# memory.  Otherwise, the simplest functions in this file would end up
# measuring memory taken by unused and irrelevant imports like 'topo',
# which doesn't ever need to be loaded for the 'execfile' example
# shown above.

import subprocess


[docs]def cmd_to_string(cmd): """Run a system command as in os.system(), but capture the stdout and return it as a string.""" return subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE).communicate()[0]
[docs]def topsize(): """Return the RES size of this process as reported by the top(1) command.""" import os top_line=cmd_to_string("top -n 1 -b -p %d | grep '^[ ]*%d '" % (os.getpid(),os.getpid())) return top_line.split()[5]
[docs]def simsize(): """ Return the size of topo.sim reported by asizeof.asizeof(). This estimate does not currently include any numpy arrays, and may also be missing other important items. Python 2.6 supports getsizeof() and a __sizeof__ attribute that user code can implement, which should provide a more accurate estimate. """ import asizeof,topo return asizeof.asizeof(topo.sim) ############################################################################### # String-formatted versions of the above
[docs]def mb(bytes): """Format the given value in bytes as a string in megabytes""" return "%dMB" % (bytes/1024.0/1024.0)
[docs]def topsize_mb(): """String-formatted version of the RES size of this process reported by top(1).""" return "topsize:%s" % topsize()
[docs]def simsize_mb(): """String-formatted version of the value reported by asizeof(topo.sim).""" return "simsize:%s" % (mb(simsize()))
[docs]def wtsize_mb(): """String-formatted version of the memory taken by the weights, from print_sizes().""" from topo.command import n_bytes return "wtsize:%s" % (mb(n_bytes()))
[docs]def allsizes_mb(): """ Collates results from topsize, simsize, and wtsize. Formatted to suggest that the topsize is made up of code (not currently estimated), topo.sim (apart from weights), and weights. """ from topo.command import n_bytes return "%s =? code + %s + %s (%s tot)" % (topsize_mb(),simsize_mb(),wtsize_mb(),mb(simsize()+n_bytes())) ############################################################################### # Batch commands
[docs]def default_memuse_analysis_fn(prefix=""): """Basic memuse function for use with memuse_batch()""" import topo print "%st%s: %s" % (prefix,topo.sim.timestr(),allsizes_mb())
[docs]def plotting_and_saving_analysis_fn(prefix=""): """For use with memuse_batch() to test snapshot and plotting memory usage.""" import topo from topo.command import save_snapshot from topo.command.analysis import measure_sine_pref,save_plotgroup print "%sMemuse at time %s: %s" % (prefix,topo.sim.timestr(),allsizes_mb()) measure_sine_pref() print "%sAfter measure_sine_pref: %s" % (prefix,allsizes_mb()) save_plotgroup("Orientation Preference") print "%sAfter save_plotgroup: %s" % (prefix,allsizes_mb()) save_snapshot("/tmp/tmp.typ") print "%sAfter save_snapshot: %s" % (prefix,allsizes_mb())
[docs]def memuse_batch(script_file,times=[0],analysis_fn=default_memuse_analysis_fn,**params): """ Similar to run_batch, but analyzes the memory requirement of a simulation at different times. First, the specified script file will be run using the specified parameters. Then at each of the specified times, the given analysis_fn (which calls allsizes_mb() by default) is run. The output is labeled with the script file, time, and parameters so that results from different runs can be compared. """ import os,re,__main__,topo from topo.misc.commandline import global_params # Construct simulation name, etc. scriptbase= re.sub('.ty$','',os.path.basename(script_file)) prefix = "" #prefix += time.strftime("%Y%m%d%H%M") + "_" prefix += scriptbase simname = prefix # Construct parameter-value portion of filename; should do more filtering for a,val in params.iteritems(): # Special case to give reasonable filenames for lists valstr= ("_".join([str(i) for i in val]) if isinstance(val,list) else str(val)) prefix += "," + a + "=" + valstr # Set provided parameter values in main namespace global_params.set_in_context(**params) # Run script in main try: execfile(script_file,__main__.__dict__) #global_params.context global_params.check_for_unused_names() topo.sim.name=simname # Run each segment, doing the analysis and saving the script state each time for run_to in times: topo.sim.run(run_to - topo.sim.time()) analysis_fn(prefix=prefix+": ") except: import traceback,sys traceback.print_exc(file=sys.stdout) sys.stderr.write("Warning -- Error detected: execution halted.\n")