Source code for netpyne.sim.load

Module for loading of data and simulations


    to_unicode = unicode
except NameError:
    to_unicode = str
except NameError:
    basestring = str

import sys
from collections import OrderedDict
from ..specs import Dict, ODict
from .. import specs
from . import utils
from . import setup

# ------------------------------------------------------------------------------
# Load data from file
# ------------------------------------------------------------------------------
def _loadFile(filename):
    from .. import sim
    import os

    def _byteify(data, ignore_dicts=False):
        # if this is a unicode string, return its string representation
        if isinstance(data, basestring):
            return data.encode('utf-8')
        # if this is a list of values, return list of byteified values
        if isinstance(data, list):
            return [_byteify(item, ignore_dicts=True) for item in data]
        # if this is a dictionary, return dictionary of byteified keys and values
        # but only if we haven't already byteified it
        if isinstance(data, dict) and not ignore_dicts:
            return OrderedDict(
                {_byteify(key, ignore_dicts=True): _byteify(value, ignore_dicts=True) for key, value in data.items()}
        # if it's anything else, return it in its original form
        return data

    if hasattr(sim, 'cfg') and sim.cfg.timing:
        sim.timing('start', 'loadFileTime')
    ext = os.path.basename(filename).split('.')[-1]

    # load pickle file
    if ext == 'pkl':
        import pickle

        print(('Loading file %s ... ' % (filename)))
        with open(filename, 'rb') as fileObj:
            if sys.version_info[0] == 2:
                data = pickle.load(fileObj)
                data = pickle.load(fileObj, encoding='latin1')

    # load dpk file
    elif ext == 'dpk':
        import gzip

        print(('Loading file %s ... ' % (filename)))
        # fn=sim.cfg.filename #.split('.')
        #, 'wb').write(pk.dumps(dataSave)) # write compressed string
        print('NOT IMPLEMENTED!')

    # load json file
    elif ext == 'json':
        import json

        print(('Loading file %s ... ' % (filename)))
        with open(filename, 'r') as fileObj:
            data = json.load(fileObj)  # works with py2 and py3
    # load mat file
    elif ext == 'mat':
        from import loadmat

        print(('Loading file %s ... ' % (filename)))
        dataraw = loadmat(filename, struct_as_record=False, squeeze_me=True)
        data = utils._mat2dict(dataraw)
        data = utils._restoreFromMat(data)

    # load HDF5 file (uses very inefficient hdf5storage module which supports dicts)
    elif ext in ['hdf5', 'h5']:
        import hdf5storage

        print(('Loading file %s ... ' % (filename)))
        keys ='__np_keys__', filename=filename)
        data = {k:, filename=filename) for k in keys}

    # load CSV file (currently only saves spikes)
    elif ext == 'csv':
        import csv

        print(('Loading file %s ... ' % (filename)))
        writer = csv.writer(open(sim.cfg.filename + '.csv', 'wb'))
        # for dic in dataSave['simData']:
        #    for values in dic:
        #        writer.writerow(values)
        print('NOT IMPLEMENTED!')

    # load Dat file(s)
    elif ext == 'dat':
        print(('Loading file %s ... ' % (filename)))
        print('NOT IMPLEMENTED!')
        # traces = sim.cfg.recordTraces
        # for ref in traces.keys():
        #     for cellid in sim.allSimData[ref].keys():
        #         dat_file_name = '%s_%s.dat'%(ref,cellid)
        #         dat_file = open(dat_file_name, 'w')
        #         trace = sim.allSimData[ref][cellid]
        #         print("Saving %i points of data on: %s:%s to %s"%(len(trace),ref,cellid,dat_file_name))
        #         for i in range(len(trace)):
        #             dat_file.write('%s\t%s\n'%((i*sim.cfg.dt/1000),trace[i]/1000))

        print(('Format not recognized for file %s' % (filename)))

    if hasattr(sim, 'rank') and sim.rank == 0 and hasattr(sim, 'cfg') and sim.cfg.timing:
        sim.timing('stop', 'loadFileTime')
        print(('  Done; file loading time = %0.2f s' % sim.timingData['loadFileTime']))

    return data

# ------------------------------------------------------------------------------
# Load simulation config from file
# ------------------------------------------------------------------------------
[docs] def loadSimCfg(filename, data=None, variable='simConfig', setLoaded=True): """ Function for/to <short description of `netpyne.sim.load.loadSimCfg`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* data : <``None``?> <Short description of data> **Default:** ``None`` **Options:** ``<option>`` <description of option> setLoaded : bool <Short description of setLoaded> **Default:** ``True`` **Options:** ``<option>`` <description of option> """ if not data: data = _loadFile(filename) print('Loading simConfig...') if variable in data: rawSimConfig = data[variable] if setLoaded: setup.setSimCfg(rawSimConfig) else: return specs.SimConfig(rawSimConfig) else: print(f' {variable} not found in file {filename}') pass
# ------------------------------------------------------------------------------ # Load netParams from cell # ------------------------------------------------------------------------------
[docs] def loadNetParams(filename, data=None, variable=None, setLoaded=True): """ Function for/to <short description of `netpyne.sim.load.loadNetParams`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* data : <``None``?> <Short description of data> **Default:** ``None`` **Options:** ``<option>`` <description of option> setLoaded : bool <Short description of setLoaded> **Default:** ``True`` **Options:** ``<option>`` <description of option> """ if not data: data = _loadFile(filename) print('Loading netParams...') if variable is not None and variable in data: rawNetParams = data[variable] elif 'net' in data and 'params' in data['net']: rawNetParams = data['net']['params'] else: print(('netParams not found in file %s' % (filename))) return if setLoaded: setup.setNetParams(rawNetParams) else: return specs.NetParams(rawNetParams)
# ------------------------------------------------------------------------------ # Load cells and pops from file and create NEURON objs # ------------------------------------------------------------------------------
[docs] def loadNet(filename, data=None, instantiate=True, compactConnFormat=False): """ Function for/to <short description of `netpyne.sim.load.loadNet`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* data : <``None``?> <Short description of data> **Default:** ``None`` **Options:** ``<option>`` <description of option> instantiate : bool <Short description of instantiate> **Default:** ``True`` **Options:** ``<option>`` <description of option> compactConnFormat : bool <Short description of compactConnFormat> **Default:** ``False`` **Options:** ``<option>`` <description of option> """ from .. import sim if not data: data = _loadFile(filename) if not hasattr(sim, 'net'): sim.initialize() if 'net' in data and 'cells' in data['net'] and 'pops' in data['net']: loadNow = True if hasattr(sim, 'rank'): if sim.rank != 0: loadNow = False if loadNow: sim.timing('start', 'loadNetTime') print('Loading net...') if compactConnFormat: compactToLongConnFormat(data['net']['cells'], compactConnFormat) # convert loaded data to long format = data['net']['pops'] loadedPops = data['net']['pops'] if loadedPops is ODict: = loadedPops else: # if populations order is not preserved (for example if loaded from JSON), need to sort them again = ODict() loadedPops = list(loadedPops.items()) def sort(popKeyValue): # the assumption while sorting is that populations order corresponds to cell gids in this population cellGids = popKeyValue[1].get('cellGids', []) if len(cellGids) > 0: return cellGids[0] else: return -1 loadedPops.sort(key=sort) for pop in loadedPops:[pop[0]] = pop[1] rawCells = data['net']['cells'] = [c if isinstance(c, Dict) else Dict(c) for c in rawCells] if instantiate: try: # calculate cells to instantiate in this node if hasattr(sim, 'rank'): if isinstance(instantiate, list): cellsNode = [ data['net']['cells'][i] for i in range(int(sim.rank), len(data['net']['cells']), sim.nhosts) if i in instantiate ] else: cellsNode = [ data['net']['cells'][i] for i in range(int(sim.rank), len(data['net']['cells']), sim.nhosts) ] else: if isinstance(instantiate, list): cellsNode = [ data['net']['cells'][i] for i in range(0, len(data['net']['cells']), 1) if i in instantiate ] else: cellsNode = [data['net']['cells'][i] for i in range(0, len(data['net']['cells']), 1)] except: print('Unable to instantiate network...') try: if sim.cfg.createPyStruct: for popLoadLabel, popLoad in data['net']['pops'].items(): pop = sim.Pop(popLoadLabel, popLoad['tags']) pop.cellGids = popLoad['cellGids'][popLoadLabel] = pop for cellLoad in cellsNode: # create new cell object and add attributes, but don't create sections or associate gid yet secs = cellLoad.get('secs', None) gid = cellLoad['gid'] tags = cellLoad['tags'] if secs: cell = sim.CompartCell(gid, tags, create=False, associateGid=False) cell.secs = Dict(secs) cell.create(createNEURONObj=False) # avoid creating NEURON Objs now; just need pystruct else: tags['params'] = cellLoad['params'] cell = sim.PointCell(gid, tags, create=False, associateGid=False) try: cell.conns = [Dict(conn) for conn in cellLoad['conns']] except: if sim.cfg.verbose: print(' Unable to load cell conns') try: cell.stims = [Dict(stim) for stim in cellLoad['stims']] except: if sim.cfg.verbose: print(' Unable to load cell stims') print((' Created %d cells' % (len( print((' Created %d connections' % (sum([len(c.conns) for c in])))) print((' Created %d stims' % (sum([len(c.stims) for c in])))) except Exception as e: print(f'Unable to create Python structure: {e}') try: # only create NEURON objs, if there is Python struc (fix so minimal Python struct is created) if sim.cfg.createNEURONObj: if sim.cfg.verbose: print(" Adding NEURON objects...") # create NEURON sections, mechs, syns, etc; and associate gid for cell in if cell.secs: prop = {'secs': cell.secs} # TODO: `cell.secs` should probably be deepcopied to avoid potential "ghost" errors later in `CompartCell.createNEURONObj()` # but currently deepcopying of ODict fails cell.createNEURONObj(prop) # use same syntax as when creating based on high-level specs cell.associateGid() # can only associate once the hSection obj has been created # create all NEURON Netcons, NetStims, etc sim.pc.barrier() for cell in try: cell.addStimsNEURONObj() # add stims first so can then create conns between netstims cell.addConnsNEURONObj() except: if sim.cfg.verbose: 'Unable to load instantiate cell conns or stims' print((' Added NEURON objects to %d cells' % (len( except Exception as e: print(f'Unable to create NEURON objects: {e}') if loadNow and sim.cfg.timing: # if sim.rank == 0 and sim.cfg.timing: sim.timing('stop', 'loadNetTime') print((' Done; re-instantiate net time = %0.2f s' % sim.timingData['loadNetTime'])) else: print((' netCells and/or netPops not found in file %s' % (filename)))
# ------------------------------------------------------------------------------ # Load simData from file # ------------------------------------------------------------------------------
[docs] def loadSimData(filename, data=None): """ Function to load simulation data from a file Parameters ---------- filename : str The path and name of the file where the data is stored. **Default:** *required* data : dict A dictionary containing a `simData` key. **Default:** ``None`` """ from .. import sim if not data: data = _loadFile(filename) print('Loading simData...') if 'simData' in data: sim.allSimData = data['simData'] else: print((' simData not found in file %s' % (filename))) if 'net' in data: if 'recXElectrode' in data['net']: from import RecXElectrode xElectrode = data['net']['recXElectrode'] if False == isinstance(xElectrode, RecXElectrode): xElectrode = RecXElectrode.fromJSON(xElectrode) = xElectrode
# ------------------------------------------------------------------------------ # Load all data in file # ------------------------------------------------------------------------------
[docs] def loadAll(filename, data=None, instantiate=True, createNEURONObj=True): """ Function for/to <short description of `netpyne.sim.load.loadAll`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* data : <``None``?> <Short description of data> **Default:** ``None`` **Options:** ``<option>`` <description of option> instantiate : bool <Short description of instantiate> **Default:** ``True`` **Options:** ``<option>`` <description of option> createNEURONObj : bool <Short description of createNEURONObj> **Default:** ``True`` **Options:** ``<option>`` <description of option> """ from .. import sim if not data: data = _loadFile(filename) loadSimCfg(filename, data=data) sim.cfg.createNEURONObj = createNEURONObj # set based on argument loadNetParams(filename, data=data) if hasattr(sim.cfg, 'compactConnFormat'): connFormat = sim.cfg.compactConnFormat else: print('Error: no connFormat provided in simConfig') sys.exit() loadNet(filename, data=data, instantiate=instantiate, compactConnFormat=connFormat) loadSimData(filename, data=data)
[docs] def loadFromIndexFile(index): return loadModel(index, loadMechs=False)
[docs] def loadModel(path, loadMechs=True, forceCompileMechs=False, returnLoadMechStatus=False): import __main__ import json from .. import sim import os originalDir = os.getcwd() absPath = os.path.abspath(path) if os.path.isdir(absPath): # if no index file specified explicitly, use default index = os.path.join(absPath, 'index.npjson') dir = absPath else: index = absPath dir = os.path.dirname(absPath) print(f'Loading model from {index} ... ') with open(index, 'r') as fileObj: indexData = json.load(fileObj) if loadMechs: modFolder = indexData.get('mod_folder') if modFolder: modFolderPath = os.path.join(dir, modFolder) loadedMods, skippedMods = processModFiles(modFolderPath, forceCompile=forceCompileMechs) else: loadedMods, skippedMods = [], [] # for consistency os.chdir(dir) configFile = indexData.get('simConfig') if not configFile: raise Exception("Model index is missing 'simConfig' attribute") print(f'\n Loading simConfig: {configFile} ... ') if configFile[-3:] == '.py': cfgModule = sim.loadPythonModule(configFile) configVar = indexData.get('simConfig_variable', 'cfg') cfg = getattr(cfgModule, configVar) else: configVar = indexData.get('simConfig_variable', 'simConfig') cfg = sim.loadSimCfg(configFile, variable=configVar, setLoaded=False) netParamsFile = indexData.get('netParams') if not netParamsFile: raise Exception("Model index is missing 'netParams' attribute") print(f'\n Loading netParams: {netParamsFile} ... ') if netParamsFile[-3:] == '.py': __main__.cfg = cfg # this is often required by netParams netParamsModule = sim.loadPythonModule(netParamsFile) paramsVar = indexData.get('netParams_variable', 'netParams') netParams = getattr(netParamsModule, paramsVar) else: paramsVar = indexData.get('netParams_variable', None) netParams = sim.loadNetParams(netParamsFile, variable=paramsVar, setLoaded=False) os.chdir(originalDir) res = cfg, netParams if returnLoadMechStatus: res += loadedMods, skippedMods return res
[docs] def processModFiles(modFolderPath, forceCompile=False): import os, glob import neuron from neuron import h if not os.path.exists(modFolderPath): print(f"Warning: specified 'mod_folder' path {modFolderPath} doesn't exist") return None, None originalDir = os.getcwd() os.chdir(modFolderPath) # all mod file names in this dir (abs path) allMods = glob.glob(f"{os.getcwd()}/*.mod") # mapping from file name to mechanism name filesAndMechs = {file: __extractModelName(file) for file in allMods} # first, try to identify which mechs are already loaded modsToSkip = [] modsToSkipStatus = [] for fileName in allMods: mechName = filesAndMechs[fileName] if not hasattr(h, mechName): continue # this mech is already loaded, so it will be skipped modsToSkip.append(fileName) # now try to figure out if it's differs from loaded version thisFileHash = utils.fileDigest(fileName) alreadyLoadedModHash = neuron._netpyne_mech_hashes.get(mechName) if not alreadyLoadedModHash: modsToSkipStatus.append('unknown') elif thisFileHash == alreadyLoadedModHash: modsToSkipStatus.append('unmodified') else: modsToSkipStatus.append('modified') modsToLoad = set(allMods) - set(modsToSkip) if not modsToLoad: print('All mods are already loaded. Skipping..') elif len(modsToLoad) == len(allMods): # no mods to skip, compile/load all at once _compileAndLoadMechanisms(forceCompile=forceCompile, filesToMechsMapping=filesAndMechs) else: # proceed only with files not loaded before _compileAndLoadMechanisms(files=modsToLoad, forceCompile=forceCompile, filesToMechsMapping=filesAndMechs) loadedMods = [(fileName, filesAndMechs[fileName]) for fileName in modsToLoad] skippedMods = [(fileName, filesAndMechs[fileName], status) for fileName, status in zip(modsToSkip, modsToSkipStatus)] os.chdir(originalDir) return [loadedMods, skippedMods]
def __extractModelName(modFile): import re lines = [] with open(modFile, "r") as f: lines = f.readlines() for line in lines: line = line.split(':')[0] # drop code comments for modType in ['suffix', 'point_process', 'artificial_cell']: if (modType + ' ') in line.lower(): partWithName = re.split(modType, line, flags=re.IGNORECASE)[1] partWithName = partWithName.replace("}", " ") # for the case like "NEURON {SUFFIX NAME}" name = partWithName.strip().split(" ")[0] return name return None def _compileAndLoadMechanisms(path=None, files=None, forceCompile=False, filesToMechsMapping={}): import neuron, os, subprocess, shutil, platform, time origPath = os.getcwd() if path: os.chdir(path) if files: files = [os.path.abspath(file) for file in files] tmpDir = None if hasattr(neuron, 'nrn_dll_loaded') \ and (os.getcwd() in neuron.nrn_dll_loaded): # this path is already stored in NEURON's internal cache, which makes it not possible to load anything else from it. # need to re-compile into another folder to load from there compile = True tmpDir = f'tmp_{os.getpid()}_{time.time()}' os.mkdir(tmpDir) os.chdir(tmpDir) elif not os.path.exists(platform.machine()): # there is no folder with compiled files (e.g. 'x86_64') at given path compile = True elif files: # if about to load a specific subset of files, compilation is necessary compile = True else: compile = forceCompile if compile: if files: # compile specific files cmd = ['nrnivmodl'] + files elif tmpDir: # being in temp dir, compile all files in parent folder cmd = ['nrnivmodl', '..'] else: # compile all files in current folder cmd = ['nrnivmodl'] neuron.load_mechanisms(os.path.abspath('.')) # load compiled dlls # save md5 hashes of just loaded mod files if not files: import glob files = glob.glob(f"{os.getcwd()}/*.mod") for file in files: mech = filesToMechsMapping.get(file, __extractModelName(file)) neuron._netpyne_mech_hashes[mech] = utils.fileDigest(file) if tmpDir: os.chdir('..') os.system(f'rm -rf {tmpDir}') os.chdir(origPath) # ------------------------------------------------------------------------------ # Convert compact (list-based) to long (dict-based) conn format # ------------------------------------------------------------------------------
[docs] def compactToLongConnFormat(cells, connFormat): """ Function for/to <short description of `netpyne.sim.load.compactToLongConnFormat`> Parameters ---------- cells : <type> <Short description of cells> **Default:** *required* connFormat : <type> <Short description of connFormat> **Default:** *required* """ formatIndices = {key: connFormat.index(key) for key in connFormat} try: for cell in cells: for iconn, conn in enumerate(cell['conns']): cell['conns'][iconn] = {key: conn[index] for key, index in formatIndices.items()} return cells except: print("Error converting conns from compact to long format") return cells
# ------------------------------------------------------------------------------ # load HDF5 (conns for now) # ------------------------------------------------------------------------------
[docs] def loadHDF5(filename): """ Function for/to <short description of `netpyne.sim.load.loadHDF5`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* """ from .. import sim import h5py if sim.rank == 0: timing('start', 'loadTimeHDF5') connsh5 = h5py.File(filename, 'r') conns = [list(x) for x in connsh5['conns']] connsFormat = list(connsh5['connsFormat']) if sim.rank == 0: timing('stop', 'loadTimeHDF5') return conns, connsFormat
# ------------------------------------------------------------------------------ # Load cell tags and conns using ijson (faster!) # ------------------------------------------------------------------------------
[docs] def ijsonLoad( filename, tagsGidRange=None, connsGidRange=None, loadTags=True, loadConns=True, tagFormat=None, connFormat=None, saveTags=None, saveConns=None, ): """ Function for/to <short description of `netpyne.sim.load.ijsonLoad`> Parameters ---------- filename : <type> <Short description of filename> **Default:** *required* tagsGidRange : <``None``?> <Short description of tagsGidRange> **Default:** ``None`` **Options:** ``<option>`` <description of option> connsGidRange : <``None``?> <Short description of connsGidRange> **Default:** ``None`` **Options:** ``<option>`` <description of option> loadTags : bool <Short description of loadTags> **Default:** ``True`` **Options:** ``<option>`` <description of option> loadConns : bool <Short description of loadConns> **Default:** ``True`` **Options:** ``<option>`` <description of option> tagFormat : <``None``?> <Short description of tagFormat> **Default:** ``None`` **Options:** ``<option>`` <description of option> connFormat : <``None``?> <Short description of connFormat> **Default:** ``None`` **Options:** ``<option>`` <description of option> saveTags : <``None``?> <Short description of saveTags> **Default:** ``None`` **Options:** ``<option>`` <description of option> saveConns : <``None``?> <Short description of saveConns> **Default:** ``None`` **Options:** ``<option>`` <description of option> """ # requires: 1) pip install ijson, 2) brew install yajl from .. import sim import ijson.backends.yajl2_cffi as ijson import json from time import time tags, conns = {}, {} if connFormat: conns['format'] = connFormat if tagFormat: tags['format'] = tagFormat with open(filename, 'rb') as fd: start = time() print('Loading data ...') objs = ijson.items(fd, 'net.cells.item') if loadTags and loadConns: print('Storing tags and conns ...') for cell in objs: if tagsGidRange == None or cell['gid'] in tagsGidRange: print('Cell gid: %d' % (cell['gid'])) if tagFormat: tags[int(cell['gid'])] = [cell['tags'][param] for param in tagFormat] else: tags[int(cell['gid'])] = cell['tags'] if connsGidRange == None or cell['gid'] in connsGidRange: if connFormat: conns[int(cell['gid'])] = [[conn[param] for param in connFormat] for conn in cell['conns']] else: conns[int(cell['gid'])] = cell['conns'] elif loadTags: print('Storing tags ...') if tagFormat: tags.update( { int(cell['gid']): [cell['tags'][param] for param in tagFormat] for cell in objs if tagsGidRange == None or cell['gid'] in tagsGidRange } ) else: tags.update( { int(cell['gid']): cell['tags'] for cell in objs if tagsGidRange == None or cell['gid'] in tagsGidRange } ) elif loadConns: print('Storing conns...') if connFormat: conns.update( { int(cell['gid']): [[conn[param] for param in connFormat] for conn in cell['conns']] for cell in objs if connsGidRange == None or cell['gid'] in connsGidRange } ) else: conns.update( { int(cell['gid']): cell['conns'] for cell in objs if connsGidRange == None or cell['gid'] in connsGidRange } ) print('time ellapsed (s): ', time() - start) tags = utils.decimalToFloat(tags) conns = utils.decimalToFloat(conns) if saveTags and tags: outFilename = saveTags if isinstance(saveTags, basestring) else 'filename'[:-4] + '_tags.json' print('Saving tags to %s ...' % (outFilename)) sim.saveJSON(outFilename, {'tags': tags}) if saveConns and conns: outFilename = saveConns if isinstance(saveConns, basestring) else 'filename'[:-4] + '_conns.json' print('Saving conns to %s ...' % (outFilename)) sim.saveJSON(outFilename, {'conns': conns}) return tags, conns