Closing the files?
Hi,
after loading a number of files (2097) I got this error:
OSError: [Errno 24] Too many open files
so, my question is: is there any way to close each file after it was extracted? I ofc tried .close()
, but it's not working.
Or maybe I'm doing something wrong? This is the script I used:
import km3pipe as kp
from km3pipe.dataclasses import Table
import numpy as np
from glob import glob
import os.path
import km3io as ki
import h5py
DET = '00000044'
FOLDER = '/sps/km3net/users/kakiczi/CORSIKA_checks/SIBYLL-2.3/gamma_-1/' # Lyon
FOLDERS = FOLDER+'*/gSeaGen_processed/trigger_like_data_flim_1000/mc/manual/KM3NeT_'+DET+'/v5.40/reco'
FNAMES = glob(os.path.join(FOLDERS, "*aanet*root")) # " + i + "
FNAMES.sort() # ensures that we go in alphabetical/numeric order
VARS = [[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],]
for f in FNAMES:
print("Opening ", f)
r = ki.OfflineReader(f)
if len(r.events)>1: # only non-empty files
premask = [rstgs.size>1 for rstgs in r.events.tracks.id]
mask = [[1, 2, 5, 3, 5, 4] == list(s[0]) for s in r.events.tracks.rec_stages[premask]]
VARS[0].extend([sum(tots) for tots in r.events.hits.tot[premask][mask]])
VARS[1].extend([sum(nt) for nt in r.events.hits.trig[premask][mask]])
VARS[2].extend(r.events.n_hits[premask][mask])
VARS[3].extend(r.events.overlays[premask][mask])
VARS[4].extend(r.events.n_mc_tracks[premask][mask])
VARS[5].extend(r.events.tracks.pos_x[premask][mask][:,0])
VARS[6].extend(r.events.tracks.pos_y[premask][mask][:,0])
VARS[7].extend(r.events.tracks.pos_z[premask][mask][:,0])
VARS[8].extend(r.events.tracks.dir_x[premask][mask][:,0])
VARS[9].extend(r.events.tracks.dir_y[premask][mask][:,0])
VARS[10].extend(r.events.tracks.dir_z[premask][mask][:,0])
VARS[11].extend(r.events.tracks.E[premask][mask][:,0]) # energy
VARS[12].extend(r.events.mc_tracks.type[premask][mask][:,0]) # primary
VARS[13].extend(r.events.mc_tracks.E[premask][mask][:,0]) # primary energy
VARS[14].extend(np.ones(len(r.events.mc_tracks.E[premask][mask][:,0]))*1000.*len(FNAMES)/5.)
VARS[15].extend(r.events.w[premask][mask][:,2]) # weights
out = {}
out['Muons'] = Table(
{
'summed_ToT': VARS[0],
'triggered_PMTs': VARS[1],
'hit_PMTs': VARS[2],
'overlays': VARS[3],
'multiplicity': VARS[4],
'pos_x': VARS[5],
'pos_y': VARS[6],
'pos_z': VARS[7],
'dir_x': VARS[8],
'dir_y': VARS[9],
'dir_z': VARS[10],
'energy': VARS[11],
'primary': VARS[12],
'primary_energy': VARS[13],
'num_showers': VARS[14],
'w3': VARS[15],
}, h5loc='muons')
hf = h5py.File(os.path.join(FOLDER,'gamma_-1_v5.8_'+DET+'_all_reco_'+str(len(FNAMES))+'_files.h5'), 'w') # _'+PRIM+'
hf.create_dataset('muons', data=out['Muons'])
hf.close()