Skip to content

Closing the files?

Hi,

after loading a number of files (2097) I got this error: OSError: [Errno 24] Too many open files so, my question is: is there any way to close each file after it was extracted? I ofc tried .close(), but it's not working.

Or maybe I'm doing something wrong? This is the script I used:

import km3pipe as kp
from km3pipe.dataclasses import Table
import numpy as np
from glob import glob
import os.path
import km3io as ki
import h5py

DET = '00000044'
FOLDER = '/sps/km3net/users/kakiczi/CORSIKA_checks/SIBYLL-2.3/gamma_-1/' # Lyon
FOLDERS = FOLDER+'*/gSeaGen_processed/trigger_like_data_flim_1000/mc/manual/KM3NeT_'+DET+'/v5.40/reco'
FNAMES = glob(os.path.join(FOLDERS, "*aanet*root"))  # " + i + "
FNAMES.sort()  # ensures that we go in alphabetical/numeric order

VARS = [[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],]
for f in FNAMES:
    print("Opening ", f)
    r = ki.OfflineReader(f)
    if len(r.events)>1: # only non-empty files
        premask = [rstgs.size>1 for rstgs in r.events.tracks.id]
        mask = [[1, 2, 5, 3, 5, 4] == list(s[0]) for s in r.events.tracks.rec_stages[premask]]
        VARS[0].extend([sum(tots) for tots in r.events.hits.tot[premask][mask]])
        VARS[1].extend([sum(nt) for nt in r.events.hits.trig[premask][mask]])
        VARS[2].extend(r.events.n_hits[premask][mask])
        VARS[3].extend(r.events.overlays[premask][mask])
        VARS[4].extend(r.events.n_mc_tracks[premask][mask])
        VARS[5].extend(r.events.tracks.pos_x[premask][mask][:,0])
        VARS[6].extend(r.events.tracks.pos_y[premask][mask][:,0])
        VARS[7].extend(r.events.tracks.pos_z[premask][mask][:,0])
        VARS[8].extend(r.events.tracks.dir_x[premask][mask][:,0])
        VARS[9].extend(r.events.tracks.dir_y[premask][mask][:,0])
        VARS[10].extend(r.events.tracks.dir_z[premask][mask][:,0])
        VARS[11].extend(r.events.tracks.E[premask][mask][:,0]) # energy 
        VARS[12].extend(r.events.mc_tracks.type[premask][mask][:,0]) # primary 
        VARS[13].extend(r.events.mc_tracks.E[premask][mask][:,0]) # primary energy        
        VARS[14].extend(np.ones(len(r.events.mc_tracks.E[premask][mask][:,0]))*1000.*len(FNAMES)/5.)
        VARS[15].extend(r.events.w[premask][mask][:,2]) # weights

out = {}
out['Muons'] = Table(
    {
        'summed_ToT': VARS[0],
        'triggered_PMTs': VARS[1],
        'hit_PMTs': VARS[2],
        'overlays': VARS[3],
        'multiplicity': VARS[4],
        'pos_x': VARS[5],
        'pos_y':  VARS[6],
        'pos_z': VARS[7],
        'dir_x': VARS[8],
        'dir_y': VARS[9],
        'dir_z': VARS[10],
        'energy': VARS[11],
        'primary': VARS[12],
        'primary_energy': VARS[13],
        'num_showers': VARS[14],
        'w3': VARS[15],
    }, h5loc='muons')

hf = h5py.File(os.path.join(FOLDER,'gamma_-1_v5.8_'+DET+'_all_reco_'+str(len(FNAMES))+'_files.h5'), 'w') # _'+PRIM+'
hf.create_dataset('muons', data=out['Muons'])
hf.close()