Gfyuki

When I call Keras' fit_generator(), passing in a custom generator class I created, I see "Epoch 1/1" in the spew and that's all. It hangs right there, and the generator is never called. I know this because I put print statements in getitem that are never printed.

This data generator is modified version of Shervine Amidi's tutorial example of a generator that inherits from the Keras Sequence object:

class DataGenerator(keras.utils.Sequence):

    'Generates data for Keras'

    def __init__(self, batchID, 

                 batch_size = 32, 

                 dim = (32,32,32)):



        self.dim = dim

        self.batch_size = batch_size

        self.datafile_IDs = 

        self.labelfile_IDs = 

        self.batchID = batchID    

        self.DataDir = "data/"

        self.BatchDir = ""



        DataDir = self.DataDir

        BatchDir = DataDir + batchID + "/"

        self.BatchDir = BatchDir



        path = BatchDir + "datafilenames_" + batchID + ".pkl"                                    

        fd = open(path, "rb")

        self.datafile_IDs = pkl.load(fd)

        fd.close()



        path = BatchDir + "labelfilenames_" + batchID + ".pkl"        

        fd = open(path, "rb")

        self.labelfile_IDs = pkl.load(fd)

        fd.close()



    def __len__(self):

        'Denotes the number of batches per epoch'

        return int(

            np.floor(len(self.datafile_IDs) / self.batch_size))



    def __getitem__(self, index):



        'Generate one batch of data'



        datafn = self.datafile_IDs[index]        

        labelfn = self.labelfile_IDs[index]  



        print("In getitem: index = %d, datafn = %s, labelfn = %s" % (

               index, datafn, labelfn))



        batch_size = self.batch_size



        # Initialize data arrays for this batch     

        X = np.empty((self.batch_size, *self.dim))

        y = np.empty((self.batch_size), dtype=int)         



        BatchDir = self.BatchDir



        # Store data

        datafn = BatchDir + datafn

        X = np.load(datafn)



        # Store label

        labelfn = BatchDir + labelfn

        y = np.load(labelfn)



        return X, y        



genbatchfiles(df_short, batchID = "short", batch_size = 20)

params = {'batchID': "short", 'batch_size': 20, 'dim': (100, 10088)} 

dg = DataGenerator(**params) 

time_series_length, input_dim, output_dim = 100, 10088, 1

model = Sequential()

model.add(LSTM(20, input_shape=(time_series_length, input_dim))) 



# The max output value is > 1 so relu is used as final activation.

model.add(Dense(output_dim, activation='relu'))



model.compile(loss='mean_squared_error',

              optimizer='sgd',

              metrics=['accuracy'])

model.fit_generator(generator = dg,

                    steps_per_epoch = 5,

                    use_multiprocessing = True, 

                    workers = 6,

                    verbose = 2)