PanopticNet train four head error

[fanweiya]

I use deepcell model train four head, prompt loss error. this is my code and error.

from deepcell.model_zoo.panopticnet import PanopticNet
classes = {
    'inner_distance': 1,  # inner distance
    'outer_distance': 1,  # outer distance
    'fgbg': 2,  # foreground/background separation
    'pixelwise': 2,  # pixelwise
}

model = PanopticNet(
    backbone='resnet50',
    input_shape=X_train.shape[1:],
    norm_method='std',
    num_semantic_classes=classes)

# Create a dictionary of losses for each semantic head
from tensorflow.keras.losses import MSE
#from tensorflow.keras.losses import CategoricalCrossentropy
from deepcell import losses 
import tensorflow as tf
from tensorflow.keras import backend as K
def weighted_categorical_crossentropy(y_true, y_pred,
                                      n_classes=3, axis=None,
                                      from_logits=False):
    """Categorical crossentropy between an output tensor and a target tensor.
    Automatically computes the class weights from the target image and uses
    them to weight the cross entropy

    Args:
        y_true: A tensor of the same shape as ``y_pred``.
        y_pred: A tensor resulting from a softmax
            (unless ``from_logits`` is ``True``, in which
            case ``y_pred`` is expected to be the logits).
        from_logits: Boolean, whether ``y_pred`` is the
            result of a softmax, or is a tensor of logits.

    Returns:
        tensor: Output tensor.
    """
    if from_logits:
        raise Exception('weighted_categorical_crossentropy cannot take logits')
    y_pred = tf.convert_to_tensor(y_pred)
    y_true = K.cast(y_true, y_pred.dtype)
    n_classes = K.cast(n_classes, y_pred.dtype)
    if axis is None:
        axis = 1 if K.image_data_format() == 'channels_first' else K.ndim(y_pred) - 1
    reduce_axis = [x for x in list(range(K.ndim(y_pred))) if x != axis]
    # scale preds so that the class probas of each sample sum to 1
    y_pred = y_pred / K.sum(y_pred, axis=axis, keepdims=True)
    # manual computation of crossentropy
    _epsilon = tf.convert_to_tensor(K.epsilon(), y_pred.dtype.base_dtype)
    y_pred = tf.clip_by_value(y_pred, _epsilon, 1. - _epsilon)
    total_sum = K.sum(y_true)
    class_sum = K.sum(y_true, axis=reduce_axis, keepdims=True)
    class_weights = 1.0 / n_classes * tf.divide(total_sum, class_sum + 1.)
    print(y_true.shape,y_pred.shape)
    return - K.sum((y_true * K.log(y_pred) * class_weights), axis=axis)

def semantic_loss(n_classes):
    def _semantic_loss(y_true, y_pred):
        if n_classes > 1:
            return  weighted_categorical_crossentropy(y_true, y_pred, n_classes=n_classes) * 0.01 
            #weighted_categorical_crossentropy
        return MSE(y_true, y_pred)
    return _semantic_loss


loss = {}

# Give losses for all of the semantic heads
for layer in model.layers:
    if layer.name.startswith('semantic_'):
        n_classes = layer.output_shape[-1]
        loss[layer.name] = semantic_loss(n_classes)
        
model.compile(loss=loss, optimizer=optimizer)

InvalidArgumentError                      Traceback (most recent call last)
Cell In[134], line 21
     11 print('Training on', num_gpus, 'GPUs.')
     13 train_callbacks = get_callbacks(
     14     model_path,
     15     lr_sched=lr_sched,
   (...)
     18     monitor='val_loss',
     19     verbose=1)
---> 21 loss_history = model.fit(
     22     train_data,
     23     steps_per_epoch=train_data.y.shape[0] // batch_size,
     24     epochs=n_epoch,
     25     validation_data=val_data,
     26     validation_steps=val_data.y.shape[0] // batch_size,
     27     callbacks=train_callbacks)

File /data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/utils/traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     65 except Exception as e:  # pylint: disable=broad-except
     66   filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67   raise e.with_traceback(filtered_tb) from None
     68 finally:
     69   del filtered_tb

File /data/anaconda3/envs/deepcell/lib/python3.8/site-packages/tensorflow/python/eager/execute.py:54, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     52 try:
     53   ctx.ensure_initialized()
---> 54   tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
     55                                       inputs, attrs, num_outputs)
     56 except core._NotOkStatusException as e:
     57   if name is not None:

InvalidArgumentError: Graph execution error:

Detected at node 'semantic_loss_3/mul_1' defined at (most recent call last):
    File "/data/anaconda3/envs/deepcell/lib/python3.8/runpy.py", line 192, in _run_module_as_main
      return _run_code(code, main_globals, None,
    File "/data/anaconda3/envs/deepcell/lib/python3.8/runpy.py", line 85, in _run_code
      exec(code, run_globals)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel_launcher.py", line 17, in <module>
      app.launch_new_instance()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/traitlets/config/application.py", line 1041, in launch_instance
      app.start()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/kernelapp.py", line 711, in start
      self.io_loop.start()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/tornado/platform/asyncio.py", line 215, in start
      self.asyncio_loop.run_forever()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/asyncio/base_events.py", line 563, in run_forever
      self._run_once()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/asyncio/base_events.py", line 1844, in _run_once
      handle._run()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/asyncio/events.py", line 81, in _run
      self._context.run(self._callback, *self._args)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/kernelbase.py", line 510, in dispatch_queue
      await self.process_one()
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/kernelbase.py", line 499, in process_one
      await dispatch(*args)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/kernelbase.py", line 406, in dispatch_shell
      await result
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/kernelbase.py", line 729, in execute_request
      reply_content = await reply_content
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/ipkernel.py", line 411, in do_execute
      res = shell.run_cell(
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/ipykernel/zmqshell.py", line 530, in run_cell
      return super().run_cell(*args, **kwargs)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 2945, in run_cell
      result = self._run_cell(
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3000, in _run_cell
      return runner(coro)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/async_helpers.py", line 129, in _pseudo_sync_runner
      coro.send(None)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3203, in run_cell_async
      has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3382, in run_ast_nodes
      if await self.run_code(code, result, async_=asy):
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/IPython/core/interactiveshell.py", line 3442, in run_code
      exec(code_obj, self.user_global_ns, self.user_ns)
    File "/tmp/ipykernel_55359/2068720184.py", line 21, in <module>
      loss_history = model.fit(
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/utils/traceback_utils.py", line 64, in error_handler
      return fn(*args, **kwargs)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 1384, in fit
      tmp_logs = self.train_function(iterator)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 1021, in train_function
      return step_function(self, iterator)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 1010, in step_function
      outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 1000, in run_step
      outputs = model.train_step(data)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 860, in train_step
      loss = self.compute_loss(x, y, y_pred, sample_weight)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/training.py", line 918, in compute_loss
      return self.compiled_loss(
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/engine/compile_utils.py", line 201, in __call__
      loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/losses.py", line 141, in __call__
      losses = call_fn(y_true, y_pred)
    File "/data/anaconda3/envs/deepcell/lib/python3.8/site-packages/keras/losses.py", line 245, in call
      return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File "/tmp/ipykernel_55359/1622303093.py", line 46, in _semantic_loss
      if n_classes > 1:
    File "/tmp/ipykernel_55359/1622303093.py", line 47, in _semantic_loss
      return  weighted_categorical_crossentropy(y_true, y_pred, n_classes=n_classes) * 0.01
    File "/tmp/ipykernel_55359/1622303093.py", line 42, in weighted_categorical_crossentropy
      return - K.sum((y_true * K.log(y_pred) * class_weights), axis=axis)
Node: 'semantic_loss_3/mul_1'
required broadcastable shapes
	 [[{{node semantic_loss_3/mul_1}}]] [Op:__inference_train_function_324275]

[rossbar]

Are you defining a custom weighted_categorical_crossentropy function or using the one directly from the losses module?

The traceback indicates that the problem appears to be due to a shape mismatch in the return statement of the weighted_categorical_crossentropy function. The most likely scenario is that the inputs don't conform to the expected shape(s).

[fanweiya]

Are you defining a custom weighted_categorical_crossentropy function or using the one directly from the losses module?

The traceback indicates that the problem appears to be due to a shape mismatch in the return statement of the weighted_categorical_crossentropy function. The most likely scenario is that the inputs don't conform to the expected shape(s).

No,I only take weighted_categorical_crossentropy print log to sort out the problem,I initially reported this error with weighted_categorical_crossentropy from the losses module, and then I tried to take weighted_categorical_crossentropy print log to sort out the problem, but I still couldn't find anything wrong?