UnknownError: Failed to get convolution algorithm. This is probably because cuDNN failed to initialize

---------------------------------------------------------------------------
UnknownError                              Traceback (most recent call last)
Cell In[1], line 59
     54 model.add(Dense(num_classes, activation='softmax'))
     56 model.compile(loss=keras.losses.categorical_crossentropy,
     57               optimizer=keras.optimizers.Adadelta(),
     58               metrics=['accuracy'])
---> 59 model.fit(x_train, y_train,
     60           batch_size=batch_size,
     61           epochs=epochs,
     62           verbose=1,
     63           validation_data=(x_test, y_test))
     64 score = model.evaluate(x_test, y_test, verbose=0)
     65 print('Test loss:', score[0])

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:66, in enable_multi_worker.<locals>._method_wrapper(self, *args, **kwargs)
     64 def _method_wrapper(self, *args, **kwargs):
     65   if not self._in_multi_worker_mode():  # pylint: disable=protected-access
---> 66     return method(self, *args, **kwargs)
     68   # Running inside `run_distribute_coordinator` already.
     69   if dc_context.get_current_worker_context():

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/keras/engine/training.py:848, in Model.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
    841 with traceme.TraceMe(
    842     'TraceContext',
    843     graph_type='train',
    844     epoch_num=epoch,
    845     step_num=step,
    846     batch_size=batch_size):
    847   callbacks.on_train_batch_begin(step)
--> 848   tmp_logs = train_function(iterator)
    849   # Catch OutOfRangeError for Datasets of unknown size.
    850   # This blocks until the batch has finished executing.
    851   # TODO(b/150292341): Allow multiple async steps here.
    852   if not data_handler.inferred_steps:

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py:580, in Function.__call__(self, *args, **kwds)
    578     xla_context.Exit()
    579 else:
--> 580   result = self._call(*args, **kwds)
    582 if tracing_count == self._get_tracing_count():
    583   self._call_counter.called_without_tracing()

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/def_function.py:644, in Function._call(self, *args, **kwds)
    640     pass  # Fall through to cond-based initialization.
    641   else:
    642     # Lifting succeeded, so variables are initialized and we can run the
    643     # stateless function.
--> 644     return self._stateless_fn(*args, **kwds)
    645 else:
    646   canon_args, canon_kwds = \
    647       self._stateful_fn._function_spec.canonicalize_function_inputs(  # pylint: disable=protected-access
    648           *args, **kwds)

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/function.py:2420, in Function.__call__(self, *args, **kwargs)
   2418 with self._lock:
   2419   graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
-> 2420 return graph_function._filtered_call(args, kwargs)

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/function.py:1661, in ConcreteFunction._filtered_call(self, args, kwargs)
   1647 def _filtered_call(self, args, kwargs):
   1648   """Executes the function, filtering arguments from the Python function.
   1649 
   1650   Objects aside from Tensors, CompositeTensors, and Variables are ignored.
   (...)
   1659     `args` and `kwargs`.
   1660   """
-> 1661   return self._call_flat(
   1662       (t for t in nest.flatten((args, kwargs), expand_composites=True)
   1663        if isinstance(t, (ops.Tensor,
   1664                          resource_variable_ops.BaseResourceVariable))),
   1665       self.captured_inputs)

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/function.py:1745, in ConcreteFunction._call_flat(self, args, captured_inputs, cancellation_manager)
   1740 possible_gradient_type = (
   1741     pywrap_tfe.TFE_Py_TapeSetPossibleGradientTypes(args))
   1742 if (possible_gradient_type == _POSSIBLE_GRADIENT_TYPES_NONE
   1743     and executing_eagerly):
   1744   # No tape is watching; skip to running the function.
-> 1745   return self._build_call_outputs(self._inference_function.call(
   1746       ctx, args, cancellation_manager=cancellation_manager))
   1747 forward_backward = self._select_forward_and_backward_functions(
   1748     args,
   1749     possible_gradient_type,
   1750     executing_eagerly)
   1751 forward_function, args_with_tangents = forward_backward.forward()

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/function.py:593, in _EagerDefinedFunction.call(self, ctx, args, cancellation_manager)
    591 with _InterpolateFunctionError(self):
    592   if cancellation_manager is None:
--> 593     outputs = execute.execute(
    594         str(self.signature.name),
    595         num_outputs=self._num_outputs,
    596         inputs=args,
    597         attrs=attrs,
    598         ctx=ctx)
    599   else:
    600     outputs = execute.execute_with_cancellation(
    601         str(self.signature.name),
    602         num_outputs=self._num_outputs,
   (...)
    605         ctx=ctx,
    606         cancellation_manager=cancellation_manager)

File /home/software/anaconda3/envs/mydlenv/lib/python3.8/site-packages/tensorflow/python/eager/execute.py:59, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     57 try:
     58   ctx.ensure_initialized()
---> 59   tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
     60                                       inputs, attrs, num_outputs)
     61 except core._NotOkStatusException as e:
     62   if name is not None:

UnknownError:  Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above.
	 [[node sequential/conv2d/Conv2D (defined at tmp/ipykernel_30155/4033766983.py:59) ]] [Op:__inference_train_function_848]

Function call stack:
train_function