Detected at node 'Conv2D_25' defined at (most recent call last):
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 890, in _bootstrap
self._bootstrap_inner()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "<frozen mainscripts.Trainer>", line 46, in trainerThread
File "<frozen models.ModelBase>", line 207, in __init__
File "<frozen models.Model_SAEHD.Model>", line 414, in on_initialize
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 226, in forward
x = self.res2(x)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 82, in forward
x = self.conv1(inp)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\LayerBase.py", line 14, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\Conv2D.py", line 101, in forward
x = tf.nn.conv2d(x, weight, strides, 'VALID', dilations=dilations, data_format=nn.data_format)
Node: 'Conv2D_25'
Detected at node 'Conv2D_25' defined at (most recent call last):
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 890, in _bootstrap
self._bootstrap_inner()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "<frozen mainscripts.Trainer>", line 46, in trainerThread
File "<frozen models.ModelBase>", line 207, in __init__
File "<frozen models.Model_SAEHD.Model>", line 414, in on_initialize
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 226, in forward
x = self.res2(x)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 82, in forward
x = self.conv1(inp)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\LayerBase.py", line 14, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\Conv2D.py", line 101, in forward
x = tf.nn.conv2d(x, weight, strides, 'VALID', dilations=dilations, data_format=nn.data_format)
Node: 'Conv2D_25'
2 root error(s) found.
(0) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[8,384,96,96] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator PluggableDevice_0_bfc
[[{{node Conv2D_25}}]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.
[[concat_5/concat/_1457]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.
(1) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[8,384,96,96] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator PluggableDevice_0_bfc
[[{{node Conv2D_25}}]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.
Original stack trace for 'Conv2D_25':
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 890, in _bootstrap
self._bootstrap_inner()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "<frozen mainscripts.Trainer>", line 46, in trainerThread
File "<frozen models.ModelBase>", line 207, in __init__
File "<frozen models.Model_SAEHD.Model>", line 414, in on_initialize
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 226, in forward
x = self.res2(x)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 82, in forward
x = self.conv1(inp)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\LayerBase.py", line 14, in __call__
return self.forward(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\DeepFaceLab\core\leras\layers\Conv2D.py", line 101, in forward
x = tf.nn.conv2d(x, weight, strides, 'VALID', dilations=dilations, data_format=nn.data_format)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\traceback_utils.py", line 150, in error_handler
return fn(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 1176, in op_dispatch_handler
return dispatch_target(*args, **kwargs)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 2458, in conv2d
return gen_nn_ops.conv2d(
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1242, in conv2d
_, _, _op, _outputs = _op_def_library._apply_op_helper(
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 797, in _apply_op_helper
op = g._create_op_internal(op_type_name, inputs, dtypes=None,
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 3800, in _create_op_internal
ret = Operation(
Traceback (most recent call last):
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1378, in _do_call
return fn(*args)
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1361, in _run_fn
return self._call_tf_sessionrun(options, feed_dict, fetch_list,
File "D:\ai\DeepFaceLab\DeepFaceLab_MZHH_DirectML_24_0601\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1454, in _call_tf_sessionrun
return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.ResourceExhaustedError: 2 root error(s) found.
(0) RESOURCE_EXHAUSTED: OOM when allocating tensor with shape[8,384,96,96] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator PluggableDevice_0_bfc
[[{{node Conv2D_25}}]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.
[[concat_5/concat/_1457]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info. This isn't available when running in Eager mode.