运行不起来 报错 不知道咋回事 求帮助

乔大仙

星级打分

• 1
• 2
• 3
• 4
• 5

平均分:NAN  参与人数:0  我的评分:未评

Starting. Press "Enter" to stop training and save model.
Error: OOM when allocating tensor with shape[4,128,258,258] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
         [[node gradients/Conv2D_18_grad/Conv2DBackpropInput (defined at \DeepFaceLab\core\leras\ops\__init__.py:55) ]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.


Errors may have originated from an input operation.
Input Source operations connected to node gradients/Conv2D_18_grad/Conv2DBackpropInput:
decoder_src/res2/conv2/weight/read (defined at \DeepFaceLab\core\leras\layers\Conv2D.py:61)

Original stack trace for 'gradients/Conv2D_18_grad/Conv2DBackpropInput':
  File "\python-3.8.5\lib\threading.py", line 890, in _bootstrap
    self._bootstrap_inner()
  File "\python-3.8.5\lib\threading.py", line 932, in _bootstrap_inner
    self.run()
  File "\python-3.8.5\lib\threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "\DeepFaceLab\mainscripts\Trainer.py", line 47, in trainerThread
    model = models.import_model(model_class_name)(
  File "\DeepFaceLab\models\ModelBase.py", line 193, in __init__
    self.on_initialize()
  File "\DeepFaceLab\models\Model_SAEHD\Model.py", line 547, in on_initialize
    gpu_G_loss_gvs += [ nn.gradients ( gpu_G_loss, self.src_dst_trainable_weights )]
  File "\DeepFaceLab\core\leras\ops\__init__.py", line 55, in tf_gradients
    grads = gradients.gradients(loss, vars, colocate_gradients_with_ops=True )
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 169, in gradients
    return gradients_util._GradientsHelper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 681, in _GradientsHelper
    in_grads = _MaybeCompile(grad_scope, op, func_call,
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 338, in _MaybeCompile
    return grad_fn()  # Exit early
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 682, in <lambda>
    lambda: grad_fn(op, *out_grads))
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\nn_grad.py", line 581, in _Conv2DGrad
    gen_nn_ops.conv2d_backprop_input(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1284, in conv2d_backprop_input
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 748, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\ops.py", line 3557, in _create_op_internal
    ret = Operation(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\ops.py", line 2045, in __init__
    self._traceback = tf_stack.extract_stack_for_node(self._c_op)

...which was originally created as op 'Conv2D_18', defined at:
  File "\python-3.8.5\lib\threading.py", line 890, in _bootstrap
    self._bootstrap_inner()
[elided 3 identical lines from previous traceback]
  File "\DeepFaceLab\models\ModelBase.py", line 193, in __init__
    self.on_initialize()
  File "\DeepFaceLab\models\Model_SAEHD\Model.py", line 409, in on_initialize
    gpu_pred_src_src, gpu_pred_src_srcm = self.decoder_src(gpu_src_code)
  File "\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 226, in forward
    x = self.res2(x)
  File "\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 84, in forward
    x = self.conv2(x)
  File "\DeepFaceLab\core\leras\layers\LayerBase.py", line 14, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\layers\Conv2D.py", line 101, in forward
    x = tf.nn.conv2d(x, weight, strides, 'VALID', dilations=dilations, data_format=nn.data_format)
  File "\python-3.8.5\lib\site-packages\tensorflow\python\util\dispatch.py", line 206, in wrapper
    return target(*args, **kwargs)
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 2269, in conv2d
    return gen_nn_ops.conv2d(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 968, in conv2d
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 748, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,

Traceback (most recent call last):
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1375, in _do_call
    return fn(*args)
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1359, in _run_fn
    return self._call_tf_sessionrun(options, feed_dict, fetch_list,
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1451, in _call_tf_sessionrun
    return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[4,128,258,258] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
         [[{{node gradients/Conv2D_18_grad/Conv2DBackpropInput}}]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.


During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "D:\DeepFaceLab\_internal\DeepFaceLab\mainscripts\Trainer.py", line 141, in trainerThread
    iter, iter_time = model.train_one_iter()
  File "D:\DeepFaceLab\_internal\DeepFaceLab\models\ModelBase.py", line 474, in train_one_iter
    losses = self.onTrainOneIter()
  File "D:\DeepFaceLab\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 774, in onTrainOneIter
    src_loss, dst_loss = self.src_dst_train (warped_src, target_src, target_srcm, target_srcm_em, warped_dst, target_dst, target_dstm, target_dstm_em)
  File "D:\DeepFaceLab\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 576, in src_dst_train
    s, d = nn.tf_sess.run ( [ src_loss, dst_loss, src_dst_loss_gv_op],
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 967, in run
    result = self._run(None, fetches, feed_dict, options_ptr,
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1190, in _run
    results = self._do_run(handle, final_targets, final_fetches,
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1368, in _do_run
    return self._do_call(_run_fn, feeds, fetches, targets, options,
  File "D:\DeepFaceLab\_internal\python-3.8.5\lib\site-packages\tensorflow\python\client\session.py", line 1394, in _do_call
    raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor with shape[4,128,258,258] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc
         [[node gradients/Conv2D_18_grad/Conv2DBackpropInput (defined at \DeepFaceLab\core\leras\ops\__init__.py:55) ]]
Hint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.


Errors may have originated from an input operation.
Input Source operations connected to node gradients/Conv2D_18_grad/Conv2DBackpropInput:
decoder_src/res2/conv2/weight/read (defined at \DeepFaceLab\core\leras\layers\Conv2D.py:61)

Original stack trace for 'gradients/Conv2D_18_grad/Conv2DBackpropInput':
  File "\python-3.8.5\lib\threading.py", line 890, in _bootstrap
    self._bootstrap_inner()
  File "\python-3.8.5\lib\threading.py", line 932, in _bootstrap_inner
    self.run()
  File "\python-3.8.5\lib\threading.py", line 870, in run
    self._target(*self._args, **self._kwargs)
  File "\DeepFaceLab\mainscripts\Trainer.py", line 47, in trainerThread
    model = models.import_model(model_class_name)(
  File "\DeepFaceLab\models\ModelBase.py", line 193, in __init__
    self.on_initialize()
  File "\DeepFaceLab\models\Model_SAEHD\Model.py", line 547, in on_initialize
    gpu_G_loss_gvs += [ nn.gradients ( gpu_G_loss, self.src_dst_trainable_weights )]
  File "\DeepFaceLab\core\leras\ops\__init__.py", line 55, in tf_gradients
    grads = gradients.gradients(loss, vars, colocate_gradients_with_ops=True )
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 169, in gradients
    return gradients_util._GradientsHelper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 681, in _GradientsHelper
    in_grads = _MaybeCompile(grad_scope, op, func_call,
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 338, in _MaybeCompile
    return grad_fn()  # Exit early
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 682, in <lambda>
    lambda: grad_fn(op, *out_grads))
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\nn_grad.py", line 581, in _Conv2DGrad
    gen_nn_ops.conv2d_backprop_input(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1284, in conv2d_backprop_input
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 748, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\ops.py", line 3557, in _create_op_internal
    ret = Operation(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\ops.py", line 2045, in __init__
    self._traceback = tf_stack.extract_stack_for_node(self._c_op)

...which was originally created as op 'Conv2D_18', defined at:
  File "\python-3.8.5\lib\threading.py", line 890, in _bootstrap
    self._bootstrap_inner()
[elided 3 identical lines from previous traceback]
  File "\DeepFaceLab\models\ModelBase.py", line 193, in __init__
    self.on_initialize()
  File "\DeepFaceLab\models\Model_SAEHD\Model.py", line 409, in on_initialize
    gpu_pred_src_src, gpu_pred_src_srcm = self.decoder_src(gpu_src_code)
  File "\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 226, in forward
    x = self.res2(x)
  File "\DeepFaceLab\core\leras\models\ModelBase.py", line 117, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\archis\DeepFakeArchi.py", line 84, in forward
    x = self.conv2(x)
  File "\DeepFaceLab\core\leras\layers\LayerBase.py", line 14, in __call__
    return self.forward(*args, **kwargs)
  File "\DeepFaceLab\core\leras\layers\Conv2D.py", line 101, in forward
    x = tf.nn.conv2d(x, weight, strides, 'VALID', dilations=dilations, data_format=nn.data_format)
  File "\python-3.8.5\lib\site-packages\tensorflow\python\util\dispatch.py", line 206, in wrapper
    return target(*args, **kwargs)
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 2269, in conv2d
    return gen_nn_ops.conv2d(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 968, in conv2d
    _, _, _op, _outputs = _op_def_library._apply_op_helper(
  File "\python-3.8.5\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 748, in _apply_op_helper
    op = g._create_op_internal(op_type_name, inputs, dtypes=None,

sdsAWDQAW

爆显存了 BS设置小一点

wwe456999

虚拟内存加到200g

PRCEnderman

这是一个训练 AI 模型时的错误信息，表明内存不足以分配一个形状为 [4,128,258,258] 的张量。这个错误可能是由于设备内存不足造成的。可以考虑减少 batch_size，或者使用更高配置的设备来解决这个问题。