سلام
وقتتون بخیر
من Training را انجام میدهم اما به ارور libcuda.so cannot found برخورد میکنم ، ممنون میشم راهنماییم کنید.
tset: symbol lookup error: tset: undefined symbol: _nc_cur_term, version NCURSES6_TINFO_5.7.20081102
u111365@en-7-2:~> export nnUNet_raw=/home/u111365/wrkdir/BraTS/nnUNet_raw
u111365@en-7-2:~> export nnUNet_preprocessed=/home/u111365/wrkdir/BraTS/nnUNet_preprocessed
u111365@en-7-2:~> export nnUNet_results=/home/u111365/wrkdir/BraTS/nnUNet_results
u111365@en-7-2:~>
u111365@en-7-2:~> echo $nnUNet_raw
/home/u111365/wrkdir/BraTS/nnUNet_raw
u111365@en-7-2:~> echo $nnUNet_preprocessed
/home/u111365/wrkdir/BraTS/nnUNet_preprocessed
u111365@en-7-2:~> echo $nnUNet_results
/home/u111365/wrkdir/BraTS/nnUNet_results
u111365@en-7-2:~> nnUNetv2_train 003 2d 0 -device cuda -num_gpus 1
############################
INFO: You are using the old nnU-Net default plans. We have updated our recommendations. Please consider using those instead! Read more here: https://github.com/MIC-DKFZ/nnUNet/blob/master/documentation/resenc_presets.md
############################
Using device: cuda:0
#######################################################################
Please cite the following paper when using nnU-Net:
Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211.
#######################################################################
2025-12-19 16:34:23.448397: Using torch.compile...
2025-12-19 16:34:24.683508: do_dummy_2d_data_aug: False
2025-12-19 16:34:24.699586: Creating new 5-fold cross-validation split...
2025-12-19 16:34:24.708977: Desired fold for training: 0
2025-12-19 16:34:24.714038: This split has 88 training and 22 validation cases.
using pin_memory on device 0
using pin_memory on device 0
This is the configuration used by this training:
Configuration name: 2d
{'data_identifier': 'nnUNetPlans_2d', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 105, 'patch_size': [192, 160], 'median_image_size_in_voxels': [167.0, 136.0], 'spacing': [1.0, 1.0], 'normalization_schemes': ['ZScoreNormalization', 'ZScoreNormalization', 'ZScoreNormalization', 'ZScoreNormalization'], 'use_mask_for_norm': [True, True, True, True], 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'architecture': {'network_class_name': 'dynamic_network_architectures.architectures.unet.PlainConvUNet', 'arch_kwargs': {'n_stages': 6, 'features_per_stage': [32, 64, 128, 256, 512, 512], 'conv_op': 'torch.nn.modules.conv.Conv2d', 'kernel_sizes': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'strides': [[1, 1], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]], 'n_conv_per_stage': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'conv_bias': True, 'norm_op': 'torch.nn.modules.instancenorm.InstanceNorm2d', 'norm_op_kwargs': {'eps': 1e-05, 'affine': True}, 'dropout_op': None, 'dropout_op_kwargs': None, 'nonlin': 'torch.nn.LeakyReLU', 'nonlin_kwargs': {'inplace': True}}, '_kw_requires_import': ['conv_op', 'norm_op', 'dropout_op', 'nonlin']}, 'batch_dice': True}
These are the global plan.json settings:
{'dataset_name': 'Dataset003_Glioma', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [1.0, 1.0, 1.0], 'original_median_shape_after_transp': [138, 167, 136], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 3675.261962890625, 'mean': 274.6809387207031, 'median': 230.46986389160156, 'min': 0.0, 'percentile_00_5': 37.6374397277832, 'percentile_99_5': 884.48876953125, 'std': 160.11782836914062}, '1': {'max': 5681.2666015625, 'mean': 362.312255859375, 'median': 273.904052734375, 'min': 0.0, 'percentile_00_5': 38.66485595703125, 'percentile_99_5': 2244.980712890625, 'std': 307.4349060058594}, '2': {'max': 3824.501953125, 'mean': 721.8897705078125, 'median': 616.6156616210938, 'min': 0.0, 'percentile_00_5': 254.9205322265625, 'percentile_99_5': 2219.014892578125, 'std': 358.75250244140625}, '3': {'max': 3607.873046875, 'mean': 454.9068298339844, 'median': 428.2197570800781, 'min': 0.0, 'percentile_00_5': 15.681903839111328, 'percentile_99_5': 1437.018798828125, 'std': 302.9181213378906}}}
2025-12-19 16:34:50.299024: Unable to plot network architecture: nnUNet_compile is enabled!
2025-12-19 16:34:50.327047:
2025-12-19 16:34:50.363022: Epoch 0
2025-12-19 16:34:50.407262: Current learning rate: 0.01
/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/lowering.py:7007: UserWarning:
Online softmax is disabled on the fly since Inductor decides to
split the reduction. Cut an issue to PyTorch if this is an
important use case and you want to speed it up with online
softmax.
warnings.warn(
/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/lowering.py:7007: UserWarning:
Online softmax is disabled on the fly since Inductor decides to
split the reduction. Cut an issue to PyTorch if this is an
important use case and you want to speed it up with online
softmax.
warnings.warn(
Traceback (most recent call last):
File "/home/u111365/.local/bin/nnUNetv2_train", line 8, in <module>
sys.exit(run_training_entry())
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/run/run_training.py", line 266, in run_training_entry
run_training(args.dataset_name_or_id, args.configuration, args.fold, args.tr, args.p, args.pretrained_weights,
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/run/run_training.py", line 207, in run_training
nnunet_trainer.run_training()
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py", line 1371, in run_training
train_outputs.append(self.train_step(next(self.dataloader_train)))
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/training/nnUNetTrainer/nnUNetTrainer.py", line 991, in train_step
l = self.loss(output, target)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/training/loss/deep_supervision.py", line 29, in forward
return sum([weights[i] * self.loss(*inputs) for i, inputs in enumerate(zip(*args)) if weights[i] != 0.0])
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/training/loss/deep_supervision.py", line 29, in <listcomp>
return sum([weights[i] * self.loss(*inputs) for i, inputs in enumerate(zip(*args)) if weights[i] != 0.0])
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/home/u111365/.local/lib/python3.10/site-packages/nnunetv2/training/loss/compound_losses.py", line 50, in forward
dc_loss = self.dc(net_output, target_dice, loss_mask=mask) \
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py", line 663, in _fn
raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 760, in _compile_fx_inner
raise InductorError(e, currentframe()).with_traceback(
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 745, in _compile_fx_inner
mb_compiled_graph = fx_codegen_and_compile(
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1295, in fx_codegen_and_compile
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/compile_fx.py", line 1197, in codegen_and_compile
compiled_fn = graph.compile_to_module().call
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2083, in compile_to_module
return self._compile_to_module()
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/graph.py", line 2130, in _compile_to_module
mod = PyCodeCache.load_by_key_path(
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 2747, in load_by_key_path
mod = _reload_python_module(key, path)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/runtime/compile_tasks.py", line 36, in _reload_python_module
exec(code, mod.__dict__, mod.__dict__)
File "/tmp/torchinductor_u111365/7j/c7jpbmuonotdq4wzqlecjzyds2ni5n533cwm5ktrvdogoatqb4a2.py", line 407, in <module>
async_compile.wait(globals())
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 424, in wait
self._wait_futures(scope)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 445, in _wait_futures
scope[key] = result.result()
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/codecache.py", line 3224, in result
return self.result_fn()
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/async_compile.py", line 325, in get_result
kernel.precompile(
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 277, in precompile
self._make_launchers()
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 434, in _make_launchers
launchers.append(result.make_launcher())
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 1091, in make_launcher
binary._init_handles()
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/triton/compiler/compiler.py", line 397, in _init_handles
self.run = driver.active.launcher_cls(self.src, self.metadata)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 516, in __init__
mod = compile_module_from_src(src, "__triton_launcher")
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 66, in compile_module_from_src
so = _build(name, src_path, tmpdir, library_dirs(), include_dir, libraries)
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 47, in library_dirs
return [libdevice_dir, *libcuda_dirs()]
File "/share/apps/eb/PyTorch/2.7.1-Py3.10.8/lib/python3.10/site-packages/triton/backends/nvidia/driver.py", line 41, in libcuda_dirs
assert any(os.path.exists(os.path.join(path, 'libcuda.so.1')) for path in dirs), msg
torch._inductor.exc.InductorError: AssertionError: libcuda.so cannot found!
Please make sure GPU is set up and then run "/sbin/ldconfig" (requires sudo) to refresh the linker cache.
Set TORCHDYNAMO_VERBOSE=1 for the internal stack trace (please do this especially if you're reporting a bug to PyTorch). For even more developer context, set TORCH_LOGS="+dynamo"
Exception in thread Thread-2 (results_loop):
Traceback (most recent call last):
File "/share/apps/eb/Python/3.10.8-GCCcore-12.2.0/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
Exception in thread Thread-3 (results_loop):
Traceback (most recent call last):
File "/share/apps/eb/Python/3.10.8-GCCcore-12.2.0/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/share/apps/eb/Python/3.10.8-GCCcore-12.2.0/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/home/u111365/.local/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 125, in results_loop
self.run()
File "/share/apps/eb/Python/3.10.8-GCCcore-12.2.0/lib/python3.10/threading.py", line 953, in run
self._target(*self._args, **self._kwargs)
File "/home/u111365/.local/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 125, in results_loop
raise e
File "/home/u111365/.local/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 103, in results_loop
raise e
File "/home/u111365/.local/lib/python3.10/site-packages/batchgenerators/dataloading/nondet_multi_threaded_augmenter.py", line 103, in results_loop
raise RuntimeError("One or more background workers are no longer alive. Exiting. Please check the "
RuntimeError: One or more background workers are no longer alive. Exiting. Please check the print statements above for the actual error message
u111365@en-7-2:~>