thread '<unnamed>' panicked at 'failed to create ColumnTreeBuilder: ClError(BusIdNotAvailable)' - Full logs

[seccentral]

Environment:
debian 10
rtx 2080

root@filecoin:~/rust-fil-proofs# modinfo nvidia
filename:       /lib/modules/4.19.0-14-amd64/updates/dkms/nvidia.ko
alias:          char-major-195-*
version:        460.39
supported:      external
license:        NVIDIA
srcversion:     6D32E1B1DFCA8DA07AD1677
alias:          pci:v000010DEd*sv*sd*bc03sc02i00*
alias:          pci:v000010DEd*sv*sd*bc03sc00i00*
depends:        
retpoline:      Y
name:           nvidia
vermagic:       4.19.0-14-amd64 SMP mod_unload modversions 
parm:           NvSwitchRegDwords:NvSwitch regkey (charp)
parm:           NvSwitchBlacklist:NvSwitchBlacklist=uuid[,uuid...] (charp)
parm:           nv_cap_enable_devfs:Enable (1) or disable (0) nv-caps devfs support. Default: 1 (int)
parm:           NVreg_ResmanDebugLevel:int
parm:           NVreg_RmLogonRC:int
parm:           NVreg_ModifyDeviceFiles:int
parm:           NVreg_DeviceFileUID:int
parm:           NVreg_DeviceFileGID:int
parm:           NVreg_DeviceFileMode:int
parm:           NVreg_InitializeSystemMemoryAllocations:int
parm:           NVreg_UsePageAttributeTable:int
parm:           NVreg_RegisterForACPIEvents:int
parm:           NVreg_EnablePCIeGen3:int
parm:           NVreg_EnableMSI:int
parm:           NVreg_TCEBypassMode:int
parm:           NVreg_EnableStreamMemOPs:int
parm:           NVreg_EnableBacklightHandler:int
parm:           NVreg_RestrictProfilingToAdminUsers:int
parm:           NVreg_PreserveVideoMemoryAllocations:int
parm:           NVreg_EnableS0ixPowerManagement:int
parm:           NVreg_S0ixPowerManagementVideoMemoryThreshold:int
parm:           NVreg_DynamicPowerManagement:int
parm:           NVreg_DynamicPowerManagementVideoMemoryThreshold:int
parm:           NVreg_EnableUserNUMAManagement:int
parm:           NVreg_MemoryPoolSize:int
parm:           NVreg_KMallocHeapMaxSize:int
parm:           NVreg_VMallocHeapMaxSize:int
parm:           NVreg_IgnoreMMIOCheck:int
parm:           NVreg_NvLinkDisable:int
parm:           NVreg_EnablePCIERelaxedOrderingMode:int
parm:           NVreg_RegisterPCIDriver:int
parm:           NVreg_RegistryDwords:charp
parm:           NVreg_RegistryDwordsPerDevice:charp
parm:           NVreg_RmMsg:charp
parm:           NVreg_GpuBlacklist:charp
parm:           NVreg_TemporaryFilePath:charp

root@filecoin:~/rust-fil-proofs# nvidia-detect 
Detected NVIDIA GPUs:
0e:00.0 VGA compatible controller [0300]: NVIDIA Corporation TU104 [GeForce RTX 2080 Rev. A] [10de:1e87] (rev a1)

Checking card:  NVIDIA Corporation TU104 [GeForce RTX 2080 Rev. A] (rev a1)
Your card is supported by the default drivers.
It is recommended to install the
    nvidia-driver
package.

Filecoin proof parameters:

root@filecoin:~/rust-fil-proofs# ls -l /var/tmp/filecoin-proof-parameters/
total 106958008
-rw-r--r-- 1 fil fil    11501496 Feb 18 20:17 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.params
-rw-r--r-- 1 fil fil        3076 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0170db1f394b35d995252228ee359194b13199d259380541dc529fb0099096b0.vk
-rw-r--r-- 1 fil fil        3076 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-0cfb4f178bbb71cf2ecfcd42accce558b27199ab4fb59cb78f2483fe21ef36d9.vk
-rw-r--r-- 1 fil fil    47299128 Feb 18 20:17 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.params
-rw-r--r-- 1 fil fil       13636 Feb 18 06:38 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-3ea05428c9d11689f23529cde32fd30aabd50f7d2c93657c1d3650bca3e8ea9e.vk
-rw-r--r-- 1 fil fil       13636 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-50c7368dea9593ed0989e70974d28024efa9d156d585b7eea1be22b2e753f331.vk
-rw-r--r-- 1 fil fil        3076 Feb 18 06:38 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-5294475db5237a2e83c3e52fd6c2b03859a1831d45ed08c4f35dbf9a803165a9.vk
-rw-r--r-- 1 fil fil       13636 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-0-0-7d739b8cf60f1b0709eeebee7730e297683552e4b69cab6984ec0285663c5781.vk
-rw-r--r-- 1 fil fil 60780374232 Feb 18 06:37 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.params
-rw-r--r-- 1 fil fil     2481508 Feb 18 03:56 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-0377ded656c6f524f1618760bffe4e0a1c51d5a70c4509eedae8a27555733edc.vk
-rw-r--r-- 1 fil fil   192050232 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.params
-rw-r--r-- 1 fil fil       13636 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-0-559e581f022bb4e4ec6e719e563bf0e026ad6de42e56c18714a2c692b1b88d7e.vk
-rw-r--r-- 1 fil fil     2429764 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-2627e4006b67f99cef990c0a47d5426cb7ab0a0ad58fc1061547bf2d28b09def.vk
-rw-r--r-- 1 fil fil       13636 Feb 18 05:07 v28-proof-of-spacetime-fallback-merkletree-poseidon_hasher-8-8-2-b62098629d07946e9028127e70295ed996fe3ed25b0f9f88eb610a0ab4385a3c.vk
-rw-r--r-- 1 fil fil  1114707768 Feb 18 20:19 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.params
-rw-r--r-- 1 fil fil        4708 Feb 18 05:07 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-032d3138d22506ec0082ed72b2dcba18df18477904e35bafee82b3793b06832f.vk
-rw-r--r-- 1 fil fil        4708 Feb 18 05:07 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-6babf46ce344ae495d558e7770a585b2382d54f225af8ed0397b8be7c3fcd472.vk
-rw-r--r-- 1 fil fil        4708 Feb 18 05:07 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-0-0-sha256_hasher-ecd683648512ab1765faa2a5f14bab48f676e633467f0aa8aad4b55dcb0652bb.vk
-rw-r--r-- 1 fil fil 47373928632 Feb 18 05:06 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.params
-rw-r--r-- 1 fil fil       32356 Feb 18 05:07 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-0-sha256_hasher-82a357d2f2ca81dc61bb45f4a762807aedee1b0a53fd6c4e77b46a01bfef7820.vk
-rw-r--r-- 1 fil fil       32356 Feb 18 05:07 v28-stacked-proof-of-replication-merkletree-poseidon_hasher-8-8-2-sha256_hasher-96f1b4a04c5c51e4759bbf224bbc2ef5a42c7100f16ec0637123f16a845ddfb2.vk

filecoin parents:

root@filecoin:~/rust-fil-proofs# ls -l /var/tmp/filecoin-parents/
total 58734620
-rw-r--r-- 1 fil fil 60129542144 Feb 18 06:59 v28-sdr-parent-21981246c370f9d76c7a77ab273d94bde0ceb4e938292334960bce05585dc117.cache
-rw-r--r-- 1 fil fil        3584 Feb 18 20:17 v28-sdr-parent-494d91dc80f2df5272c4b9e129bc7ade9405225993af9fe34e6542a39a47554b.cache
-rw-r--r-- 1 fil fil    14680064 Feb 19 16:11 v28-sdr-parent-520dd701e660dbfdfd40ab66a0720b0abfa5f438f5805580dea368e6dfcc9fa0.cache

rust-fil-proofs settings:

root@filecoin:~/rust-fil-proofs# cargo run --bin settings
    Finished dev [unoptimized + debuginfo] target(s) in 1.32s
     Running `target/debug/settings`
Settings {
    verify_cache: true,
    verify_production_params: false,
    use_gpu_column_builder: true,
    max_gpu_column_batch_size: 400000,
    column_write_batch_size: 262144,
    use_gpu_tree_builder: true,
    max_gpu_tree_batch_size: 700000,
    rows_to_discard: 2,
    sdr_parents_cache_size: 2048,
    window_post_synthesis_num_cpus: 16,
    parameter_cache: "/var/tmp/filecoin-proof-parameters/",
    parent_cache: "/var/tmp/filecoin-parents/",
    use_multicore_sdr: true,
    multicore_sdr_producers: 3,
    multicore_sdr_producer_stride: 128,
    multicore_sdr_lookahead: 800,
}

export RUST_BACKTRACE=full

Problem:

root@filecoin:~/rust-fil-proofs# cargo run --release --bin gpu-cpu-test
    Finished release [optimized] target(s) in 1.09s
     Running `target/release/gpu-cpu-test`
thread '<unnamed>' panicked at 'failed to create ColumnTreeBuilder: ClError(BusIdNotAvailable)', /root/rust-fil-proofs/storage-proofs-porep/src/stacked/vanilla/proof.rs:560:22
stack backtrace:
   0:     0x55cfa2a733e5 - backtrace::backtrace::libunwind::trace::h14d338b30b3ea0a7
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/libunwind.rs:86
   1:     0x55cfa2a733e5 - backtrace::backtrace::trace_unsynchronized::h73ea91d74a3fd67f
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/mod.rs:66
   2:     0x55cfa2a733e5 - std::sys_common::backtrace::_print_fmt::hd42948c952866e12
                               at src/libstd/sys_common/backtrace.rs:78
   3:     0x55cfa2a733e5 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::ha8f928866ff7571e
                               at src/libstd/sys_common/backtrace.rs:59
   4:     0x55cfa2a9fc9c - core::fmt::write::he0c1e5f7426d2718
                               at src/libcore/fmt/mod.rs:1076
   5:     0x55cfa2a6e972 - std::io::Write::write_fmt::hf3afc6cfd57d0033
                               at src/libstd/io/mod.rs:1537
   6:     0x55cfa2a760a0 - std::sys_common::backtrace::_print::hfc0110703f3696fd
                               at src/libstd/sys_common/backtrace.rs:62
   7:     0x55cfa2a760a0 - std::sys_common::backtrace::print::h3f77c6990ddfaa22
                               at src/libstd/sys_common/backtrace.rs:49
   8:     0x55cfa2a760a0 - std::panicking::default_hook::{{closure}}::heae49580a8d62d75
                               at src/libstd/panicking.rs:198
   9:     0x55cfa2a75dec - std::panicking::default_hook::hecc34e3f729e213c
                               at src/libstd/panicking.rs:217
  10:     0x55cfa2a766e3 - std::panicking::rust_panic_with_hook::he82f5d0644692441
                               at src/libstd/panicking.rs:526
  11:     0x55cfa2a762db - rust_begin_unwind
                               at src/libstd/panicking.rs:437
  12:     0x55cfa2a9e381 - core::panicking::panic_fmt::h09c929f06bb87c98
                               at src/libcore/panicking.rs:85
  13:     0x55cfa2a9e1a3 - core::option::expect_none_failed::h188f17af6c9f404b
                               at src/libcore/option.rs:1269
  14:     0x55cfa2629fc8 - <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h06ec3e699459a1f2
  15:     0x55cfa26cddc7 - std::panicking::try::hc11f2780affa6caf
  16:     0x55cfa273f504 - <rayon_core::job::HeapJob<BODY> as rayon_core::job::Job>::execute::h85ccafe74ae4e012
  17:     0x55cfa29865c1 - rayon_core::registry::WorkerThread::wait_until_cold::hb862e4ba412ff0d5
  18:     0x55cfa2984afb - rayon_core::registry::ThreadBuilder::run::h59cdb2b1b6db1d0f
  19:     0x55cfa29871d1 - std::sys_common::backtrace::__rust_begin_short_backtrace::h4e48a0eefe8b0b12
  20:     0x55cfa29820af - core::ops::function::FnOnce::call_once{{vtable.shim}}::h6d6e99d9b2ecb2a7
  21:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::hd2b3bc04af94a84f
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  22:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h1044417e186e567a
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  23:     0x55cfa2a7eb3a - std::sys::unix::thread::Thread::new::thread_start::h276e6ca033938925
                               at src/libstd/sys/unix/thread.rs:87
  24:     0x7fbb01fa5fa3 - start_thread
  25:     0x7fbb01ebc4cf - clone
  26:                0x0 - <unknown>
thread 'thread '<unnamed><unnamed>' panicked at '' panicked at 'failed to send columns: "SendError(..)"failed to receive base_data, tree_data for tree_c: RecvError', ', /root/rust-fil-proofs/storage-proofs-porep/src/stacked/vanilla/proof.rs/root/rust-fil-proofs/storage-proofs-porep/src/stacked/vanilla/proof.rs::545605::3426

stack backtrace:
   0:     0x55cfa2a733e5 - backtrace::backtrace::libunwind::trace::h14d338b30b3ea0a7
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/libunwind.rs:86
   1:     0x55cfa2a733e5 - backtrace::backtrace::trace_unsynchronized::h73ea91d74a3fd67f
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/mod.rs:66
   2:     0x55cfa2a733e5 - std::sys_common::backtrace::_print_fmt::hd42948c952866e12
                               at src/libstd/sys_common/backtrace.rs:78
   3:     0x55cfa2a733e5 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::ha8f928866ff7571e
                               at src/libstd/sys_common/backtrace.rs:59
   4:     0x55cfa2a9fc9c - core::fmt::write::he0c1e5f7426d2718
                               at src/libcore/fmt/mod.rs:1076
   5:     0x55cfa2a6e972 - std::io::Write::write_fmt::hf3afc6cfd57d0033
                               at src/libstd/io/mod.rs:1537
   6:     0x55cfa2a760a0 - std::sys_common::backtrace::_print::hfc0110703f3696fd
                               at src/libstd/sys_common/backtrace.rs:62
   7:     0x55cfa2a760a0 - std::sys_common::backtrace::print::h3f77c6990ddfaa22
                               at src/libstd/sys_common/backtrace.rs:49
   8:     0x55cfa2a760a0 - std::panicking::default_hook::{{closure}}::heae49580a8d62d75
                               at src/libstd/panicking.rs:198
   9:     0x55cfa2a75dec - std::panicking::default_hook::hecc34e3f729e213c
                               at src/libstd/panicking.rs:217
  10:     0x55cfa2a766e3 - std::panicking::rust_panic_with_hook::he82f5d0644692441
                               at src/libstd/panicking.rs:526
  11:     0x55cfa2a762db - rust_begin_unwind
                               at src/libstd/panicking.rs:437
  12:     0x55cfa2a9e381 - core::panicking::panic_fmt::h09c929f06bb87c98
                               at src/libcore/panicking.rs:85
  13:     0x55cfa2a9e1a3 - core::option::expect_none_failed::h188f17af6c9f404b
                               at src/libcore/option.rs:1269
  14:     0x55cfa262a7cf - <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h08bf5d5924c28496
  15:     0x55cfa26caf4e - std::panicking::try::h0f9385e8757dc4e8
  16:     0x55cfa273f202 - <rayon_core::job::HeapJob<BODY> as rayon_core::job::Job>::execute::h624b2c91ddcbd8c5
  17:     0x55cfa29865c1 - rayon_core::registry::WorkerThread::wait_until_cold::hb862e4ba412ff0d5
  18:     0x55cfa2984afb - rayon_core::registry::ThreadBuilder::run::h59cdb2b1b6db1d0f
  19:     0x55cfa29871d1 - std::sys_common::backtrace::__rust_begin_short_backtrace::h4e48a0eefe8b0b12
  20:     0x55cfa29820af - core::ops::function::FnOnce::call_once{{vtable.shim}}::h6d6e99d9b2ecb2a7
  21:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::hd2b3bc04af94a84f
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  22:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h1044417e186e567a
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  23:     0x55cfa2a7eb3a - std::sys::unix::thread::Thread::new::thread_start::h276e6ca033938925
                               at src/libstd/sys/unix/thread.rs:87
  24:     0x7fbb01fa5fa3 - start_thread
  25:     0x7fbb01ebc4cf - clone
  26:                0x0 - <unknown>
stack backtrace:
   0:     0x55cfa2a733e5 - backtrace::backtrace::libunwind::trace::h14d338b30b3ea0a7
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/libunwind.rs:86
   1:     0x55cfa2a733e5 - backtrace::backtrace::trace_unsynchronized::h73ea91d74a3fd67f
                               at /cargo/registry/src/github.com-1ecc6299db9ec823/backtrace-0.3.46/src/backtrace/mod.rs:66
   2:     0x55cfa2a733e5 - std::sys_common::backtrace::_print_fmt::hd42948c952866e12
                               at src/libstd/sys_common/backtrace.rs:78
   3:     0x55cfa2a733e5 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::ha8f928866ff7571e
                               at src/libstd/sys_common/backtrace.rs:59
   4:     0x55cfa2a9fc9c - core::fmt::write::he0c1e5f7426d2718
                               at src/libcore/fmt/mod.rs:1076
   5:     0x55cfa2a6e972 - std::io::Write::write_fmt::hf3afc6cfd57d0033
                               at src/libstd/io/mod.rs:1537
   6:     0x55cfa2a760a0 - std::sys_common::backtrace::_print::hfc0110703f3696fd
                               at src/libstd/sys_common/backtrace.rs:62
   7:     0x55cfa2a760a0 - std::sys_common::backtrace::print::h3f77c6990ddfaa22
                               at src/libstd/sys_common/backtrace.rs:49
   8:     0x55cfa2a760a0 - std::panicking::default_hook::{{closure}}::heae49580a8d62d75
                               at src/libstd/panicking.rs:198
   9:     0x55cfa2a75dec - std::panicking::default_hook::hecc34e3f729e213c
                               at src/libstd/panicking.rs:217
  10:     0x55cfa2a766e3 - std::panicking::rust_panic_with_hook::he82f5d0644692441
                               at src/libstd/panicking.rs:526
  11:     0x55cfa2a762db - rust_begin_unwind
                               at src/libstd/panicking.rs:437
  12:     0x55cfa2a9e381 - core::panicking::panic_fmt::h09c929f06bb87c98
                               at src/libcore/panicking.rs:85
  13:     0x55cfa2a9e1a3 - core::option::expect_none_failed::h188f17af6c9f404b
                               at src/libcore/option.rs:1269
  14:     0x55cfa2633934 - <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h71dff3dd2ce184df
  15:     0x55cfa2726722 - rayon_core::scope::ScopeBase::complete::ha860769376374138
  16:     0x55cfa263ecf9 - <std::panic::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::he164046d6d5d2ae6
  17:     0x55cfa2742530 - <rayon_core::job::StackJob<L,F,R> as rayon_core::job::Job>::execute::h29a44d04a09bb62c
  18:     0x55cfa29865c1 - rayon_core::registry::WorkerThread::wait_until_cold::hb862e4ba412ff0d5
  19:     0x55cfa2984afb - rayon_core::registry::ThreadBuilder::run::h59cdb2b1b6db1d0f
  20:     0x55cfa29871d1 - std::sys_common::backtrace::__rust_begin_short_backtrace::h4e48a0eefe8b0b12
  21:     0x55cfa29820af - core::ops::function::FnOnce::call_once{{vtable.shim}}::h6d6e99d9b2ecb2a7
  22:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::hd2b3bc04af94a84f
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  23:     0x55cfa2a7eb3a - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h1044417e186e567a
                               at /rustc/04488afe34512aa4c33566eb16d8c912a3ae04f9/src/liballoc/boxed.rs:1081
  24:     0x55cfa2a7eb3a - std::sys::unix::thread::Thread::new::thread_start::h276e6ca033938925
                               at src/libstd/sys/unix/thread.rs:87
  25:     0x7fbb01fa5fa3 - start_thread
  26:     0x7fbb01ebc4cf - clone
  27:                0x0 - <unknown>

Initially I thought it has to be a permission error user/group acl thing related to the user that's why I ran it as root

[seccentral]

answer found by chance after a few days:

Fix:

user@host$: apt update
user@host$: apt install -y nvidia-opencl-icd

PLEASE UPDATE YOUR DOCS
https://docs.filecoin.io/get-started/lotus/installation/#software-dependencies

[vmx]

I have opened filecoin-project/filecoin-docs#694 to track the documentation issue. Thanks for pointing directly to the documentation page you had the issue with.