diff --git a/docs/node/README.mdx b/docs/node/README.mdx
index 5c9568afd1..3b571d8690 100644
--- a/docs/node/README.mdx
+++ b/docs/node/README.mdx
@@ -70,7 +70,7 @@ to use.
[system]: ./run-your-node/prerequisites/system-configuration.mdx
[Oasis Node]: ./run-your-node/prerequisites/oasis-node.md
[Run a Validator Node]: ./run-your-node/validator-node.mdx
-[trusted execution environment (TEE)]: ./run-your-node/prerequisites/set-up-trusted-execution-environment-tee.md
+[trusted execution environment (TEE)]: ./run-your-node/prerequisites/set-up-tee.mdx
[Run a ParaTime node]: ./run-your-node/paratime-node.mdx
[ParaTime client node]: ./run-your-node/paratime-node.mdx
[Non-validator node]: ./run-your-node/non-validator-node.mdx
diff --git a/docs/node/run-your-node/README.mdx b/docs/node/run-your-node/README.mdx
index 4a940c285f..e003542744 100644
--- a/docs/node/run-your-node/README.mdx
+++ b/docs/node/run-your-node/README.mdx
@@ -58,9 +58,7 @@ an interface for users or other applications to interact with the blockchain.
Unlike compute nodes, which handle transaction processing and smart contract
execution, client nodes are primarily responsible for tasks such as querying
the blockchain, submitting transactions, and retrieving other data from the
-network. For confidential ParaTimes such as Sapphire and Cipher, there is a
-special mode of client node called [Observer Node] that supports confidential
-smart contact queries.
+network.
- **[Non-Validator Node]** is a type of node in the Oasis Network that does not
participate in the consensus process of validating and proposing new blocks.
@@ -79,18 +77,20 @@ for its strong privacy features, allowing for the execution of confidential
smart contracts and the development of privacy-preserving decentralized
applications (dApps).
-- **[Emerald Client Node]** is a specific type of client node within the Oasis
-Network, designed to interact with the Emerald ParaTime. The Emerald ParaTime is
-an Ethereum-compatible environment on the Oasis Network, allowing developers
-to deploy and manage decentralized applications (dApps) that utilize the
-Ethereum Virtual Machine (EVM).
+- **[ROFL Node]** is a **Sapphire Client Node** that supports the TEE and hosts
+one or more [ROFLs].
+
+- **[Observer Node]** is a special type of client node for confidential
+ParaTimes such as Sapphire and Cipher that supports confidential smart contact
+queries.
[Non-Validator Node]: ./non-validator-node.mdx
[Client Node]: ./paratime-client-node.mdx
[Observer Node]: ./paratime-observer-node.mdx
[Sapphire Client Node]: ./paratime-client-node.mdx
[Cipher Client Node]: ./paratime-client-node.mdx
-[Emerald Client Node]: ./paratime-client-node.mdx
+[ROFL Node]: ./rofl-node.mdx
+[ROFLs]: https://github.com/oasisprotocol/oasis-sdk/blob/main/docs/rofl/README.mdx
## Archive Node
diff --git a/docs/node/run-your-node/advanced/copy-state-from-one-node-to-the-other.md b/docs/node/run-your-node/advanced/copy-state-from-one-node-to-the-other.md
index 66dd4fedf6..c97988634e 100644
--- a/docs/node/run-your-node/advanced/copy-state-from-one-node-to-the-other.md
+++ b/docs/node/run-your-node/advanced/copy-state-from-one-node-to-the-other.md
@@ -73,4 +73,4 @@ Before starting your new Oasis node, do the following:
Finally, you can start your new Oasis node for the first time.
-[TEE-enabled ParaTimes]: ../prerequisites/set-up-trusted-execution-environment-tee.md
+[TEE-enabled ParaTimes]: ../prerequisites/set-up-tee.mdx
diff --git a/docs/node/run-your-node/keymanager-node/README.md b/docs/node/run-your-node/keymanager-node/README.md
index 2be3975da0..a20e653b36 100644
--- a/docs/node/run-your-node/keymanager-node/README.md
+++ b/docs/node/run-your-node/keymanager-node/README.md
@@ -42,7 +42,7 @@ Reading the rest of the [validator node setup instructions](../validator-node.md
### Setting up Trusted Execution Environment (TEE)
-The key manager ParaTime requires the use of a TEE. See the [Set up trusted execution environment](../prerequisites/set-up-trusted-execution-environment-tee.md) doc for instructions on how to set it up before proceeding.
+The key manager ParaTime requires the use of a TEE. See the [Set up trusted execution environment](../prerequisites/set-up-tee.mdx) doc for instructions on how to set it up before proceeding.
## Configuration
diff --git a/docs/node/run-your-node/paratime-node.mdx b/docs/node/run-your-node/paratime-node.mdx
index f21494851a..bfe11aac5d 100644
--- a/docs/node/run-your-node/paratime-node.mdx
+++ b/docs/node/run-your-node/paratime-node.mdx
@@ -261,7 +261,7 @@ Ubuntu 18.04 LTS (and earlier) provide overly-old `bubblewrap`. Follow _Other Di
### Setting up Trusted Execution Environment (TEE)
-If a ParaTime requires the use of a TEE, then make sure you set up TEE as instructed in the [Set up trusted execution environment (TEE)](prerequisites/set-up-trusted-execution-environment-tee.md) doc.
+If a ParaTime requires the use of a TEE, then make sure you set up TEE as instructed in the [Set up trusted execution environment (TEE)](prerequisites/set-up-tee.mdx) doc.
## Configuration
@@ -348,7 +348,7 @@ oasis-node control status -a unix:/node/data/internal.sock
## Troubleshooting
-See the general [Node troubleshooting](troubleshooting.md) and [Set up TEE troubleshooting](prerequisites/set-up-trusted-execution-environment-tee.md#troubleshooting) sections before proceeding with ParaTime node-specific troubleshooting.
+See the general [Node troubleshooting](troubleshooting.md) and [Set up TEE troubleshooting](prerequisites/set-up-tee.mdx#troubleshooting) sections before proceeding with ParaTime node-specific troubleshooting.
### Too Old Bubblewrap Version
@@ -432,7 +432,7 @@ You can use the [attestation tool] (at least version 0.3.4) that also checks if
:::
-[BIOS settings]: prerequisites/set-up-trusted-execution-environment-tee.md#bios-configuration
+[BIOS settings]: prerequisites/set-up-tee.mdx#sgx-bios-configuration
[attestation tool]: https://github.com/oasisprotocol/tools/tree/main/attestation-tool#readme
## See also
diff --git a/docs/node/run-your-node/paratime-observer-node.mdx b/docs/node/run-your-node/paratime-observer-node.mdx
index c3d217f6da..c69965c1f9 100644
--- a/docs/node/run-your-node/paratime-observer-node.mdx
+++ b/docs/node/run-your-node/paratime-observer-node.mdx
@@ -72,7 +72,7 @@ See the [Stake Requirements] page for more details.
[Stake Requirements]: prerequisites/stake-requirements.md
[Run a ParaTime Node]: ../../get-involved/run-node/paratime-node.mdx
[Common Staking Info]: ../../general/manage-tokens/cli/network.md#show-native-token
-[TEE support]: prerequisites/set-up-trusted-execution-environment-tee.md
+[TEE support]: prerequisites/set-up-tee.mdx
[Oasis CLI tools]: ../../general/manage-tokens/cli/account.md#delegate
[`oasis account show`]: ../../general/manage-tokens/cli/account.md#show
diff --git a/docs/node/run-your-node/prerequisites/set-up-trusted-execution-environment-tee.md b/docs/node/run-your-node/prerequisites/set-up-tee.mdx
similarity index 68%
rename from docs/node/run-your-node/prerequisites/set-up-trusted-execution-environment-tee.md
rename to docs/node/run-your-node/prerequisites/set-up-tee.mdx
index c73f6e4838..49471b5bbd 100644
--- a/docs/node/run-your-node/prerequisites/set-up-trusted-execution-environment-tee.md
+++ b/docs/node/run-your-node/prerequisites/set-up-tee.mdx
@@ -1,20 +1,30 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
# Set up Trusted Execution Environment (TEE)
-:::info
+Most Oasis ParaTimes and ROFLs are configured to run in the TEE. There are two
+kinds of Intel TEEs currently in use:
-In case the ParaTime you want to run does not require the use of a TEE (e.g.
-Intel SGX), you can skip setting up a TEE.
+- the [Intel SGX] is required by the ParaTimes and [ROFL SGX][rofl-types] apps.
+- the [Intel TDX] is required by the [ROFL TDX raw and
+ container-based][rofl-types] apps.
-:::
+To run SGX/TDX enclaves:
-If the ParaTime is configured to run in a TEE (currently only [Intel SGX]), you
-must make sure that your system supports running SGX enclaves. This requires
-that your hardware has SGX support, that SGX support is enabled and that the
-additional driver and software components are properly installed and running.
+1. your hardware must have SGX/TDX support,
+2. you have the latest BIOS updates installed,
+3. you have SGX/TDX enabled in your BIOS,
+4. you have the linux kernel, drivers and software components properly installed
+ and running.
[Intel SGX]: https://www.intel.com/content/www/us/en/architecture-and-technology/software-guard-extensions.html
+[Intel TDX]: https://www.intel.com/content/www/us/en/products/docs/accelerator-engines/trust-domain-extensions.html
+[rofl-types]: https://github.com/oasisprotocol/oasis-sdk/blob/main/docs/rofl/app.mdx
+
+## Safeguard Extension (SGX) {#sgx}
-## BIOS Configuration
+### BIOS Configuration {#sgx-bios-configuration}
To enable Intel SGX on your hardware, you also need to configure the BIOS.
First, **update the BIOS to the latest version with the latest microcode** and
@@ -29,13 +39,13 @@ set the BIOS settings as follows:
- **CPU AES**: ENABLE
- **SGX Auto MP Registration**: ENABLE
-## Ensure Clock Synchronization
+### Ensure Clock Synchronization
Due to additional sanity checks within runtime enclaves, you should ensure that
the node's local clock is synchronized (e.g. using NTP). If it is off by more
than half a second you may experience unexpected runtime aborts.
-## Ensure Proper SGX Device Permissions
+### Ensure Proper SGX Device Permissions
Make sure that the user that is running the Oasis Node binary has access to the
SGX device (e.g. `/dev/sgx_enclave`). This can usually be achieved by adding
@@ -58,7 +68,7 @@ runtime startup.
If you are planning to run your node from an interactive session, make sure to
log out for permissions to take effect.
-## AESM Service
+### AESM Service
To allow execution of SGX enclaves, several **Architectural Enclaves (AE)** are
involved (i.e. Launch Enclave, Provisioning Enclave, Provisioning Certificate
@@ -82,9 +92,9 @@ and look for the following line:
SGX_LC: SGX launch config supported = true
```
-## DCAP Attestation
+### DCAP Attestation
-### Ubuntu 22.04+
+#### Ubuntu 22.04+
A convenient way to install the AESM service on Ubuntu 22.04 systems
is to use the Intel's [official Intel SGX APT repository](https://download.01.org/intel-sgx/sgx_repo/).
@@ -109,7 +119,7 @@ The AESM service should be up and running. To confirm that, use:
sudo systemctl status aesmd.service
```
-### Configuring the Quote Provider
+#### Configuring the Quote Provider
The Intel Quote Provider (`libsgx-dcap-default-qpl`) needs to be configured in
order to use either the Intel PCS, the PCCS of your cloud service provider, or
@@ -122,7 +132,7 @@ configuration, via:
sudo systemctl restart aesmd.service
```
-#### Intel PCS
+##### Intel PCS
Using the Intel PCS is the simplest and most generic way, but it may be less
reliable than using your own PCCS. Some cloud providers (see the [following section](#cloud-service-providers-pccs))
@@ -151,7 +161,7 @@ file invalid.**
:::
-#### Cloud Service Provider's PCCS
+##### Cloud Service Provider's PCCS
Some cloud providers require you to use their PCCS.
@@ -172,13 +182,13 @@ use one of the other PCCS options.
[Alibaba Cloud documentation]: https://www.alibabacloud.com/help/en/ecs/user-guide/build-an-sgx-encrypted-computing-environment
[IBM Cloud documentation]: https://cloud.ibm.com/docs/vpc?topic=vpc-about-attestation-sgx-dcap-vpc
-#### Own PCCS
+##### Own PCCS
It is also possible to run PCCS yourself. Follow [official Intel instructions] on how to setup your own PCCS.
[official Intel Instructions]: https://www.intel.com/content/www/us/en/developer/articles/guide/intel-software-guard-extensions-data-center-attestation-primitives-quick-install-guide.html
-### DCAP Attestation Docker
+#### DCAP Attestation Docker
Alternatively, an easy way to install and run the AESM service on a [Docker](https://docs.docker.com/engine/)-enabled
system is to use [our AESM container image](https://github.com/oasisprotocol/oasis-core/pkgs/container/aesmd).
@@ -218,7 +228,7 @@ docker run \
The default Intel Quote Provider config is available in [Intel SGX Github repository](https://github.com/intel/SGXDataCenterAttestationPrimitives/blob/master/QuoteGeneration/qcnl/linux/sgx_default_qcnl.conf).
-### Multi-socket Systems
+#### Multi-socket Systems
Note that platform provisioning for multi-socket systems (e.g. systems with
multiple CPUs) is more complex, especially if one is using a hypervisor and
@@ -230,7 +240,7 @@ provisioning process uses UEFI variables to communicate with the BIOS. In
addition the **SGX Auto MP Registration** BIOS configuration setting should be
set to _enabled_.
-#### Ubuntu 22.04+
+##### Ubuntu 22.04+
To provision and register your multi-socket system you need to install the Intel
SGX Multi-Package Registration Agent Service as follows (assuming Intel's SGX
@@ -246,14 +256,14 @@ enabled SGX Auto MP Registration in the BIOS as mentioned above. You can also
perform re-provisioning by rebooting and setting the **SGX Factory Reset**
option to _enabled_.
-#### VMware vSphere 8.0+
+##### VMware vSphere 8.0+
In order to enable SGX remote attestation on VMware vSphere-based systems,
please follow [the vSphere guide].
[the vSphere guide]: https://docs.vmware.com/en/VMware-vSphere/8.0/vsphere-vcenter-esxi-management/GUID-F16476FD-3B66-462F-B7FB-A456BEDC3549.html
-## Migrate from EPID Attestation to DCAP Attestation
+### Migrate from EPID Attestation to DCAP Attestation
EPID attestation will be discontinued in 2025 and will no longer be available on
any processors. All nodes using EPID attestation must migrate to DCAP
@@ -287,7 +297,7 @@ support DCAP attestation, you'll need to migrate your node to newer hardware.
[Configure the Quote Provider]: #configuring-the-quote-provider
[attestation tool]: #oasis-attestation-tool
-## Check SGX Setup
+### Check SGX Setup
In order to make sure that your SGX setup is working, you can use the
`sgx-detect` tool from the [sgxs-tools](https://lib.rs/crates/sgxs-tools) Rust
@@ -295,7 +305,7 @@ package.
There are no pre-built packages for it, so you will need to compile it yourself.
-### Install Dependencies
+#### Install Dependencies
Make sure you have the following installed on your system:
@@ -304,19 +314,24 @@ Make sure you have the following installed on your system:
* [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config).
* [OpenSSL](https://www.openssl.org) development package.
-On Fedora, you can install all the above with:
+
+
-```
+```shell
sudo dnf install gcc protobuf-compiler pkg-config openssl-devel
```
-On Ubuntu, you can install all the above with:
+
+
-```
+```shell
sudo apt install gcc protobuf-compiler pkg-config libssl-dev
```
-### Install [Rust](https://www.rust-lang.org)
+
+
+
+#### Install [Rust](https://www.rust-lang.org)
We follow [Rust upstream's recommendation](https://www.rust-lang.org/tools/install)
on using [rustup](https://rustup.rs) to install and manage Rust versions.
@@ -343,7 +358,7 @@ the latest stable version of Rust on your system.
:::
-### Build and Install sgxs-tools
+#### Build and Install sgxs-tools
```bash
cargo install sgxs-tools
@@ -398,7 +413,7 @@ _Debug mode_ and _Production mode (Intel whitelisted)_.
In case you encounter errors, see the [list of common SGX installation issues](https://edp.fortanix.com/docs/installation/help/)
for help.
-### Oasis Attestation tool
+#### Oasis Attestation tool
To test if your settings are correct, you may use the [attestation tool]
([binary]) for testing remote attestation against Intel SGX's
@@ -407,176 +422,148 @@ development server.
[attestation tool]: https://github.com/oasisprotocol/tools/tree/main/attestation-tool#readme
[binary]: https://github.com/oasisprotocol/tools/releases
-## Troubleshooting
-
-See [the general troubleshooting section](../troubleshooting.md), before
-proceeding with ParaTime node-specific troubleshooting.
-
-### AESM could not be contacted
-
-If running `sgx-detect --verbose` reports:
-
-```
-🕮 SGX system software > AESM service
-AESM could not be contacted. AESM is needed for launching enclaves and generating attestations.
-
-Please check your AESM installation.
-
-debug: error communicating with aesm
-debug: cause: Connection refused (os error 111)
+## Trust Domain Extensions (TDX) {#tdx}
-More information: https://edp.fortanix.com/docs/installation/help/#aesm-service
-```
-
-Ensure you have completed all the necessary installation steps outlined in
-[DCAP Attestation](#dcap-attestation) section.
-
-### AESM: error 30
-
-If you are encountering the following error message in your node's logs:
-
-```
-failed to initialize TEE: error while getting quote info from AESMD: aesm: error 30
-```
-
-Ensure you have all required SGX driver libraries installed as listed in
-[DCAP Attestation](#dcap-attestation) section.
-
-### Permission Denied When Accessing SGX Kernel Device
+Before proceeding with the TDX installation, **make sure you followed the SGX
+installation steps above and you have a working SGX environment**!
-If running `sgx-detect --verbose` reports:
+### BIOS configuration {#tdx-bios-configuration}
-```
-🕮 SGX system software > SGX kernel device
-Permission denied while opening the SGX device (/dev/sgx/enclave, /dev/sgx or
-/dev/isgx). Make sure you have the necessary permissions to create SGX enclaves.
-If you are running in a container, make sure the device permissions are
-correctly set on the container.
-
-debug: Error opening device: Permission denied (os error 13)
-debug: cause: Permission denied (os error 13)
-```
-
-Ensure you are running the `sgx-detect` tool as `root` via:
-
-```
-sudo $(which sgx-detect) --verbose
-```
-
-### Error Opening SGX Kernel Device
+- **SGX**: ENABLE
+- **SGX memory**: at least 256MB
+- **SGX Auto MP Registration**: ENABLE
+- **Hyper-Threading**: DISABLE
+- **Intel SpeedStep**: DISABLE
+- **SecureBoot**: DISABLE
+- **All Internal Graphics**: DISABLE
+- **Turbo Mode**: DISABLE
+- **CPU AES**: ENABLE
+- **TDX**: ENABLE
+- **Memory Encryption (TME)**: ENABLE
+- **Total Memory Encryption Bypass**: ENABLE
+- **Total Memory Encryption Multi-Tenant (TME-MT)**: ENABLE
+- **TME-MT memory integrity**: DISABLE
+- **TDX Secure Arbitration Mode Loader (SEAM Loader)**: ENABLE
+- **TME-MT/TDX key split**: non-zero value
+- **ACPI S3 and deeper power states**: DISABLED
+
+### Host OS setup
+
+The following section contains summarized instructions for setting up an
+environment for running ROFL node and other TDX services on Ubuntu 24.04 or
+later. Check out the official [Canonical TDX repository] for details.
+
+[Canonical TDX repository]: https://github.com/canonical/tdx
+
+1. Add the following TDX PPAs to your APT sources and the keyring:
+
+ ```shell
+ sudo add-apt-repository ppa:kobuk-team/tdx-release
+ sudo add-apt-repository ppa:kobuk-team/tdx-attestation-release
+ sudo apt update
+ ```
+2. Install the TDX quote generation service and QEMU for running
+ guest virtual machines:
+
+ ```shell
+ sudo apt install tdx-qgs qemu-utils qemu-system-x86
+ ```
+
+3. Install a special TDX-enabled Linux kernel:
+
+ ```shell
+ sudo apt install linux-image-intel
+ ```
+
+3. Disable ACPI S3 (add kernel parameter: `nohibernate`):
+
+ ```
+ sed -i -E "s/GRUB_CMDLINE_LINUX=\"(.*)\"/GRUB_CMDLINE_LINUX=\"\1 nohibernate\"/g" /etc/default/grub
+ update-grub
+ ```
+
+4. Make sure the non-root user running Oasis-node is a member of `sgx`,
+ `sgx_prv` and `kvm` groups on host (access to `/dev/sgx*`, `/dev/kvm*` and
+ `/dev/*vsock*` devices).
+
+5. Reboot your system and select the new `-intel` kernel.
-If running `sgx-detect --verbose` reports:
+:::tip
-```
-🕮 SGX system software > SGX kernel device
-The SGX device (/dev/sgx/enclave, /dev/sgx or /dev/isgx) could not be opened:
-"/dev" mounted with `noexec` option.
+If you don't have access to the grub selector during machine startup, you can
+also detect and set the correct default kernel by executing the script below
+with elevated privileges:
-debug: Error opening device: "/dev" mounted with `noexec` option
-debug: cause: "/dev" mounted with `noexec` option
+```bash
+export KERNEL_RELEASE=$(apt show "linux-image-intel" 2>&1 | gawk 'match($0, /Depends:.* linux-image-([^, ]+)/, a) {print a[1]}')
+if [ -z "${KERNEL_RELEASE}" ]; then
+ echo "ERROR : unable to determine kernel release"
+ exit 1
+fi
+MID=$(awk '/Advanced options for Ubuntu/{print $(NF-1)}' /boot/grub/grub.cfg | cut -d\' -f2)
+KID=$(awk "/with Linux $KERNEL_RELEASE/"'{print $(NF-1)}' /boot/grub/grub.cfg | cut -d\' -f2 | head -n1)
+cat > /etc/default/grub.d/99-tdx-kernel.cfg < Able to launch enclaves > Debug mode
-The enclave could not be launched.
-
-debug: failed to load report enclave
-debug: cause: failed to load report enclave
-debug: cause: Failed to map enclave into memory.
-debug: cause: Operation not permitted (os error 1)
+crw-rw---- 1 root kvm 10, 232 Apr 10 09:56 /dev/kvm
```
-Ensure your system's [`/dev` is NOT mounted with the `noexec` mount option][dev-noexec].
-
-[dev-noexec]: #ensure-dev-is-not-mounted-with-the-noexec-option
-
-### Unable to Launch Enclaves: Invalid argument
-
-If running `sgx-detect --verbose` reports:
+If any of the above is missing, then the following debug tools may help you:
+```shell
+apt install msr-tools
+git clone https://github.com/canonical/tdx && ./tdx/system-report.sh
```
-🕮 SGX system software > Able to launch enclaves > Debug mode
-The enclave could not be launched.
-debug: failed to load report enclave
-debug: cause: Failed to call EINIT.
-debug: cause: I/O ctl failed.
-debug: cause: Invalid argument (os error 22)
```
-
-This may be related to a bug in the Linux kernel when attempting to run enclaves
-on certain hardware configurations. Upgrading the Linux kernel to a version
-equal to or greater than 6.5.0 may solve the issue.
-
-### Unable to Launch Enclaves: Input/output error
-
-If running `sgx-detect --verbose` reports:
-
+### Model specific registers (MSRs)
+MK_TME_ENABLED bit: 1 (expected value: 1)
+SEAM_RR bit: 1 (expected value: 1)
+NUM_TDX_PRIV_KEYS: 20
+SGX_AND_MCHECK_STATUS: 0 (expected value: 0)
+Production platform: Production (expected value: Production)
```
-🕮 SGX system software > Able to launch enclaves > Debug mode
-The enclave could not be launched.
-debug: failed to load report enclave
-debug: cause: Failed to call ECREATE.
-debug: cause: I/O ctl failed.
-debug: cause: Input/output error (os error 5)
+```shell
+sudo rdmsr 0x503
```
-This may be related to a bug in the [`rust-sgx`](https://github.com/fortanix/rust-sgx/issues/565)
-library causing `sgx-detect` (and `attestation-tool`) to fail and report that
-debug enclaves cannot be launched. This is a known issue and is being worked on.
-If the `sgx-detect` is reporting that production enclaves can be launched, you
-can ignore this error when setting up the Oasis node.
-
-### Couldn't find the platform library
-
-If AESMD service log reports:
-
```
-[read_persistent_data ../qe_logic.cpp:1084] Couldn't find the platform library. (null)
-[get_platform_quote_cert_data ../qe_logic.cpp:438] Couldn't load the platform library. (null)
+0 # 0 if SGX in production mode
```
-It may be that the [DCAP quote provider] is misconfigured or the configuration
-file is not a valid JSON file but is malformed. Double-check that its
-configuration file (e.g. `/etc/sgx_default_qcnl.conf`) is correct.
+## Troubleshooting
-[DCAP quote provider]: #configuring-the-quote-provider
+See [the TEE troubleshooting section](../troubleshooting.md#tee) for a list of
+most common errors and solutions.
\ No newline at end of file
diff --git a/docs/node/run-your-node/prerequisites/stake-requirements.md b/docs/node/run-your-node/prerequisites/stake-requirements.md
index 3855c9be43..1ea1d7deac 100644
--- a/docs/node/run-your-node/prerequisites/stake-requirements.md
+++ b/docs/node/run-your-node/prerequisites/stake-requirements.md
@@ -8,15 +8,16 @@ of the Oasis CLI.
:::
-| | Mainnet | Testnet |
-|---------------------------------------------------------|----------------------------------------------------------:|-------------------------------------------------:|
-| Registration of entity[^entity-reg] | 100 ROSE | 100 TEST |
-| Registration of node | 100 ROSE | 100 TEST |
-| Size of the validator set[^validator-set] | 120[^validator-set-mainnet] | 110[^validator-set-testnet] |
-| Run Sapphire or Emerald compute node[^compute-node] | 5,000,000 ROSE
+ member of the validator set[^member] | / |
-| Run Cipher compute node[^compute-node] | member of the validator set[^member] | / |
-| Create ROFL app on Sapphire or Cipher[^rofl-app-create] | 10,000 ROSE[^rofl-app-create-amount] | 100 TEST[^rofl-app-create-amount] |
-| Create a ParaTime | 50,000 ROSE | 10,000 TEST |
+| | Mainnet | Testnet |
+|---------------------------------------------------------|----------------------------------------------------------:|----------------------------------:|
+| Registration of entity[^entity-reg] | 100 ROSE | 100 TEST |
+| Registration of node | 100 ROSE | 100 TEST |
+| Size of the validator set[^validator-set] | 120[^validator-set-mainnet] | 110[^validator-set-testnet] |
+| Run Sapphire or Emerald compute node[^compute-node] | 5,000,000 ROSE
+ member of the validator set[^member] | / |
+| Run Cipher compute node[^compute-node] | member of the validator set[^member] | / |
+| Create ROFL app on Sapphire or Cipher[^rofl-app-create] | 100 ROSE[^rofl-app-create-amount] | 100 TEST[^rofl-app-create-amount] |
+| Create ROFL provider | 100 ROSE | 100 TEST |
+| Create a ParaTime | 50,000 ROSE | 10,000 TEST |
[^entity-reg]: You can fetch the latest entity registration stake requirements
by running [`oasis network show native-token`].
diff --git a/docs/node/run-your-node/prerequisites/system-configuration.mdx b/docs/node/run-your-node/prerequisites/system-configuration.mdx
index f253c2ac9c..6bdfa307c2 100644
--- a/docs/node/run-your-node/prerequisites/system-configuration.mdx
+++ b/docs/node/run-your-node/prerequisites/system-configuration.mdx
@@ -156,19 +156,34 @@ In case your system is using AppArmor and is restricting unprivileged user names
creation, you may need to allow them for Bubblewrap (the sandbox that Oasis Node is
using to execute runtimes).
-You can use the following policy in `/etc/apparmor.d/bwrap`:
+
+
-```
-abi ,
-include
+ You can add the following policy in `/etc/apparmor.d/bwrap`:
-profile bwrap /usr/bin/bwrap flags=(unconfined) {
- userns,
+ ```
+ abi ,
+ include
- # Site-specific additions and overrides. See local/README for details.
- include if exists
-}
-```
+ profile bwrap /usr/bin/bwrap flags=(unconfined) {
+ userns,
+
+ # Site-specific additions and overrides. See local/README for details.
+ include if exists
+ }
+ ```
+
+
+
+
+ Enable the Bubblewrap user namespace restriction policy:
+
+ ```shell
+ sudo ln -s /usr/share/apparmor/extra-profiles/bwrap-userns-restrict /etc/apparmor.d/
+ ```
+
+
+
Then reload AppArmor policies by running:
diff --git a/docs/node/run-your-node/rofl-node.mdx b/docs/node/run-your-node/rofl-node.mdx
new file mode 100644
index 0000000000..ee482ea434
--- /dev/null
+++ b/docs/node/run-your-node/rofl-node.mdx
@@ -0,0 +1,332 @@
+import DocCard from '@theme/DocCard';
+import {findSidebarItem} from '@site/src/sidebarUtils';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# ROFL Node
+
+:::info
+
+These instructions are for setting up a _ROFL node_ which executes ROFLs inside
+a TEE, but otherwise only observes the ParaTime activity and can also submit
+transactions.
+
+:::
+
+This guide will cover setting up your ROFL node for running ROFL apps on the
+Oasis Network. This guide assumes some basic knowledge on the use of command
+line tools.
+
+## Prerequisites
+
+### ParaTime Client Node
+
+ROFL node is a special kind of a **ParaTime Client Node** with a TEE-capable
+hardware in order to securely run ROFLs. First, complete the ParaTime Client
+Node instructions:
+
+1. Set up your [ParaTime Client Node] including support for [TEE ParaTimes].
+2. Add the Sapphire ParaTime to your node ([Mainnet], [Testnet]).
+3. To support [ROFL TDX raw and containers][rofl-types] enable support for the
+ [Intel TDX] on your node.
+
+[ParaTime Client Node]: ./paratime-client-node.mdx#configuration
+[TEE ParaTimes]: ./paratime-client-node.mdx#configuring-tee-paratime-client-node
+[Mainnet]: ../mainnet/README.md#sapphire
+[Testnet]: ../testnet/README.md#sapphire
+[Intel TDX]: ./prerequisites/set-up-tee.mdx#tdx
+[rofl-types]: https://github.com/oasisprotocol/oasis-sdk/blob/main/docs/rofl/app.mdx
+
+### Configure Firewall
+
+Since you will be hosting 3rd party applications on your server, we strongly
+recommend to **configure your firewall for preventing any local area network
+connections from ROFL**.
+
+Using `iptables`, running something like this will prevent Oasis node and other
+processes owned by the `oasis` user accessing LAN on `192.168.0.255` except for
+the gateway:
+
+```
+iptables -A OUTPUT -d 192.168.0.1/32 -m owner --uid-owner $(id -u oasis) -j ACCEPT
+iptables -A OUTPUT -d 192.168.0.0/16 -m owner --uid-owner $(id -u oasis) -j DROP
+```
+
+You can also permanently store the rules above. On Debian-based OSes you can do
+so by running:
+
+```shell
+sudo apt install iptables-persistent
+sudo /etc/init.d/iptables-persistent save
+```
+
+### Fund Your Node
+
+The node will also need to cover any transaction fees required to maintain
+registration of the ROFL node and the apps. First, determine the address of the
+account corresponding to your node:
+
+```shell
+oasis-node identity show-address -a unix:/node/data/internal.sock
+```
+
+```
+oasis1qp6tl30ljsrrqnw2awxxu2mtxk0qxyy2nymtsy90
+```
+
+Then fund this account **on the Sapphire chain** by transferring or depositing
+tokens with the [`oasis account`] command, for example:
+
+```shell
+oasis account transfer 10 oasis1qp6tl30ljsrrqnw2awxxu2mtxk0qxyy2nymtsy90
+```
+
+If you are just testing out your node on Sapphire Testnet, you can also request
+some TEST from the [Testnet Faucet].
+
+[`oasis account`]: https://github.com/oasisprotocol/cli/blob/master/docs/account.md
+[Testnet Faucet]: https://faucet.testnet.oasis.io/?paratime=sapphire
+
+
+## Configuration
+
+There are two ways you can host ROFL apps on your ROFL node. The preferred way
+is to join a network of ROFL providers called the **ROFL marketplace** and which
+is also integrated in the [`oasis rofl deploy`] command. Alternatively, you can
+copy ROFL bundle(s) directly to your node and configure each one of them in your
+node configuration file.
+
+[`oasis rofl deploy`]: https://github.com/oasisprotocol/cli/blob/master/docs/rofl.md#deploy
+
+### Hosting via ROFL marketplace
+
+To make your ROFL node accessible through [ROFL marketplace], you will:
+
+1. Create a new [ROFL provider entity](#register-rofl-provider).
+2. [Configure one or more ROFL nodes](#configure-rofl-node-marketplace) to
+ execute ROFL transactions corresponding to that provider and/or machine.
+
+Both steps take place solely on-chain. There is no centralized mechanism or KYC
+process involved at any time.
+
+[ROFL marketplace]: https://github.com/oasisprotocol/oasis-sdk/blob/main/docs/rofl/features/marketplace.mdx
+
+#### Register your ROFL provider {#register-rofl-provider}
+
+To register a new ROFL provider, using the [Oasis CLI] to run the following
+command:
+
+```shell
+oasis rofl provider init
+```
+
+Then edit your `rofl-provider.yaml` and add one or more hosting offers. Now
+obtain your [Node ID] and decide how much resources you are willing to lend out.
+In the example below, we'll offer *small* instances with 4 GiB of memory, 2 CPUs
+and 20 GB of storage. Hourly rent will cost 10 tokens and there can be at most
+50 active instances at a time.
+
+[Node ID]: ./validator-node.mdx#obtain-the-node-id
+
+```yaml title="rofl-provider.yaml"
+network: testnet
+paratime: sapphire
+provider: test:erin
+nodes:
+ - 5MsgQwijUlpH9+0Hbyors5jwmx7tTmKMA4c9leV3prI=
+scheduler_app: rofl1qrqw99h0f7az3hwt2cl7yeew3wtz0fxunu7luyfg
+payment_address: test:erin
+offers:
+ - id: small
+ resources:
+ tee: tdx
+ memory: 4096
+ cpus: 2
+ storage: 20000
+ payment:
+ native:
+ terms:
+ hourly: 10
+ capacity: 50
+```
+
+To register a new provider using the configuration above, run:
+
+```shell
+oasis rofl provider create
+```
+
+The account signing the transaction is now registered as a ROFL provider
+on-chain. In our case, the built-in `test:erin` account which we used for
+signing has address `oasis1qqcd0qyda6gtwdrfcqawv3s8cr2kupzw9v967au6`.
+
+:::info
+
+Registering a new ROFL provider requires depositing
+[100 tokens][stake-requirements] which are returned to you, when you deregister
+it.
+
+:::
+
+[stake-requirements]: ./prerequisites/stake-requirements.md
+
+[Oasis CLI]: https://github.com/oasisprotocol/cli/blob/master/docs/README.md
+
+#### Configure your ROFL node for the marketplace {#configure-rofl-node-marketplace}
+
+1. Download the [latest release of the Scheduler ROFL app][rofl-scheduler]
+ and save it to your ROFL node, for example `/node/rofls/`. This app will
+ listen to ROFL hosting requests, configure any incoming ROFLs and spin them
+ up. It will also listen to ROFL admin request for stopping or restarting the
+ ROFL machine.
+2. Add the Scheduler ROFL app to your Oasis node's `config.yml` inside the
+ `runtime.paths` and configure the Scheduler specifics such as the provider
+ address, acceptable offers and capacities on this node:
+
+ ```yaml title="config.yml"
+ runtime:
+ sgx:
+ loader: /node/bin/oasis-core-runtime-loader
+ paths:
+ - /node/rofls/rofl-scheduler.testnet.orc
+ runtimes:
+ # Sapphire Testnet RONL with SGX
+ - id: "000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c"
+ config:
+ allowed_queries:
+ - all_expensive: true
+ components:
+ - id: rofl.rofl1qrqw99h0f7az3hwt2cl7yeew3wtz0fxunu7luyfg # ROFL scheduler app ID, should not change
+ permissions:
+ - bundle_add
+ - bundle_remove
+ - volume_add
+ - volume_remove
+ config:
+ rofl_scheduler:
+ provider_address: oasis1qqcd0qyda6gtwdrfcqawv3s8cr2kupzw9v967au6 # Your provider address
+ offers:
+ - small # Your offer name(s)
+ capacity:
+ instances: 24
+ memory: 65536
+ cpus: 24
+ storage: 549755813888
+ ```
+3. Restart your Oasis node. After a while, your ROFL node will be ready to
+ accept ROFLs. ROFL app developers can now simply deploy their ROFL to your
+ node by providing the `--provider ` to the [`oasis rofl deploy`]
+ command.
+
+ ```shell
+ oasis rofl deploy --provider oasis1qqcd0qyda6gtwdrfcqawv3s8cr2kupzw9v967au6
+ ```
+
+:::tip Multiple ROFL nodes
+
+If you configured multiple ROFL nodes for a single provider, the machine
+instantiated to execute the ROFL app will be arbitrarily picked depending on
+which ROFL node register transaction appears first one on the chain.
+
+:::
+
+#### Multiple ROFL replicas on a single node {#rofl-app-id-remap}
+
+To avoid collisions of multiple ROFL app replicas sharing the same app ID on the
+same node, the Scheduler will **remap** the ROFL app ID to a unique value on
+each deployment. You can look for the `starting processor` message in
+[your logs](#checking-status) to figure out the remapped value, for example:
+
+```json
+{
+ "app_id":"rofl1qrjtky678pd3uchsdlhqtjugnsvtck3wyg7w5324",
+ "component":"rofl.4bd2d31255ae7e5cec31084cde02fb40640d4d678db111d1c6ba53478f5f2fc2",
+ "level":"info",
+ "module":"runtime/modules/rofl/app",
+ "msg":"starting processor",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "runtime_name":"",
+ "ts":"2025-05-28T10:13:45.279027974Z"
+}
+```
+
+In the example above, the original ROFL app ID `rofl1qrjtky678pd3uchsdlhqtjugnsvtck3wyg7w5324`
+was remapped to `4bd2d31255ae7e5cec31084cde02fb40640d4d678db111d1c6ba53478f5f2fc2`.
+
+[rofl-scheduler]: https://github.com/oasisprotocol/oasis-sdk/releases
+
+### Hosting the ROFL App Bundle Directly
+
+To execute a ROFL app on your node, simply copy it over to your node, for
+example inside the `/node/rofls` folder. Then, put the location of the ORC
+bundle to the `runtime.paths` section of your configuration similarly to
+how other ParaTimes can be enabled on your node:
+
+```yaml title="config.yml"
+runtime:
+ # ... other options omitted ...
+ paths:
+ - /node/rofls/myapp.default.orc
+```
+
+Check that the path to your ROFL app bundle is correct. After starting your
+node, please make sure that the node is fully synchronized with Sapphire.
+
+### Exposing the ports
+
+To expose a specific TCP port of a ROFL app externally, add the following
+configuration to your Oasis node config:
+
+```yaml title="config.yml"
+runtime:
+ runtimes:
+ - id: "000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c"
+ components:
+ - id: rofl.rofl1qpkplp3uq5yage4kunt0ylmulett0arzwcdjvc8u # Your ROFL app ID
+ networking:
+ incoming:
+ - ip: 192.168.0.10
+ protocol: tcp
+ src_port: 443
+ dst_port: 443
+```
+
+In the example above, we exposed a TCP port `443` externally on the IP address
+`192.168.0.10` of our host.
+
+[client node documentation]: https://github.com/oasisprotocol/docs/blob/main/docs/node/run-your-node/paratime-client-node.mdx#configuring-tee-paratime-client-node
+
+## Persistent storage
+
+The encrypted persistent storage of each ROFL app replica lives in
+`/node/data/runtimes/volumes/{random hex value}` folder.
+
+It is generated when a ROFL app is executed for the first time and will remain
+intact even during ROFL upgrades and removal. Beside is also a `descriptor.json`
+that contains information which ROFL app does this volume belong to and other
+metadata.
+
+## Checking status
+
+You can check the logs of any ROFL app by grepping its app ID in your Oasis node
+log file. Since a Scheduler app to manage your ROFLs is also just a ROFL app,
+you can check if there are any issues reported by Scheduler by grepping for
+the its app ID:
+
+```shell
+grep rofl.rofl1qrqw99h0f7az3hwt2cl7yeew3wtz0fxunu7luyfg /node/data/node.log
+```
+
+To extract only the relevant `msg` field you may do:
+
+```shell
+for l in "$(grep rofl.rofl1qrqw99h0f7az3hwt2cl7yeew3wtz0fxunu7luyfg /node/data/node.log)"; do echo "$l" | jq -r '.msg'; done
+```
+
+When exploring the logs, keep in mind that the ROFL app IDs of the
+Scheduler-managed apps [will be remapped](#rofl-app-id-remap).
+
+## See also
+
+
diff --git a/docs/node/run-your-node/troubleshooting.md b/docs/node/run-your-node/troubleshooting.md
index b5ba7b5807..9ba527c859 100644
--- a/docs/node/run-your-node/troubleshooting.md
+++ b/docs/node/run-your-node/troubleshooting.md
@@ -1,8 +1,13 @@
+---
+description: Oasis Node Troubleshooting for Node Operators and Hackers
+---
+
# Troubleshooting
:::info **BEFORE YOU BEGIN TROUBLESHOOTING**
-Before you begin troubleshooting we suggest you check all of the following:
+Before you begin troubleshooting your Oasis Node we suggest you check all of the
+following:
* Check that your current binary version is the latest listed on the Network Parameters page ([Mainnet](../mainnet/README.md), [Testnet](../testnet/README.md))
* Check the version on your localhost using `oasis-node --version`
@@ -25,7 +30,7 @@ Before you begin troubleshooting we suggest you check all of the following:
:::
-## Running a Node
+## Starting a Node
### Invalid Permissions
@@ -84,8 +89,15 @@ no such file or directory
file does not exist
```
-```text
-ts=2019-11-17T03:42:09.778647033Z level=error module=cmd/registry/node caller=node.go:127 msg="failed to load entity" err="file does not exist"
+```json
+{
+ "ts":"2019-11-17T03:42:09.778647033Z",
+ "level":"error",
+ "module":"cmd/registry/node",
+ "caller":"node.go:127",
+ "msg":"failed to load entity",
+ "err":"file does not exist"
+}
```
More often than you'd expect, this error is the result of setting the path incorrectly. You may have left something like `--genesis.file $GENESIS_FILE_PATH` in the command without setting `$GENESIS_FILE_PATH` first, or set the path incorrectly. Check that `echo $GENESIS_FILE_PATH` matches your expectation or provide a path in the command.
@@ -104,3 +116,368 @@ module=cmd/stake caller=stake.go:70 msg="failed to submit transaction" err="rpc
The docs are now updated to show that you need to add `--stake.transaction.fee.gas` and `--stake.transaction.fee.amount` flags when generating your transaction. Note that if you're re-generating a transaction, you will need to increment the `--nonce` flag.
+## Trusted Execution Environment (TEE) {#tee}
+
+### AESM could not be contacted
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > AESM service
+AESM could not be contacted. AESM is needed for launching enclaves and generating attestations.
+
+Please check your AESM installation.
+
+debug: error communicating with aesm
+debug: cause: Connection refused (os error 111)
+
+More information: https://edp.fortanix.com/docs/installation/help/#aesm-service
+```
+
+Ensure you have completed all the necessary installation steps outlined in
+[DCAP Attestation][tee-dcap-attestation] section.
+
+[tee-dcap-attestation]: prerequisites/set-up-tee.mdx#dcap-attestation
+
+### AESM: error 30
+
+If you are encountering the following error message in your node's logs:
+
+```
+failed to initialize TEE: error while getting quote info from AESMD: aesm: error 30
+```
+
+Ensure you have all required SGX driver libraries installed as listed in
+[DCAP Attestation][tee-dcap-attestation] section.
+
+### Permission Denied When Accessing SGX Kernel Device
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > SGX kernel device
+Permission denied while opening the SGX device (/dev/sgx/enclave, /dev/sgx or
+/dev/isgx). Make sure you have the necessary permissions to create SGX enclaves.
+If you are running in a container, make sure the device permissions are
+correctly set on the container.
+
+debug: Error opening device: Permission denied (os error 13)
+debug: cause: Permission denied (os error 13)
+```
+
+Ensure you are running the `sgx-detect` tool as `root` via:
+
+```
+sudo $(which sgx-detect) --verbose
+```
+
+### Error Opening SGX Kernel Device
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > SGX kernel device
+The SGX device (/dev/sgx/enclave, /dev/sgx or /dev/isgx) could not be opened:
+"/dev" mounted with `noexec` option.
+
+debug: Error opening device: "/dev" mounted with `noexec` option
+debug: cause: "/dev" mounted with `noexec` option
+```
+
+#### Ensure `/dev` is NOT Mounted with the `noexec` Option
+
+Some Linux distributions mount `/dev` with the `noexec` mount option. If that is
+the case, it will prevent the enclave loader from mapping executable pages.
+
+Ensure your `/dev` (i.e. `devtmpfs`) is not mounted with the `noexec` option.
+To check that, use:
+
+```
+cat /proc/mounts | grep devtmpfs
+```
+
+To temporarily remove the `noexec` mount option for `/dev`, run:
+
+```
+sudo mount -o remount,exec /dev
+```
+
+To permanently remove the `noexec` mount option for `/dev`, add the following to
+the system's `/etc/fstab` file:
+
+```
+devtmpfs /dev devtmpfs defaults,exec 0 0
+```
+
+:::info
+
+This is the recommended way to modify mount options for virtual (i.e. API) file
+system as described in [systemd's API File Systems](https://www.freedesktop.org/wiki/Software/systemd/APIFileSystems/)
+documentation.
+
+:::
+
+
+### Unable to Launch Enclaves: Operation not permitted
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > Able to launch enclaves > Debug mode
+The enclave could not be launched.
+
+debug: failed to load report enclave
+debug: cause: failed to load report enclave
+debug: cause: Failed to map enclave into memory.
+debug: cause: Operation not permitted (os error 1)
+```
+
+Ensure your system's [`/dev` is NOT mounted with the `noexec` mount option][tee-dev-noexec].
+
+[tee-dev-noexec]: #ensure-dev-is-not-mounted-with-the-noexec-option
+
+### Unable to Launch Enclaves: Invalid argument
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > Able to launch enclaves > Debug mode
+The enclave could not be launched.
+
+debug: failed to load report enclave
+debug: cause: Failed to call EINIT.
+debug: cause: I/O ctl failed.
+debug: cause: Invalid argument (os error 22)
+```
+
+This may be related to a bug in the Linux kernel when attempting to run enclaves
+on certain hardware configurations. Upgrading the Linux kernel to a version
+equal to or greater than 6.5.0 may solve the issue.
+
+### Unable to Launch Enclaves: Input/output error
+
+If running `sgx-detect --verbose` reports:
+
+```
+🕮 SGX system software > Able to launch enclaves > Debug mode
+The enclave could not be launched.
+
+debug: failed to load report enclave
+debug: cause: Failed to call ECREATE.
+debug: cause: I/O ctl failed.
+debug: cause: Input/output error (os error 5)
+```
+
+This may be related to a bug in the [`rust-sgx`](https://github.com/fortanix/rust-sgx/issues/565)
+library causing `sgx-detect` (and `attestation-tool`) to fail and report that
+debug enclaves cannot be launched. This is a known issue and is being worked on.
+If the `sgx-detect` is reporting that production enclaves can be launched, you
+can ignore this error when setting up the Oasis node.
+
+### Couldn't find the platform library
+
+If AESMD service log reports:
+
+```
+[read_persistent_data ../qe_logic.cpp:1084] Couldn't find the platform library. (null)
+[get_platform_quote_cert_data ../qe_logic.cpp:438] Couldn't load the platform library. (null)
+```
+
+It may be that the [DCAP quote provider][tee-dcap-quote-provider] is
+misconfigured or the configuration file is not a valid JSON file but is
+malformed. Double-check that its configuration file (e.g.
+`/etc/sgx_default_qcnl.conf`) is correct.
+
+[tee-dcap-quote-provider]: prerequisites/set-up-tee.mdx#configuring-the-quote-provider
+
+### [QPL] Failed to get quote config. Error code is 0xb011
+
+The following error appears in the the QGS daemon logs leaving ROFL runtime
+inoperable:
+
+```
+qgsd[1412990]: [QPL] Failed to get quote config. Error code is 0xb011
+qgsd[1412990]: [get_platform_quote_cert_data ../td_ql_logic.cpp:302] Error returned from the p_sgx_get_quote_config API. 0xe044
+qgsd[1412990]: tee_att_get_quote_size return 0x11001
+```
+
+This is a known bug, which hasn't been fixed yet at time of writing this section
+https://github.com/intel/SGXDataCenterAttestationPrimitives/issues/450.
+
+The current workaround is to restart the QGS daemon, for example
+`sudo service qgsd restart`.
+
+If you are managing your QGS daemon with Docker compose, you can configure it as
+follows:
+
+```yaml title="docker-compose.yaml"
+ command: ["/opt/intel/tdx-qgs/qgs", "--no-daemon"]
+ entrypoint: ["/bin/bash", "-c", "exec \"$0\" \"$@\" &> >(tee -a /tmp/qgsd.log)"]
+ init: true
+ healthcheck:
+ test: ["CMD", "/bin/bash", "-c", "grep 'Error code is 0xb011' /tmp/qgsd.log && (: > /tmp/qgsd.log && kill -SIGTERM 1 && exit -1) || (: > /tmp/qgsd.log && exit 0)"]
+ interval: 60s
+ timeout: 2s
+ retries: 0
+```
+
+### [QPL] No certificate data for this platform.
+
+The following error is reported on a multi-CPU systems if the user forgot to
+install and configure MPA:
+
+```
+May 09 13:24:16 oasis-node-1 qgsd[6732]: call tee_att_init_quote
+May 09 13:24:16 oasis-node-1 qgsd[6732]: [QPL] No certificate data for this platform.
+May 09 13:24:16 oasis-node-1 qgsd[6732]: [get_platform_quote_cert_data ../td_ql_logic.cpp:302] Error returned from the p_sgx_get_quote_config API. 0xe011
+May 09 13:24:16 oasis-node-1 qgsd[6732]: tee_att_init_quote return 0x11001
+May 09 13:24:16 oasis-node-1 qgsd[6732]: tee_att_get_quote_size return 0x1100f
+```
+
+Correctly configure your TEE by following the [Set up TEE - Multi-socket
+system][tee-multi-socket-systems] section.
+
+[tee-multi-socket-systems]: ./prerequisites/set-up-tee.mdx#multi-socket-systems
+
+## ROFL
+
+The following errors appear in the ROFL node logs.
+
+### Unknown enclave
+
+This error is reported when the enclave ID of the ROFL provided in the .orc file
+mismatches the currently registered enclave ID of the on-chain ROFL app.
+
+```json
+{
+ "component":"rofl.rofl1qrtetspnld9efpeasxmryl6nw9mgllr0euls3dwn",
+ "err":"call failed: module=rofl code=5: unknown enclave",
+ "level":"error",
+ "module":"runtime/modules/rofl/app/registration",
+ "msg":"failed to refresh registration",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "runtime_name":"",
+ "ts":"2025-02-21T08:10:10.012956311Z"
+}
+```
+
+Update the on-chain enclave ID by running `oasis rofl update` on the machine
+where ROFL is being compiled and deployed.
+
+### Root not found
+
+This error is reported, when the node hasn't been fully synced yet. This
+includes both the consensus and the ParaTime blocks.
+
+```json
+{
+ "component":"rofl.rofl1qrtetspnld9efpeasxmryl6nw9mgllr0euls3dwn",
+ "err":"call failed: root not found",
+ "level":"error",
+ "module":"runtime/modules/rofl/app/registration",
+ "msg":"failed to refresh registration",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "runtime_name":"",
+ "ts":"2025-04-17T05:40:24.305875715Z"
+}
+```
+
+Wait for the node to sync.
+
+### Failed to resize persistent overlay image
+
+The following error is reported on the ROFL node, if there was an error during
+the persistent storage resize operation. Most commonly this happens during ROFL
+upgrade if [persistent storage size][rofl-yaml-storage] was decreased below the
+actually occupied storage.
+
+```json
+{
+ "caller":"host.go:486",
+ "err":"failed to configure process: failed to resize persistent overlay image: qemu-img: Use the --shrink option to perform a shrink operation.\nqemu-img: warning: Shrinking an image will delete all data beyond the shrunken image's end. Before performing such an operation, make sure there is no important data there.\n\nexit status 1",
+ "level":"error",
+ "module":"runtime/host/tdx/qemu",
+ "msg":"failed to start runtime",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "ts":"2025-04-17T09:56:36.321911319Z"
+}
+```
+
+Similarly, if the persistent storage is corrupted in any way, a message like
+this may appear in the logs:
+
+```json
+{
+ "component":"rofl.rofl1qrtetspnld9efpeasxmryl6nw9mgllr0euls3dwn",
+ "level":"info",
+ "module":"runtime/global",
+ "msg":"Error: writing blob: adding layer with blob \"sha256:9f202d637e1bbe0e48c7855d7872fa4ab33af88b61ef10d4cb6dd7caba0e2c8a\"/\"\"/\"sha256:b240b4f256e7bd304b5a1335b4bc73b47ce21aaf31bb1107452a89a101f50054\": readlink /storage/containers/graph/overlay/l: invalid argument",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "runtime_name":"",
+ "ts":"2025-02-25T13:44:47.05176383Z"
+}
+```
+
+ROFL admin user should run `oasis rofl machine restart --wipe-storage` to clear
+persistent storage and recreate the volume of the ROFL app.
+
+Alternatively, you can remove the persistent storage folder manually located at
+`/node/data/runtimes/volumes/` and restart the ROFL app.
+
+:::warning
+
+Both options will permanently delete persistent storage of this ROFL app on the
+ROFL node.
+
+:::
+
+[rofl-yaml-storage]: https://github.com/oasisprotocol/oasis-sdk/blob/main/docs/rofl/features/manifest.md#resources-storage
+
+### Offer not acceptable for this instance
+
+The following error occurs, if your ROFL node Scheduler configuration is not
+configured to accept the offer names of the selected provider.
+
+```json
+{
+ "component":"rofl.rofl1qrqw99h0f7az3hwt2cl7yeew3wtz0fxunu7luyfg",
+ "id":"0000000000000005",
+ "level":"info",
+ "module":"runtime/scheduler/manager",
+ "msg":"offer not acceptable for this instance",
+ "offer":"0000000000000002",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000a6d1e3ebf60dff6c",
+ "runtime_name":"",
+ "ts":"2025-04-25T09:25:57.726444176Z"
+}
+```
+
+Update your node's `runtime.runtimes.sapphire_id.components.scheduler_id.config.rofl_scheduler.offers`
+in your `config.yml` and include the valid offer name.
+
+### Image platform (linux/arm64/v8) does not match the expected platform (linux/amd64)
+
+This error occurs, if the Docker container to be executed inside the ROFL TDX
+was not compiled for the `linux/amd64` platform.
+
+```json
+{
+ "component":"rofl.rofl1qpdzzm4h73gtes04xjn4whan84s3k33l5gx787l2",
+ "level":"info",
+ "module":"runtime/global",
+ "msg":"WARNING: image platform (linux/arm64/v8) does not match the expected platform (linux/amd64)",
+ "provisioner":"tdx-qemu",
+ "runtime_id":"000000000000000000000000000000000000000000000000f80306c9858e7279",
+ "runtime_name":"",
+ "ts":"2025-04-28T06:16:24.20330395Z"
+}
+```
+
+Always compile your Docker container for ROFL with the `--platform linux/amd64`
+parameter or put the `platform: linux/amd64` line inside your `compose.yaml`.
+Then recompile and push the container to the OCI repository.
+
diff --git a/external/cli b/external/cli
index 5224ab89f9..ac1780c61b 160000
--- a/external/cli
+++ b/external/cli
@@ -1 +1 @@
-Subproject commit 5224ab89f9f093aaf56a212097b2f7af257b880d
+Subproject commit ac1780c61bec6b7756f48cf4a03736a272be3486
diff --git a/external/oasis-sdk b/external/oasis-sdk
index 8dc90a8b76..fcb160bdc6 160000
--- a/external/oasis-sdk
+++ b/external/oasis-sdk
@@ -1 +1 @@
-Subproject commit 8dc90a8b76796e02fd2324c92ab08e58486a0d46
+Subproject commit fcb160bdc63431668063f19d089b35d524fee8f3
diff --git a/redirects.ts b/redirects.ts
index 7246c2fee7..a7960c55c8 100644
--- a/redirects.ts
+++ b/redirects.ts
@@ -196,14 +196,17 @@ export const redirectsOptions: Options = {
to: '/node/mainnet/upgrade-log',
from: '/general/run-a-node/upgrade-log', // #200 Restructure docs
},
- {
- to: '/node/run-your-node/archive-node',
- from: '/general/run-a-node/set-up-your-node/run-archive-node', // #200 Restructure docs
- },
{
to: '/node/genesis-doc',
from: '/general/oasis-network/genesis-doc', // #200 Restructure docs
},
+ {
+ to: '/node/run-your-node',
+ from: [
+ '/general/run-a-node/set-up-your-node/run-an-ias-proxy', // #200 Restructure docs
+ '/node/run-your-node/ias-proxy', // #1076 Remove IAS Proxy
+ ],
+ },
{
to: '/node/run-your-node/advanced/remote-signer',
from: [
@@ -212,11 +215,8 @@ export const redirectsOptions: Options = {
],
},
{
- to: '/node/run-your-node',
- from: [
- '/general/run-a-node/set-up-your-node/run-an-ias-proxy', // #200 Restructure docs
- '/node/run-your-node/ias-proxy', // #1076 Remove IAS Proxy
- ],
+ to: '/node/run-your-node/archive-node',
+ from: '/general/run-a-node/set-up-your-node/run-archive-node', // #200 Restructure docs
},
{
to: '/node/run-your-node/non-validator-node',
@@ -230,6 +230,10 @@ export const redirectsOptions: Options = {
to: '/node/run-your-node/paratime-node',
from: '/general/run-a-node/set-up-your-node/run-a-paratime-node', // #200 Restructure docs
},
+ {
+ to: '/node/run-your-node/prerequisites/set-up-tee',
+ from: '/node/run-your-node/prerequisites/set-up-trusted-execution-environment-tee', // #1261 ROFL revamp for marketplace and ROFL node
+ },
{
to: '/node/run-your-node/seed-node',
from: '/general/run-a-node/set-up-your-node/run-seed-node', // #200 Restructure docs
diff --git a/sidebarBuild.ts b/sidebarBuild.ts
index 6f36923f88..b5a4929187 100644
--- a/sidebarBuild.ts
+++ b/sidebarBuild.ts
@@ -53,13 +53,26 @@ export const sidebarBuild: SidebarsConfig = {
'build/rofl/quickstart',
'build/rofl/prerequisites',
'build/rofl/app',
+ 'build/rofl/deployment',
{
- type: 'doc',
- label: 'Deployment',
- id: 'build/rofl/deployment',
+ type: 'category',
+ label: 'Features',
+ link: {
+ type: 'generated-index',
+ description: "Containerized ROFL apps automatically have access to some useful features that\n" +
+ "ease development. This chapter provides an introduction to these features.",
+ slug: '/build/rofl/features',
+ },
+ items: [
+ 'build/rofl/features/marketplace',
+ 'build/rofl/features/secrets',
+ 'build/rofl/features/storage',
+ 'build/rofl/features/manifest',
+ 'build/rofl/features/rest',
+ 'build/rofl/features/testing',
+ ]
},
- 'build/rofl/features',
- 'build/rofl/resources',
+ 'build/rofl/troubleshooting',
],
},
{
diff --git a/sidebarNode.ts b/sidebarNode.ts
index f17947a3fb..3eb86e96a5 100644
--- a/sidebarNode.ts
+++ b/sidebarNode.ts
@@ -70,7 +70,7 @@ export const sidebarNode: SidebarsConfig = {
'node/run-your-node/prerequisites/stake-requirements',
'node/run-your-node/prerequisites/oasis-node',
'node/run-your-node/prerequisites/system-configuration',
- 'node/run-your-node/prerequisites/set-up-trusted-execution-environment-tee',
+ 'node/run-your-node/prerequisites/set-up-tee',
]
},
'node/run-your-node/validator-node',
@@ -89,6 +89,7 @@ export const sidebarNode: SidebarsConfig = {
'node/run-your-node/paratime-observer-node',
],
},
+ 'node/run-your-node/rofl-node',
{
type: 'category',
label: 'Key Manager Node',