Debug EC

.github/workflows/run-tests.yml

@@ -36,9 +36,9 @@ jobs:
       neofs_other_expiration_period: ${{ vars.OTHER_EXPIRATION_PERIOD }}
       neofs_testcases_commit: ${{ inputs.neofs_testcases_ref }}
       tests_parallel_level: 3
+      neofs_s3_gw_commit: 'ec'
+      neofs_node_commit: 'bugfix/ec-forwarding'
+      tests_path: 'pytest_tests/tests/s3'
       os: >
-        [{"runner": "ubuntu-latest", "binary": "linux-amd64"},
-        {"runner": "macos-14", "binary": "darwin-arm64", "shard": 0},
-        {"runner": "macos-14", "binary": "darwin-arm64", "shard": 1},
-        {"runner": "macos-14", "binary": "darwin-arm64", "shard": 2}]
+        [{"runner": "ubuntu-latest", "binary": "linux-amd64"}]
     secrets: inherit

neofs-testlib/neofs_testlib/env/env.py

@@ -171,13 +171,14 @@ def generate_neo_go_config(self, wallet: NodeWallet):
         return neo_go_config_path
 
     @allure.step("Deploy inner ring nodes")
-    def deploy_inner_ring_nodes(self, count=1, with_main_chain=False, chain_meta_data=False, sn_validator_url=None):
+    def deploy_inner_ring_nodes(self, count=1, with_main_chain=False, chain_meta_data=False, sn_validator_url=None, allow_ec=True):
         for _ in range(count):
             new_inner_ring_node = InnerRing(
                 self,
                 len(self.inner_ring_nodes) + 1,
                 chain_meta_data=chain_meta_data,
                 sn_validator_url=sn_validator_url,
+                allow_ec=allow_ec
             )
             new_inner_ring_node.generate_network_config()
             self.inner_ring_nodes.append(new_inner_ring_node)
@@ -664,6 +665,7 @@ def deploy(
         request=None,
         chain_meta_data=False,
         sn_validator_url=None,
+        allow_ec=True,
         fschain_endpoints: Optional[list[str]] = None,
         shards_count=2,
         gc_remover_batch_size=200,
@@ -680,6 +682,7 @@ def deploy(
                 with_main_chain=with_main_chain,
                 chain_meta_data=chain_meta_data,
                 sn_validator_url=sn_validator_url,
+                allow_ec=allow_ec
             )
 
             if storage_nodes_count:
@@ -1057,6 +1060,7 @@ def __init__(
         ir_number: int,
         chain_meta_data=False,
         sn_validator_url=None,
+        allow_ec=True,
     ):
         self.neofs_env = neofs_env
         self.ir_number = ir_number
@@ -1082,6 +1086,7 @@ def __init__(
         self.prometheus_address = f"{self.neofs_env.domain}:{NeoFSEnv.get_available_port()}"
         self.ir_state_file = self.neofs_env._generate_temp_file(self.inner_ring_dir, prefix="ir_state_file")
         self.chain_meta_data = chain_meta_data
+        self.allow_ec = allow_ec
         self.sn_validator_url = sn_validator_url
         self.stdout = "Not initialized"
         self.stderr = "Not initialized"
@@ -1178,6 +1183,7 @@ def start(
                 prometheus_address=self.prometheus_address,
                 chain_meta_data=self.chain_meta_data,
                 sn_validator_url=self.sn_validator_url,
+                allow_ec=self.allow_ec,
             )
         logger.info(f"Launching Inner Ring Node:{self}")
         self._launch_process()

neofs-testlib/neofs_testlib/env/templates/ir.yaml

@@ -125,6 +125,7 @@ prometheus:
 
 experimental:
   chain_meta_data: {{ chain_meta_data }}
+  allow_ec: {{ allow_ec }}
 
 {% if sn_validator_url %}
 sn_validator:

neofs-testlib/neofs_testlib/env/templates/s3.yaml

@@ -55,7 +55,7 @@ prometheus:
 
 placement_policy:
   locations:
-    rep-2: "REP 2"
-    select: "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
+    rep-2: "EC 3/1"
+    select: "EC 3/1"
 
 internal_slicer: {{ internal_slicer }}

neofs-testlib/neofs_testlib_tests/pytest/container_policy.json

@@ -1,3 +1,3 @@
 {
-    "rep-1": "REP 1"
+    "rep-1": "EC 2/2"
 }

neofs-testlib/neofs_testlib_tests/pytest/test_env.py

@@ -56,7 +56,7 @@ def s3_creds(neofs_env: NeoFSEnv, zero_fee, wallet: NodeWallet) -> tuple:
         f"{neofs_env.neofs_s3_authmate_path} --debug --with-log --timeout 1m "
         f"issue-secret --wallet {wallet.path} --gate-public-key={gate_public_key} "
         f"--peer {neofs_env.storage_nodes[0].endpoint} --container-friendly-name {bucket} "
-        f"--bearer-rules {s3_bearer_rules} --container-placement-policy 'REP 1' "
+        f"--bearer-rules {s3_bearer_rules} --container-placement-policy 'EC 2/2' "
         f"--container-policy container_policy.json"
     )
     output = _run_with_passwd(cmd, wallet.password)
@@ -143,7 +143,7 @@ def test_gateways_put_get(neofs_env: NeoFSEnv, wallet: NodeWallet, zero_fee, gw_
     result = cli.container.create(
         rpc_endpoint=neofs_env.sn_rpc,
         wallet=wallet.path,
-        policy="REP 1 IN X CBF 1 SELECT 1 FROM * AS X",
+        policy="EC 2/2 IN X CBF 1 SELECT 1 FROM * AS X",
         basic_acl="0FBFBFFF",
         await_mode=True,
     )

pytest_tests/data/container_policy.json

@@ -1,5 +1,5 @@
 {
-    "rep-1": "REP 1",
-    "rep-3": "REP 3",
-    "complex": "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
+    "rep-1": "EC 1/1",
+    "rep-3": "EC 3/1",
+    "complex": "EC 2/2"
 }

pytest_tests/data/policy.json

@@ -1,4 +1,4 @@
 {
-    "rep-3": "REP 3",
-    "complex": "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
+    "rep-3": "EC 3/1",
+    "complex": "EC 3/1"
 }

pytest_tests/lib/helpers/container.py

@@ -18,7 +18,7 @@
 logger = logging.getLogger("NeoLogger")
 
 DEFAULT_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 4 FROM * AS X"
-SINGLE_PLACEMENT_RULE = "REP 1 IN X CBF 1 SELECT 4 FROM * AS X"
+SINGLE_PLACEMENT_RULE = "EC 2/2 IN X CBF 1 SELECT 4 FROM * AS X"
 REP_2_FOR_3_NODES_PLACEMENT_RULE = "REP 2 IN X CBF 1 SELECT 3 FROM * AS X"
 
 

pytest_tests/lib/helpers/utility.py

@@ -63,7 +63,7 @@ def placement_policy_from_container(container_info: str) -> str:
         attribute: Timestamp=1656340345 (2022-06-27 17:32:25 +0300 MSK)
         nonce: 1c511e88-efd7-4004-8dbf-14391a5d375a
         placement policy:
-        REP 1 IN LOC_PLACE
+        EC 2/2 IN LOC_PLACE
         CBF 1
         SELECT 1 FROM LOC_SW AS LOC_PLACE
         FILTER Country EQ Sweden AS LOC_SW

pytest_tests/lib/s3/s3_base.py

@@ -153,7 +153,7 @@ def init_s3_credentials(
         f"{neofs_env.neofs_s3_authmate_path} --debug --with-log --timeout 1m "
         f"issue-secret --wallet {wallet.path} --gate-public-key={gate_public_key} "
         f"--peer {neofs_env.storage_nodes[0].endpoint} --container-friendly-name {bucket} "
-        f"--bearer-rules {s3_bearer_rules} --container-placement-policy 'REP 1' "
+        f"--bearer-rules {s3_bearer_rules} --container-placement-policy 'EC 2/2' "
         f"--container-policy {policy}"
     )
 

pytest_tests/tests/conftest.py

@@ -38,6 +38,7 @@ def get_or_create_neofs_env(
     with_rest_gw=True,
     chain_meta_data=False,
     sn_validator_url=None,
+    allow_ec=True,
     shards_count=2,
     gc_remover_batch_size=200,
     gc_sleep_interval=STORAGE_GC_TIME,
@@ -56,6 +57,7 @@ def get_or_create_neofs_env(
             request=request,
             chain_meta_data=chain_meta_data,
             sn_validator_url=sn_validator_url,
+            allow_ec=allow_ec,
             shards_count=shards_count,
             gc_remover_batch_size=gc_remover_batch_size,
             gc_sleep_interval=gc_sleep_interval,
@@ -88,6 +90,13 @@ def neofs_env_chain_meta_data(temp_directory, artifacts_directory, request):
     neofs_env = get_or_create_neofs_env(request, with_s3_gw=False, with_rest_gw=False, chain_meta_data=True)
     yield neofs_env
     neofs_env.finalize(request)
+
+
+@pytest.fixture(scope="function")
+def neofs_env_ec(temp_directory, artifacts_directory, request):
+    neofs_env = get_or_create_neofs_env(request, with_s3_gw=False, with_rest_gw=False, allow_ec=True)
+    yield neofs_env
+    neofs_env.finalize(request)
 
 
 @pytest.fixture()

pytest_tests/tests/container/test_container.py

@@ -13,13 +13,15 @@
 )
 from helpers.file_helper import generate_file
 from helpers.grpc_responses import CONTAINER_DELETION_TIMED_OUT, NOT_CONTAINER_OWNER
-from helpers.neofs_verbs import get_object, put_object_to_random_node
+from helpers.neofs_verbs import get_object, put_object_to_random_node, head_object
 from helpers.node_management import wait_all_storage_nodes_returned
 from helpers.utility import placement_policy_from_container
 from helpers.wellknown_acl import PRIVATE_ACL_F, PUBLIC_ACL
 from neofs_env.neofs_env_test_base import TestNeofsBase
 from neofs_testlib.env.env import NeoFSEnv, NodeWallet, StorageNode
+import logging
 
+logger = logging.getLogger("NeoLogger")
 
 def object_should_be_gc_marked(neofs_env: NeoFSEnv, node: StorageNode, cid: str, oid: str):
     response = neofs_env.neofs_cli(node.cli_config).control.object_status(
@@ -86,6 +88,69 @@ def test_container_creation(self, default_wallet, name):
             delete_container(wallet.path, cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
             self.tick_epochs_and_wait(1)
             wait_for_container_deletion(wallet.path, cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
+
+    @pytest.mark.sanity
+    @allure.title("User can create container with EC policy")
+    def test_ec_container_creation(self, default_wallet):
+        wallet = default_wallet
+
+        placement_rule = "EC 2/2"
+        cid = create_container(
+            wallet.path,
+            rule=placement_rule,
+            name="ec-container",
+            shell=self.shell,
+            endpoint=self.neofs_env.sn_rpc,
+        )
+
+        containers = list_containers(wallet.path, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
+        assert cid in containers, f"Expected container {cid} in containers: {containers}"
+
+        container_info: str = get_container(
+            wallet.path,
+            cid,
+            json_mode=False,
+            shell=self.shell,
+            endpoint=self.neofs_env.sn_rpc,
+        )
+        container_info = container_info.casefold() 
+        logger.info(f"Container info: {container_info=}")
+
+        source_file_path = generate_file(self.neofs_env.get_object_size("simple_object_size"))
+
+        oid = put_object_to_random_node(
+            wallet.path, source_file_path, cid, shell=self.shell, neofs_env=self.neofs_env
+        )
+
+        get_object(
+            default_wallet.path,
+            cid,
+            oid,
+            self.neofs_env.shell,
+            self.neofs_env.sn_rpc,
+        )
+
+        head_object(
+            default_wallet.path,
+            cid,
+            oid,
+            self.neofs_env.shell,
+            self.neofs_env.sn_rpc,
+        )
+
+        node_info = (
+            self.neofs_env.neofs_cli(self.neofs_env.storage_nodes[0].cli_config)
+            .netmap.nodeinfo(
+                rpc_endpoint=self.neofs_env.sn_rpc,
+                wallet=self.neofs_env.storage_nodes[0].wallet.path,
+            )
+            .stdout.strip()
+        )
+
+        # with allure.step("Delete container and check it was deleted"):
+        #     delete_container(wallet.path, cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
+        #     self.tick_epochs_and_wait(1)
+        #     wait_for_container_deletion(wallet.path, cid, shell=self.shell, endpoint=self.neofs_env.sn_rpc)
 
     @allure.title("Not owner and not trusted party can NOT delete container")
     def test_only_owner_can_delete_container(self, not_owner_wallet: NodeWallet, default_wallet: str):

pytest_tests/tests/metrics/test_sn_metrics.py

@@ -62,7 +62,7 @@ def test_sn_ir_metrics(neofs_env_single_sn: NeoFSEnv, default_wallet: NodeWallet
     ir = neofs_env_single_sn.inner_ring_nodes[0]
 
     cid = create_container(
-        default_wallet.path, shell=neofs_env_single_sn.shell, endpoint=neofs_env_single_sn.sn_rpc, rule="REP 1"
+        default_wallet.path, shell=neofs_env_single_sn.shell, endpoint=neofs_env_single_sn.sn_rpc, rule="EC 2/2"
     )
 
     file_path = generate_file(simple_object_size)
@@ -390,7 +390,7 @@ def test_rest_gw_metrics(neofs_env_single_sn: NeoFSEnv, default_wallet: NodeWall
     cid = create_container_rest_gw(
         gw_endpoint,
         "rest_gw_container",
-        "REP 1",
+        "EC 2/2",
         PUBLIC_ACL,
         session_token,
         signature,

pytest_tests/tests/network/test_deployment.py

@@ -180,7 +180,7 @@ def test_4_ir_node_deployment_with_main_chain(neofs_env_4_ir_with_mainchain: Neo
         new_wallet = create_wallet_with_money(neofs_env)
         cid = create_container(
             new_wallet.path,
-            rule="REP 1",
+            rule="EC 2/2",
             shell=neofs_env.shell,
             endpoint=neofs_env.sn_rpc,
         )
@@ -200,7 +200,7 @@ def test_7_ir_node_deployment_with_main_chain(neofs_env_7_ir_with_mainchain: Neo
         new_wallet = create_wallet_with_money(neofs_env)
         cid = create_container(
             new_wallet.path,
-            rule="REP 1",
+            rule="EC 2/2",
             shell=neofs_env.shell,
             endpoint=neofs_env.sn_rpc,
         )

pytest_tests/tests/network/test_node_management.py

@@ -46,7 +46,7 @@ def create_container_and_pick_node(
         self, neofs_env: NeoFSEnv, default_wallet: NodeWallet
     ) -> Tuple[str, StorageNode]:
         file_path = generate_file(neofs_env.get_object_size("simple_object_size"))
-        placement_rule = "REP 1 IN X CBF 1 SELECT 1 FROM * AS X"
+        placement_rule = "EC 2/2 IN X CBF 1 SELECT 1 FROM * AS X"
         endpoint = neofs_env.sn_rpc
 
         cid = create_container(
@@ -194,8 +194,8 @@ def test_add_nodes(
             ("REP 2 IN X CBF 2 SELECT 2 FROM * AS X", 2),
             ("REP 2 IN X CBF 1 SELECT 2 FROM * AS X", 2),
             ("REP 3 IN X CBF 1 SELECT 3 FROM * AS X", 3),
-            ("REP 1 IN X CBF 1 SELECT 1 FROM * AS X", 1),
-            ("REP 1 IN X CBF 2 SELECT 1 FROM * AS X", 1),
+            ("EC 2/2 IN X CBF 1 SELECT 1 FROM * AS X", 1),
+            ("EC 2/2 IN X CBF 2 SELECT 1 FROM * AS X", 1),
             ("REP 4 IN X CBF 1 SELECT 4 FROM * AS X", 4),
             ("REP 2 IN X CBF 1 SELECT 4 FROM * AS X", 2),
         ],
@@ -215,17 +215,17 @@ def test_placement_policy(self, default_wallet, placement_rule, expected_copies)
         [
             ("REP 4 IN X CBF 1 SELECT 4 FROM * AS X", 4, {1, 2, 3, 4}),
             (
-                "REP 1 IN LOC_PLACE CBF 1 SELECT 1 FROM LOC_SW AS LOC_PLACE FILTER Country EQ Sweden AS LOC_SW",
+                "EC 2/2 IN LOC_PLACE CBF 1 SELECT 1 FROM LOC_SW AS LOC_PLACE FILTER Country EQ Sweden AS LOC_SW",
                 1,
                 {3},
             ),
             (
-                "REP 1 CBF 1 SELECT 1 FROM LOC_SPB FILTER 'UN-LOCODE' EQ 'RU LED' AS LOC_SPB",
+                "EC 2/2 CBF 1 SELECT 1 FROM LOC_SPB FILTER 'UN-LOCODE' EQ 'RU LED' AS LOC_SPB",
                 1,
                 {2},
             ),
             (
-                "REP 1 IN LOC_SPB_PLACE REP 1 IN LOC_MSK_PLACE CBF 1 SELECT 1 FROM LOC_SPB AS LOC_SPB_PLACE "
+                "EC 2/2 IN LOC_SPB_PLACE EC 2/2 IN LOC_MSK_PLACE CBF 1 SELECT 1 FROM LOC_SPB AS LOC_SPB_PLACE "
                 "SELECT 1 FROM LOC_MSK AS LOC_MSK_PLACE "
                 "FILTER 'UN-LOCODE' EQ 'RU LED' AS LOC_SPB FILTER 'UN-LOCODE' EQ 'RU MOW' AS LOC_MSK",
                 2,
@@ -237,7 +237,7 @@ def test_placement_policy(self, default_wallet, placement_rule, expected_copies)
                 {1, 2, 3, 4},
             ),
             (
-                "REP 1 CBF 1 SELECT 1 FROM LOC_SPB "
+                "EC 2/2 CBF 1 SELECT 1 FROM LOC_SPB "
                 "FILTER 'UN-LOCODE' NE 'RU MOW' AND 'UN-LOCODE' NE 'SE STO' AND 'UN-LOCODE' NE 'FI HEL' AS LOC_SPB",
                 1,
                 {2},

pytest_tests/tests/object/test_object_lock.py

@@ -714,7 +714,7 @@ def test_locked_object_removal_from_not_owner_node(self, default_wallet: NodeWal
                 wallet.path,
                 shell=self.shell,
                 endpoint=self.neofs_env.sn_rpc,
-                rule="REP 1 CBF 1",
+                rule="EC 2/2 CBF 1",
                 basic_acl=PUBLIC_ACL,
             )