Merge #361

361: add go-ipfs DHT interop tests to the CI, enable jsipfs interop tests r=ljedrz a=ljedrz

Initially this PR only added `go-ipfs` interop DHT tests to the Linux CI run. Even if that functionality was a tiny amount of code, I might have gotten lost in the moment and gone a bit too far with extending this PR to also introduce a `jsipfs` interop functionality 😅 - I can split it out if that's desirable.

In any case, the individual commits describe specific changes; the diff seems big, but it's mostly moving stuff around, as almost all of the `go-ipfs` interop code is compatible with `jsipfs`. The `jsipfs` interop tests are not run in the CI yet.

Co-authored-by: ljedrz <[email protected]>

Author: bors[bot]

.github/workflows/ci.yml

@@ -50,6 +50,7 @@ jobs:
       CARGO_INCREMENTAL: 0
       LLVM_CONFIG_PATH: /usr/local/opt/llvm/bin/llvm-config
       NDK_HOME: /usr/local/lib/android/sdk/ndk-bundle
+      GO_IPFS_PATH: /home/runner/work/rust-ipfs/rust-ipfs/go-ipfs/ipfs
       VCPKGRS_DYNAMIC: 1
       DEBUG: ipfsd-ctl:* # enables all debug output from javascript 'debug' lib used by js-ipfsd-ctl
 
@@ -89,6 +90,12 @@ jobs:
       if: matrix.platform.host == 'macos-latest'
       run: brew install llvm openssl
 
+    - name: Download go-ipfs on Linux
+      if: matrix.platform.host == 'ubuntu-latest'
+      run: |
+        curl -L https://github.com/ipfs/go-ipfs/releases/download/v0.6.0/go-ipfs_v0.6.0_linux-amd64.tar.gz --output go_ipfs.tar.gz
+        tar -xf go_ipfs.tar.gz
+
     - name: Install dependencies windows (openssl)
       uses: lukka/[email protected]
       id: windows-runvcpkg
@@ -113,6 +120,10 @@ jobs:
       if: matrix.platform.cross == false
       run: cargo build --locked --workspace --all-targets
 
+    - name: Run interop DHT tests with go-ipfs
+      if: matrix.platform.host == 'ubuntu-latest'
+      run: cargo test --features=test_go_interop dht
+
     - name: Setup conformance tests (non-cross targets)
       if: matrix.platform.cross == false
       run: ./setup.sh

Cargo.toml

@@ -8,7 +8,8 @@ version = "0.1.0"
 [features]
 default = []
 nightly = []
-test_dht_with_go = []
+test_go_interop = []
+test_js_interop = []
 
 [dependencies]
 anyhow = { default-features = false, version = "1.0" }

tests/common/interop.rs

@@ -0,0 +1,126 @@
+#[cfg(any(feature = "test_go_interop", feature = "test_js_interop"))]
+pub use common::ForeignNode;
+
+#[cfg(all(not(feature = "test_go_interop"), not(feature = "test_js_interop")))]
+pub struct ForeignNode;
+
+#[cfg(any(feature = "test_go_interop", feature = "test_js_interop"))]
+pub mod common {
+    use anyhow;
+    use libp2p::{core::PublicKey, Multiaddr, PeerId};
+    use rand::prelude::*;
+    use serde::Deserialize;
+    use std::time::Duration;
+    use std::{
+        env, fs,
+        path::PathBuf,
+        process::{Child, Command, Stdio},
+        thread,
+    };
+
+    // this environment variable should point to the location of the foreign ipfs binary
+    #[cfg(feature = "test_go_interop")]
+    pub const ENV_IPFS_PATH: &str = "GO_IPFS_PATH";
+    #[cfg(feature = "test_js_interop")]
+    pub const ENV_IPFS_PATH: &str = "JS_IPFS_PATH";
+
+    pub struct ForeignNode {
+        pub dir: PathBuf,
+        pub daemon: Child,
+        pub id: PeerId,
+        pub pk: PublicKey,
+        pub addrs: Vec<Multiaddr>,
+    }
+
+    impl ForeignNode {
+        pub fn new() -> ForeignNode {
+            let binary_path = env::vars()
+                .find(|(key, _val)| key == ENV_IPFS_PATH)
+                .unwrap_or_else(|| {
+                    panic!("the {} environment variable was not found", ENV_IPFS_PATH)
+                })
+                .1;
+
+            let mut tmp_dir = env::temp_dir();
+            let mut rng = rand::thread_rng();
+            tmp_dir.push(&format!("ipfs_test_{}", rng.gen::<u64>()));
+            let _ = fs::create_dir(&tmp_dir);
+
+            Command::new(&binary_path)
+                .env("IPFS_PATH", &tmp_dir)
+                .arg("init")
+                .arg("-p")
+                .arg("test")
+                .arg("--bits")
+                .arg("2048")
+                .stdout(Stdio::null())
+                .status()
+                .unwrap();
+
+            let daemon = Command::new(&binary_path)
+                .env("IPFS_PATH", &tmp_dir)
+                .arg("daemon")
+                .stdout(Stdio::null())
+                .spawn()
+                .unwrap();
+
+            // give the daemon a little bit of time to start
+            thread::sleep(Duration::from_secs(1));
+
+            let node_id = Command::new(&binary_path)
+                .env("IPFS_PATH", &tmp_dir)
+                .arg("id")
+                .output()
+                .unwrap()
+                .stdout;
+
+            let node_id_stdout = String::from_utf8_lossy(&node_id);
+            let ForeignNodeId {
+                id,
+                addresses,
+                public_key,
+                ..
+            } = serde_json::de::from_str(&node_id_stdout).unwrap();
+
+            let id = id.parse().unwrap();
+            let pk = PublicKey::from_protobuf_encoding(
+                &base64::decode(public_key.into_bytes()).unwrap(),
+            )
+            .unwrap();
+
+            ForeignNode {
+                dir: tmp_dir,
+                daemon,
+                id,
+                pk,
+                addrs: addresses,
+            }
+        }
+
+        #[allow(dead_code)]
+        pub async fn identity(&self) -> Result<(PublicKey, Vec<Multiaddr>), anyhow::Error> {
+            Ok((self.pk.clone(), self.addrs.clone()))
+        }
+    }
+
+    impl Drop for ForeignNode {
+        fn drop(&mut self) {
+            let _ = self.daemon.kill();
+            let _ = fs::remove_dir_all(&self.dir);
+        }
+    }
+
+    #[derive(Deserialize, Debug)]
+    #[cfg_attr(feature = "test_go_interop", serde(rename_all = "PascalCase"))]
+    #[cfg_attr(feature = "test_js_interop", serde(rename_all = "camelCase"))]
+    pub struct ForeignNodeId {
+        #[cfg_attr(feature = "test_go_interop", serde(rename = "ID"))]
+        pub id: String,
+        pub public_key: String,
+        pub addresses: Vec<Multiaddr>,
+        #[serde(skip)]
+        agent_version: String,
+        #[serde(skip)]
+        protocol_version: String,
+    }
+}

tests/common/mod.rs

@@ -0,0 +1 @@
+pub mod interop;

tests/connect_two.rs

@@ -3,11 +3,20 @@ use libp2p::{multiaddr::Protocol, Multiaddr};
 use std::time::Duration;
 use tokio::time::timeout;
 
+#[cfg(any(feature = "test_go_interop", feature = "test_js_interop"))]
+mod common;
+#[cfg(any(feature = "test_go_interop", feature = "test_js_interop"))]
+use common::interop::ForeignNode;
+
 // Make sure two instances of ipfs can be connected by `Multiaddr`.
 #[tokio::test(max_threads = 1)]
 async fn connect_two_nodes_by_addr() {
     let node_a = Node::new("a").await;
+
+    #[cfg(all(not(feature = "test_go_interop"), not(feature = "test_js_interop")))]
     let node_b = Node::new("b").await;
+    #[cfg(any(feature = "test_go_interop", feature = "test_js_interop"))]
+    let node_b = ForeignNode::new();
 
     let (_, mut b_addrs) = node_b.identity().await.unwrap();
     let b_addr = b_addrs.pop().unwrap();