From 761c4808fcc8ee17de1ebee99d871999b70b3424 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 24 Mar 2021 11:08:08 +0000 Subject: [PATCH 01/55] P2P: added libp2p-minimal to build process, example code and run task. --- .project | 6 ++ build.gradle | 37 ++++++++-- src/main/resources/log4j2.properties | 17 +++++ .../minima/system/network/base/P2PStart.java | 70 +++++++++++++++++++ 4 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 src/main/resources/log4j2.properties create mode 100644 src/org/minima/system/network/base/P2PStart.java diff --git a/.project b/.project index d95b6823f..e6965f6f9 100644 --- a/.project +++ b/.project @@ -10,9 +10,15 @@ + + org.eclipse.buildship.core.gradleprojectbuilder + + + org.eclipse.jdt.core.javanature + org.eclipse.buildship.core.gradleprojectnature diff --git a/build.gradle b/build.gradle index 7080be4a1..713b30d9e 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,9 @@ plugins { // Apply the java plugin to add support for Java id 'java' + // Support for kotlin (used by libraries) + id 'org.jetbrains.kotlin.jvm' version '1.4.20' + // Apply the application plugin to add support for building a CLI application. id 'application' @@ -26,6 +29,9 @@ repositories { // Use jcenter for resolving dependencies. // You can declare any Maven/Ivy/file repository here. jcenter() + + maven { url "https://dl.cloudsmith.io/public/libp2p/jvm-libp2p/maven/" } + } dependencies { @@ -38,6 +44,23 @@ dependencies { implementation 'org.mozilla:rhino:1.7R4' implementation 'com.jcraft:jsch:0.1.55' + + // build and install locally for Apple Silicon + implementation 'com.google.protobuf:protobuf-java:3.11.0' + implementation("org.bouncycastle:bcprov-jdk15on:1.62") + implementation("org.bouncycastle:bcpkix-jdk15on:1.62") + implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0-M1") + + implementation 'io.libp2p:jvm-libp2p-minimal:0.8.0-RELEASE' + // implementation 'io.libp2p:jvm-libp2p-minimal' +// compile files('libs/jvm-libp2p-minimal-0.8.0-RELEASE.jar') + + api("io.netty:netty-all:4.1.36.Final") + + implementation("org.apache.logging.log4j:log4j-api:2.11.2") + implementation("org.apache.logging.log4j:log4j-core:2.11.2") + } sourceSets { @@ -66,11 +89,17 @@ application { //mainClass.set("org.minima.Start") // new syntax for gradle 6.7+, plugins not compatible yet } -jar { - manifest { - attributes 'Main-Class': 'org.minima.Start' - } +// alternative main methods - see https://stackoverflow.com/questions/43937169/gradle-application-plugin-with-multiple-main-classes/46938169 + +// define a task for any alternative static void main method - first arg is gradle task name: ./gradlew runp2p +task(runp2p, dependsOn: 'classes', type: JavaExec) { + main = 'org.minima.system.network.base.P2PStart' // org/minima/system/network/base/P2PStart + classpath = sourceSets.main.runtimeClasspath + // args ''/ip4/127.0.0.1/tcp/63407/ipfs/QmRGduDqyGXEsAGxAw9gM6tZrJbg1NEKSvmqwnjQqKwRVk' // use this line to hardcode args + // systemProperty 'simple.message', 'Hello ' + // systemProperty 'log4j2.debug', 'true' } + jar { manifest { attributes 'Main-Class': 'org.minima.Start' diff --git a/src/main/resources/log4j2.properties b/src/main/resources/log4j2.properties new file mode 100644 index 000000000..6038cd72a --- /dev/null +++ b/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ + +log=/tmp/minima-logstxt + +# Define the root loogger with appender +log4j.rootlogger=DEBUG, stdout, KPLOGFILE + +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + +log4j.appender.KPLOGFILE=org.apache.log4j.FileAppender +log4j.appender.KPLOGFILE.File=${log}/kplog.out +log4j.appender.KPLOGFILE.layout=org.apache.log4j.PatternLayout +log4j.appender.KPLOGFILE.layout.conversionpattern=%m%n + +log4j.logger.kplogger=DEBUG, KPLOGFILE + diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java new file mode 100644 index 000000000..e42d762a6 --- /dev/null +++ b/src/org/minima/system/network/base/P2PStart.java @@ -0,0 +1,70 @@ +package org.minima.system.network.base; + +import io.libp2p.core.Host; +import io.libp2p.core.PeerId; +import io.libp2p.core.crypto.PrivKey; +import io.libp2p.core.dsl.Builder.Defaults; +import io.libp2p.core.dsl.BuilderJKt; +import io.libp2p.core.multiformats.Multiaddr; +import io.libp2p.core.multistream.ProtocolBinding; +import io.libp2p.core.mux.StreamMuxerProtocol; +import io.libp2p.etc.types.ByteArrayExtKt; +import io.libp2p.protocol.Identify; +import io.libp2p.protocol.Ping; +import io.libp2p.security.noise.NoiseXXSecureChannel; +import io.libp2p.transport.tcp.TcpTransport; + +import io.libp2p.core.Host; +import io.libp2p.core.dsl.HostBuilder; +import io.libp2p.core.multiformats.Multiaddr; +import io.libp2p.protocol.Ping; +import io.libp2p.protocol.PingController; + +import java.util.concurrent.ExecutionException; + +// Import log4j classes. +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; + +public class P2PStart { + + + private static final Logger logger = LogManager.getLogger(P2PStart.class); + + public static void main(String[] args) + throws ExecutionException, InterruptedException { + System.out.println("Hello world!"); + + + // Create a libp2p node and configure it + // to accept TCP connections on a random port (0) + Host node = new HostBuilder() + .protocol(new Ping()) + .listen("/ip4/127.0.0.1/tcp/0") + .build(); + + // start listening + node.start().get(); + + System.out.print("Node started and listening on "); + System.out.println(node.listenAddresses()); + + if (args.length > 0) { + Multiaddr address = Multiaddr.fromString(args[0]); + PingController pinger = new Ping().dial( + node, + address + ).getController().get(); + + System.out.println("Sending 5 ping messages to " + address.toString()); + for (int i = 1; i <= 5; ++i) { + long latency = pinger.ping().get(); + System.out.println("Ping " + i + ", latency " + latency + "ms"); + } + + node.stop().get(); + } + } + +} + From 8f9dd630918c6790db0cb0b8bda0a70506816905 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 24 Mar 2021 18:05:42 +0000 Subject: [PATCH 02/55] gradle: added eclipse plugin and removed host dependent config files, which should be generated using ./gradlew cleanEclipse eclipse --- .classpath | 27 --------------- .gitignore | 5 ++- .project | 34 ------------------- .settings/org.eclipse.buildship.core.prefs | 2 -- README.md | 32 +++++++++++++++++ build.gradle | 6 ++++ .../resources => resources}/log4j2.properties | 0 7 files changed, 42 insertions(+), 64 deletions(-) delete mode 100644 .classpath delete mode 100644 .project delete mode 100644 .settings/org.eclipse.buildship.core.prefs rename {src/main/resources => resources}/log4j2.properties (100%) diff --git a/.classpath b/.classpath deleted file mode 100644 index 010b89017..000000000 --- a/.classpath +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.gitignore b/.gitignore index 0f0434e0b..197e472e0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,13 @@ # Ignore Gradle build output directory build -# ignore IDE config +# ignore IDE config and use instead the following command to re-generate them correctly: ./gradlew cleanEclipse eclipse .settings/* .vscode/* .settings/org.eclipse.buildship.core.prefs +.settings/org.eclipse.jdt.core.prefs +.classpath +.project /.nb-gradle/ diff --git a/.project b/.project deleted file mode 100644 index e6965f6f9..000000000 --- a/.project +++ /dev/null @@ -1,34 +0,0 @@ - - - minima - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.buildship.core.gradleprojectbuilder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.buildship.core.gradleprojectnature - - - - 0 - - 30 - - org.eclipse.core.resources.regexFilterMatcher - node_modules|.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__ - - - - diff --git a/.settings/org.eclipse.buildship.core.prefs b/.settings/org.eclipse.buildship.core.prefs deleted file mode 100644 index e8895216f..000000000 --- a/.settings/org.eclipse.buildship.core.prefs +++ /dev/null @@ -1,2 +0,0 @@ -connection.project.dir= -eclipse.preferences.version=1 diff --git a/README.md b/README.md index bdb844e3f..f2801339a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,38 @@ You can add -private and all the other parameters to that. Apple Silicon: please use OpenJDK Java 11 (LTS) macOS ARM 64 bit (Zuul version 11.45.27 or later) https://www.azul.com/downloads/zulu-community/?version=java-11-lts&os=macos&architecture=arm-64-bit&package=jdk +### building Minima + +The most reliable way to build Minima is to use gradlew (the gradle wrapper). You do not need to install anything except a JDK, the wrapper will install gradle locally. + +To build using gradlew from the command line, you only need to type: +``` +./gradlew build +``` + +To build a fatjar (all in one jar containing everything needed to run Minima) from the command line, use the following command: +``` +./gradlew shadowJar +``` + +To run the fatjar: +``` +java -jar build/libs/minima-all.jar +``` + +To build from the IDE, you will first need to generate IDE configuration files. This is valid for Eclipse, VS Code, etc. It is better to stop your IDE before running this command. +``` +./gradlew cleanEclipse eclipse +``` + +If you still have project errors, try in Visual Studio Code: View > Command Palette... > Java: Clean Java Language Server Workspace > Restart and delete. + +To run non default main tasks, try: +``` +./gradlew runp2p +``` + + ### Tests You can run the tests directly from your IDE or from command-line. diff --git a/build.gradle b/build.gradle index 713b30d9e..d1905e999 100644 --- a/build.gradle +++ b/build.gradle @@ -19,6 +19,8 @@ plugins { // test coverage id 'jacoco' + // eclipse classpath generation: ./gradlew cleanEclipse eclipse + id 'eclipse' // shadowJar plugin to build fat jar id 'com.github.johnrengelman.shadow' version '6.1.0' @@ -68,6 +70,10 @@ sourceSets { java { srcDirs = ['src'] } + + resources { + srcDirs = ['resources'] + } } test { diff --git a/src/main/resources/log4j2.properties b/resources/log4j2.properties similarity index 100% rename from src/main/resources/log4j2.properties rename to resources/log4j2.properties From 890fcc3fb5e70d08a791f31a149c040a81d33213 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Mon, 29 Mar 2021 09:51:43 +0100 Subject: [PATCH 03/55] WIP: P2PMinimaDiscovery --- .../network/base/P2PMinimaDiscovery.java | 7 ++ .../base/P2PMinimaDiscoveryBinding.java | 9 ++ .../base/P2PMinimaDiscoveryProtocol.java | 118 ++++++++++++++++++ .../minima/system/network/base/P2PStart.java | 37 ++++-- 4 files changed, 161 insertions(+), 10 deletions(-) create mode 100644 src/org/minima/system/network/base/P2PMinimaDiscovery.java create mode 100644 src/org/minima/system/network/base/P2PMinimaDiscoveryBinding.java create mode 100644 src/org/minima/system/network/base/P2PMinimaDiscoveryProtocol.java diff --git a/src/org/minima/system/network/base/P2PMinimaDiscovery.java b/src/org/minima/system/network/base/P2PMinimaDiscovery.java new file mode 100644 index 000000000..76bdb9d26 --- /dev/null +++ b/src/org/minima/system/network/base/P2PMinimaDiscovery.java @@ -0,0 +1,7 @@ +package org.minima.system.network.base; + +public class P2PMinimaDiscovery extends P2PMinimaDiscoveryBinding { + public P2PMinimaDiscovery() { + super(new P2PMinimaDiscoveryProtocol()); + } +} diff --git a/src/org/minima/system/network/base/P2PMinimaDiscoveryBinding.java b/src/org/minima/system/network/base/P2PMinimaDiscoveryBinding.java new file mode 100644 index 000000000..7b64f1883 --- /dev/null +++ b/src/org/minima/system/network/base/P2PMinimaDiscoveryBinding.java @@ -0,0 +1,9 @@ +package org.minima.system.network.base; + +import io.libp2p.core.multistream.StrictProtocolBinding; + +public class P2PMinimaDiscoveryBinding extends StrictProtocolBinding { + public P2PMinimaDiscoveryBinding(P2PMinimaDiscoveryProtocol mdp) { + super("/p2p/minima/1.0.0", mdp); + } +} \ No newline at end of file diff --git a/src/org/minima/system/network/base/P2PMinimaDiscoveryProtocol.java b/src/org/minima/system/network/base/P2PMinimaDiscoveryProtocol.java new file mode 100644 index 000000000..7ad3da60e --- /dev/null +++ b/src/org/minima/system/network/base/P2PMinimaDiscoveryProtocol.java @@ -0,0 +1,118 @@ +package org.minima.system.network.base; + + +import io.libp2p.core.PeerId; +import io.libp2p.core.Stream; +//import io.libp2p.etc.types.toByteBuf; +import io.libp2p.protocol.ProtocolHandler; +import io.libp2p.protocol.ProtocolMessageHandler; +import io.netty.buffer.ByteBuf; + +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.util.concurrent.CompletableFuture; + +interface P2PMinimaDiscoveryProtocolController { + //fun send(message: String) + public void send(String message); +} + +// class Ping : PingBinding(PingProtocol()) + +// open class PingBinding(ping: PingProtocol) : +// StrictProtocolBinding("/ipfs/ping/1.0.0", ping) + +public class P2PMinimaDiscoveryProtocol extends ProtocolHandler { + + public P2PMinimaDiscoveryProtocol() { + super(Long.MAX_VALUE, Long.MAX_VALUE); + //TODO Auto-generated constructor stub + } + + private CompletableFuture onStart(Stream stream) { + // TODO: add a handler and do something + CompletableFuture ready = new CompletableFuture(); + // val handler = MDPHandler(chatCallback, ready) + //stream.pushHandler(handler) + //return ready.thenApply { handler } + return ready; + } + + protected CompletableFuture onStartInitiator(Stream stream) { + return onStart(stream); + } + + protected CompletableFuture onStartResponder(Stream stream) { + return onStart(stream); + } + + class MDPHandler implements ProtocolMessageHandler, P2PMinimaDiscoveryProtocolController { + + private Stream stream; + // private mdpCallback; + + public MDPHandler() { + //todo: add OnChatMessage and ready equivalents in constructor and save as private fields + } + + @Override + public void onActivated(Stream stream) { + this.stream = stream; + //ready.complete(null); + } + + @Override + public void send(String message) { + byte[] data = message.getBytes(Charset.defaultCharset()); + stream.writeAndFlush(data); // does this need to be in a ByteBuffer + } + + @Override + public void fireMessage(Stream arg0, Object arg1) { + // TODO Auto-generated method stub + + } + + @Override + public void onClosed(Stream arg0) { + // TODO Auto-generated method stub + + } + + @Override + public void onException(Throwable arg0) { + // TODO Auto-generated method stub + + } + + @Override + public void onMessage(Stream stream, ByteBuf msg) { + String msgStr = msg.toString(Charset.defaultCharset()); + System.out.println("Received message: " + msgStr); + //mdpCallback(stream.remotePeerId(), msgStr); + } + } + +// open inner class Chatter( +// private val chatCallback: OnChatMessage, +// val ready: CompletableFuture +// ) : ProtocolMessageHandler, ChatController { +// lateinit var stream: Stream + +// override fun onActivated(stream: Stream) { +// this.stream = stream +// ready.complete(null) +// } + +// override fun onMessage(stream: Stream, msg: ByteBuf) { +// val msgStr = msg.toString(Charset.defaultCharset()) +// chatCallback(stream.remotePeerId(), msgStr) +// } + +// override fun send(message: String) { +// val data = message.toByteArray(Charset.defaultCharset()) +// stream.writeAndFlush(data.toByteBuf()) +// } +// } + +} diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index e42d762a6..20af5525c 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -38,31 +38,48 @@ public static void main(String[] args) // Create a libp2p node and configure it // to accept TCP connections on a random port (0) + // Host node = new HostBuilder() + // .protocol(new Ping()) + // .listen("/ip4/127.0.0.1/tcp/0") + // .build(); + Host node = new HostBuilder() - .protocol(new Ping()) + .protocol(new P2PMinimaDiscovery()) .listen("/ip4/127.0.0.1/tcp/0") .build(); // start listening node.start().get(); - System.out.print("Node started and listening on "); + System.out.print("Node started and listening for P2P Minima Discovery Protocol on "); System.out.println(node.listenAddresses()); if (args.length > 0) { + System.out.println("Found args: " + args[0]); Multiaddr address = Multiaddr.fromString(args[0]); - PingController pinger = new Ping().dial( - node, - address + P2PMinimaDiscovery disc = new P2PMinimaDiscovery(); + System.out.println("Created disc, looking up controller..."); + P2PMinimaDiscoveryProtocolController discoverer = disc.dial( + node, + address ).getController().get(); - System.out.println("Sending 5 ping messages to " + address.toString()); - for (int i = 1; i <= 5; ++i) { - long latency = pinger.ping().get(); - System.out.println("Ping " + i + ", latency " + latency + "ms"); - } + // P2PMinimaDiscoveryProtocolController discoverer = new P2PMinimaDiscovery().dial( + // node, + // address + // ).getController().get(); + + System.out.println("Sending a message to the world..."); + discoverer.send("Hello world!"); node.stop().get(); + // System.out.println("Sending 5 ping messages to " + address.toString()); + // for (int i = 1; i <= 5; ++i) { + // long latency = pinger.ping().get(); + // System.out.println("Ping " + i + ", latency " + latency + "ms"); + // } + + // node.stop().get(); } } From 19f9068f715ced29f1728af5df0bf977af9a7b4d Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Tue, 30 Mar 2021 14:23:31 +0100 Subject: [PATCH 04/55] p2p: imported and refactored teku Java classes to interface with libp2p-jvm kt classes. --- .../system/network/base/AsyncRunner.java | 94 +++ .../system/network/base/Cancellable.java | 21 + .../base/ExceptionThrowingConsumer.java | 19 + .../base/ExceptionThrowingFunction.java | 18 + .../base/ExceptionThrowingFutureSupplier.java | 8 + .../base/ExceptionThrowingRunnable.java | 18 + .../base/ExceptionThrowingSupplier.java | 6 + .../system/network/base/FutureUtil.java | 70 ++ .../system/network/base/LibP2PNodeId.java | 37 + .../network/base/LibP2PParamsFactory.java | 154 +++++ .../system/network/base/LimitedMap.java | 59 ++ .../system/network/base/LimitedSet.java | 36 + .../system/network/base/SafeFuture.java | 637 ++++++++++++++++++ .../network/base/gossip/GossipNetwork.java | 33 + .../base/gossip/PreparedGossipMessage.java | 31 + .../gossip/PreparedGossipMessageFactory.java | 23 + .../network/base/gossip/TopicChannel.java | 23 + .../network/base/gossip/TopicHandler.java | 38 ++ .../base/gossip/config/GossipConfig.java | 206 ++++++ .../config/GossipPeerScoringConfig.java | 246 +++++++ .../gossip/config/GossipScoringConfig.java | 167 +++++ .../config/GossipTopicScoringConfig.java | 293 ++++++++ .../config/GossipTopicsScoringConfig.java | 66 ++ .../base/libp2p/gossip/GossipHandler.java | 119 ++++ .../base/libp2p/gossip/GossipTopicFilter.java | 19 + .../libp2p/gossip/GossipWireValidator.java | 46 ++ .../libp2p/gossip/LibP2PGossipNetwork.java | 242 +++++++ .../libp2p/gossip/LibP2PTopicChannel.java | 47 ++ .../libp2p/gossip/PreparedPubsubMessage.java | 63 ++ .../system/network/base/metrics/Counter.java | 35 + .../network/base/metrics/LabelledMetric.java | 33 + .../network/base/metrics/MetricCategory.java | 45 ++ .../network/base/metrics/MetricsSystem.java | 122 ++++ .../network/base/metrics/OperationTimer.java | 45 ++ .../base/metrics/TekuMetricCategory.java | 46 ++ .../system/network/base/peer/NodeId.java | 39 ++ 36 files changed, 3204 insertions(+) create mode 100644 src/org/minima/system/network/base/AsyncRunner.java create mode 100644 src/org/minima/system/network/base/Cancellable.java create mode 100644 src/org/minima/system/network/base/ExceptionThrowingConsumer.java create mode 100644 src/org/minima/system/network/base/ExceptionThrowingFunction.java create mode 100644 src/org/minima/system/network/base/ExceptionThrowingFutureSupplier.java create mode 100644 src/org/minima/system/network/base/ExceptionThrowingRunnable.java create mode 100644 src/org/minima/system/network/base/ExceptionThrowingSupplier.java create mode 100644 src/org/minima/system/network/base/FutureUtil.java create mode 100644 src/org/minima/system/network/base/LibP2PNodeId.java create mode 100644 src/org/minima/system/network/base/LibP2PParamsFactory.java create mode 100644 src/org/minima/system/network/base/LimitedMap.java create mode 100644 src/org/minima/system/network/base/LimitedSet.java create mode 100644 src/org/minima/system/network/base/SafeFuture.java create mode 100644 src/org/minima/system/network/base/gossip/GossipNetwork.java create mode 100644 src/org/minima/system/network/base/gossip/PreparedGossipMessage.java create mode 100644 src/org/minima/system/network/base/gossip/PreparedGossipMessageFactory.java create mode 100644 src/org/minima/system/network/base/gossip/TopicChannel.java create mode 100644 src/org/minima/system/network/base/gossip/TopicHandler.java create mode 100644 src/org/minima/system/network/base/gossip/config/GossipConfig.java create mode 100644 src/org/minima/system/network/base/gossip/config/GossipPeerScoringConfig.java create mode 100644 src/org/minima/system/network/base/gossip/config/GossipScoringConfig.java create mode 100644 src/org/minima/system/network/base/gossip/config/GossipTopicScoringConfig.java create mode 100644 src/org/minima/system/network/base/gossip/config/GossipTopicsScoringConfig.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/GossipHandler.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/GossipTopicFilter.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/GossipWireValidator.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/LibP2PTopicChannel.java create mode 100644 src/org/minima/system/network/base/libp2p/gossip/PreparedPubsubMessage.java create mode 100644 src/org/minima/system/network/base/metrics/Counter.java create mode 100644 src/org/minima/system/network/base/metrics/LabelledMetric.java create mode 100644 src/org/minima/system/network/base/metrics/MetricCategory.java create mode 100644 src/org/minima/system/network/base/metrics/MetricsSystem.java create mode 100644 src/org/minima/system/network/base/metrics/OperationTimer.java create mode 100644 src/org/minima/system/network/base/metrics/TekuMetricCategory.java create mode 100644 src/org/minima/system/network/base/peer/NodeId.java diff --git a/src/org/minima/system/network/base/AsyncRunner.java b/src/org/minima/system/network/base/AsyncRunner.java new file mode 100644 index 000000000..4b95d3e0b --- /dev/null +++ b/src/org/minima/system/network/base/AsyncRunner.java @@ -0,0 +1,94 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import com.google.common.base.Preconditions; +import java.time.Duration; +import java.util.function.Consumer; + +public interface AsyncRunner { + + default SafeFuture runAsync(final ExceptionThrowingRunnable action) { + return runAsync(() -> SafeFuture.fromRunnable(action)); + } + + SafeFuture runAsync(final ExceptionThrowingFutureSupplier action); + + SafeFuture runAfterDelay(ExceptionThrowingFutureSupplier action, final Duration delay); + + default SafeFuture runAfterDelay( + final ExceptionThrowingRunnable action, final Duration delay) { + return runAfterDelay(() -> SafeFuture.fromRunnable(action), delay); + } + + void shutdown(); + + default SafeFuture runAsync(final ExceptionThrowingSupplier action) { + return runAsync(() -> SafeFuture.of(action)); + } + + default SafeFuture getDelayedFuture(final Duration delay) { + return runAfterDelay(() -> SafeFuture.COMPLETE, delay); + } + + /** + * Schedules the recurrent task which will be repeatedly executed with the specified delay. + * + *

The returned instance can be used to cancel the task. Note that {@link Cancellable#cancel()} + * doesn't interrupt already running task. + * + *

Whenever the {@code runnable} throws exception it is notified to the {@code + * exceptionHandler} and the task recurring executions are not interrupted + */ + default Cancellable runWithFixedDelay( + final ExceptionThrowingRunnable runnable, + final Duration delay, + final Consumer exceptionHandler) { + Preconditions.checkNotNull(exceptionHandler); + + Cancellable cancellable = FutureUtil.createCancellable(); + FutureUtil.runWithFixedDelay(this, runnable, cancellable, delay, exceptionHandler); + return cancellable; + } + + /** + * Execute the future supplier until it completes normally up to some maximum number of retries. + * + * @param action The action to run + * @param retryDelay The time to wait before retrying + * @param maxRetries The maximum number of retries. A value of 0 means the action is run only once + * (no retries). + * @param The value returned by the action future + * @return A future that resolves with the first successful result, or else an error if the + * maximum retries are exhausted. + */ + default SafeFuture runWithRetry( + final ExceptionThrowingFutureSupplier action, + final Duration retryDelay, + final int maxRetries) { + + return SafeFuture.of(action) + .exceptionallyCompose( + err -> { + if (maxRetries > 0) { + // Retry after delay, decrementing the remaining available retries + final int remainingRetries = maxRetries - 1; + return runAfterDelay( + () -> runWithRetry(action, retryDelay, remainingRetries), retryDelay); + } else { + return SafeFuture.failedFuture(err); + } + }); + } +} diff --git a/src/org/minima/system/network/base/Cancellable.java b/src/org/minima/system/network/base/Cancellable.java new file mode 100644 index 000000000..9b65a8d7a --- /dev/null +++ b/src/org/minima/system/network/base/Cancellable.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public interface Cancellable { + + void cancel(); + + boolean isCancelled(); +} diff --git a/src/org/minima/system/network/base/ExceptionThrowingConsumer.java b/src/org/minima/system/network/base/ExceptionThrowingConsumer.java new file mode 100644 index 000000000..2cc4ce007 --- /dev/null +++ b/src/org/minima/system/network/base/ExceptionThrowingConsumer.java @@ -0,0 +1,19 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public interface ExceptionThrowingConsumer { + + void accept(V value) throws Throwable; +} diff --git a/src/org/minima/system/network/base/ExceptionThrowingFunction.java b/src/org/minima/system/network/base/ExceptionThrowingFunction.java new file mode 100644 index 000000000..58bee42d1 --- /dev/null +++ b/src/org/minima/system/network/base/ExceptionThrowingFunction.java @@ -0,0 +1,18 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public interface ExceptionThrowingFunction { + O apply(I value) throws Throwable; +} diff --git a/src/org/minima/system/network/base/ExceptionThrowingFutureSupplier.java b/src/org/minima/system/network/base/ExceptionThrowingFutureSupplier.java new file mode 100644 index 000000000..1648b11b3 --- /dev/null +++ b/src/org/minima/system/network/base/ExceptionThrowingFutureSupplier.java @@ -0,0 +1,8 @@ + +package org.minima.system.network.base; + +import java.util.concurrent.CompletionStage; + +public interface ExceptionThrowingFutureSupplier { + CompletionStage get() throws Throwable; +} diff --git a/src/org/minima/system/network/base/ExceptionThrowingRunnable.java b/src/org/minima/system/network/base/ExceptionThrowingRunnable.java new file mode 100644 index 000000000..4ea79c94a --- /dev/null +++ b/src/org/minima/system/network/base/ExceptionThrowingRunnable.java @@ -0,0 +1,18 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public interface ExceptionThrowingRunnable { + void run() throws Throwable; +} diff --git a/src/org/minima/system/network/base/ExceptionThrowingSupplier.java b/src/org/minima/system/network/base/ExceptionThrowingSupplier.java new file mode 100644 index 000000000..b7ac9b693 --- /dev/null +++ b/src/org/minima/system/network/base/ExceptionThrowingSupplier.java @@ -0,0 +1,6 @@ + +package org.minima.system.network.base; + +public interface ExceptionThrowingSupplier { + O get() throws Throwable; +} diff --git a/src/org/minima/system/network/base/FutureUtil.java b/src/org/minima/system/network/base/FutureUtil.java new file mode 100644 index 000000000..c06646e8d --- /dev/null +++ b/src/org/minima/system/network/base/FutureUtil.java @@ -0,0 +1,70 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.time.Duration; +import java.util.concurrent.Future; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FutureUtil { + private static final Logger LOG = LogManager.getLogger(); + + public static void ignoreFuture(final Future future) {} + + static void runWithFixedDelay( + AsyncRunner runner, + ExceptionThrowingRunnable runnable, + Cancellable task, + final Duration duration, + Consumer exceptionHandler) { + + runner + .runAfterDelay( + () -> { + if (!task.isCancelled()) { + try { + runnable.run(); + } catch (Throwable throwable) { + try { + exceptionHandler.accept(throwable); + } catch (Exception e) { + LOG.warn("Exception in exception handler", e); + } + } finally { + runWithFixedDelay(runner, runnable, task, duration, exceptionHandler); + } + } + }, + duration) + .finish(() -> {}, exceptionHandler); + } + + static Cancellable createCancellable() { + return new Cancellable() { + private volatile boolean cancelled; + + @Override + public void cancel() { + cancelled = true; + } + + @Override + public boolean isCancelled() { + return cancelled; + } + }; + } +} diff --git a/src/org/minima/system/network/base/LibP2PNodeId.java b/src/org/minima/system/network/base/LibP2PNodeId.java new file mode 100644 index 000000000..5c52446a6 --- /dev/null +++ b/src/org/minima/system/network/base/LibP2PNodeId.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import io.libp2p.core.PeerId; +//import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; +import org.minima.system.network.base.peer.NodeId; + +public class LibP2PNodeId extends NodeId { + private final PeerId peerId; + + public LibP2PNodeId(final PeerId peerId) { + this.peerId = peerId; + } + + @Override + public byte[] toBytes() { + return peerId.getBytes(); + } + + @Override + public String toBase58() { + return peerId.toBase58(); + } +} diff --git a/src/org/minima/system/network/base/LibP2PParamsFactory.java b/src/org/minima/system/network/base/LibP2PParamsFactory.java new file mode 100644 index 000000000..efcba028e --- /dev/null +++ b/src/org/minima/system/network/base/LibP2PParamsFactory.java @@ -0,0 +1,154 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import io.libp2p.core.PeerId; +import io.libp2p.pubsub.gossip.GossipParams; +import io.libp2p.pubsub.gossip.GossipPeerScoreParams; +import io.libp2p.pubsub.gossip.GossipScoreParams; +import io.libp2p.pubsub.gossip.GossipTopicScoreParams; +import io.libp2p.pubsub.gossip.GossipTopicsScoreParams; +import io.libp2p.pubsub.gossip.builders.GossipPeerScoreParamsBuilder; +import java.util.Map; +import java.util.stream.Collectors; +import kotlin.jvm.functions.Function1; +import org.minima.system.network.base.gossip.config.GossipConfig; +import org.minima.system.network.base.gossip.config.GossipPeerScoringConfig; +import org.minima.system.network.base.gossip.config.GossipScoringConfig; +import org.minima.system.network.base.gossip.config.GossipTopicScoringConfig; +import org.minima.system.network.base.gossip.config.GossipTopicScoringConfig; + +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipConfig; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipPeerScoringConfig; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipScoringConfig; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipTopicScoringConfig; +//import tech.pegasys.teku.networking.p2p.libp2p.LibP2PNodeId; + +public class LibP2PParamsFactory { + public static GossipParams createGossipParams(final GossipConfig gossipConfig) { + return GossipParams.builder() + .D(gossipConfig.getD()) + .DLow(gossipConfig.getDLow()) + .DHigh(gossipConfig.getDHigh()) + .DLazy(gossipConfig.getDLazy()) + // Calculate dScore and dOut based on other params + .DScore(gossipConfig.getD() * 2 / 3) + .DOut(Math.min(gossipConfig.getD() / 2, Math.max(0, gossipConfig.getDLow()) - 1)) + .fanoutTTL(gossipConfig.getFanoutTTL()) + .gossipSize(gossipConfig.getAdvertise()) + .gossipHistoryLength(gossipConfig.getHistory()) + .heartbeatInterval(gossipConfig.getHeartbeatInterval()) + .floodPublish(true) + .seenTTL(gossipConfig.getSeenTTL()) + .maxPublishedMessages(1000) + .maxTopicsPerPublishedMessage(1) + .maxSubscriptions(200) + .maxGraftMessages(200) + .maxPruneMessages(200) + .maxPeersPerPruneMessage(1000) + .maxIHaveLength(5000) + .maxIWantMessageIds(5000) + .build(); + } + + public static GossipScoreParams createGossipScoreParams(final GossipScoringConfig config) { + return GossipScoreParams.builder() + .peerScoreParams(createPeerScoreParams(config.getPeerScoringConfig())) + .topicsScoreParams(createTopicsScoreParams(config)) + .gossipThreshold(config.getGossipThreshold()) + .publishThreshold(config.getPublishThreshold()) + .graylistThreshold(config.getGraylistThreshold()) + .acceptPXThreshold(config.getAcceptPXThreshold()) + .opportunisticGraftThreshold(config.getOpportunisticGraftThreshold()) + .build(); + } + + public static GossipPeerScoreParams createPeerScoreParams(final GossipPeerScoringConfig config) { + final GossipPeerScoreParamsBuilder builder = + GossipPeerScoreParams.builder() + .topicScoreCap(config.getTopicScoreCap()) + .appSpecificWeight(config.getAppSpecificWeight()) + .ipColocationFactorWeight(config.getIpColocationFactorWeight()) + .ipColocationFactorThreshold(config.getIpColocationFactorThreshold()) + .behaviourPenaltyWeight(config.getBehaviourPenaltyWeight()) + .behaviourPenaltyDecay(config.getBehaviourPenaltyDecay()) + .behaviourPenaltyThreshold(config.getBehaviourPenaltyThreshold()) + .decayInterval(config.getDecayInterval()) + .decayToZero(config.getDecayToZero()) + .retainScore(config.getRetainScore()); + + // Configure optional params + config + .getAppSpecificScorer() + .ifPresent( + scorer -> { + final Function1 appSpecificScore = + peerId -> scorer.scorePeer(new LibP2PNodeId(peerId)); + builder.appSpecificScore(appSpecificScore); + }); + + config + .getDirectPeerManager() + .ifPresent( + mgr -> { + final Function1 isDirectPeer = + peerId -> mgr.isDirectPeer(new LibP2PNodeId(peerId)); + builder.isDirect(isDirectPeer); + }); + + config + .getWhitelistManager() + .ifPresent( + mgr -> { + // Ip whitelisting + final Function1 isIpWhitelisted = mgr::isWhitelisted; + builder.ipWhitelisted(isIpWhitelisted); + }); + + return builder.build(); + } + + public static GossipTopicsScoreParams createTopicsScoreParams(final GossipScoringConfig config) { + final GossipTopicScoreParams defaultTopicParams = + createTopicScoreParams(config.getDefaultTopicScoringConfig()); + final Map topicParams = + config.getTopicScoringConfig().entrySet().stream() + .collect( + Collectors.toMap(Map.Entry::getKey, e -> createTopicScoreParams(e.getValue()))); + return new GossipTopicsScoreParams(defaultTopicParams, topicParams); + } + + public static GossipTopicScoreParams createTopicScoreParams( + final GossipTopicScoringConfig config) { + return GossipTopicScoreParams.builder() + .topicWeight(config.getTopicWeight()) + .timeInMeshWeight(config.getTimeInMeshWeight()) + .timeInMeshQuantum(config.getTimeInMeshQuantum()) + .timeInMeshCap(config.getTimeInMeshCap()) + .firstMessageDeliveriesWeight(config.getFirstMessageDeliveriesWeight()) + .firstMessageDeliveriesDecay(config.getFirstMessageDeliveriesDecay()) + .firstMessageDeliveriesCap(config.getFirstMessageDeliveriesCap()) + .meshMessageDeliveriesWeight(config.getMeshMessageDeliveriesWeight()) + .meshMessageDeliveriesDecay(config.getMeshMessageDeliveriesDecay()) + .meshMessageDeliveriesThreshold(config.getMeshMessageDeliveriesThreshold()) + .meshMessageDeliveriesCap(config.getMeshMessageDeliveriesCap()) + .meshMessageDeliveriesActivation(config.getMeshMessageDeliveriesActivation()) + .meshMessageDeliveryWindow(config.getMeshMessageDeliveryWindow()) + .meshFailurePenaltyWeight(config.getMeshFailurePenaltyWeight()) + .meshFailurePenaltyDecay(config.getMeshFailurePenaltyDecay()) + .invalidMessageDeliveriesWeight(config.getInvalidMessageDeliveriesWeight()) + .invalidMessageDeliveriesDecay(config.getInvalidMessageDeliveriesDecay()) + .build(); + } +} diff --git a/src/org/minima/system/network/base/LimitedMap.java b/src/org/minima/system/network/base/LimitedMap.java new file mode 100644 index 000000000..d009a3b2e --- /dev/null +++ b/src/org/minima/system/network/base/LimitedMap.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import com.google.common.cache.CacheBuilder; +import java.util.Map; + + +/** Helper that creates a map with a maximum capacity. */ +public final class LimitedMap { + private LimitedMap() {} + + /** + * Creates a limited map. The returned map is safe for concurrent access and evicts the least + * recently used items. + * + * @param maxSize The maximum number of elements to keep in the map. + * @param The key type of the map. + * @param The value type of the map. + * @return A map that will evict elements when the max size is exceeded. + */ + public static Map create(final int maxSize) { + return defaultBuilder(maxSize).build().asMap(); + } + + /** + * Creates a limited map using soft references for values. The returned map is safe for concurrent + * access and evicts the least recently used items. + * + *

Items may be evicted before maxSize is reached if the garbage collector needs to free up + * memory. + * + * @param maxSize The maximum number of elements to keep in the map. + * @param The key type of the map. + * @param The value type of the map. + * @return A map that will evict elements when the max size is exceeded or when the GC evicts + * them. + */ + public static Map createSoft(final int maxSize) { + return defaultBuilder(maxSize).softValues().build().asMap(); + } + + private static CacheBuilder defaultBuilder(final int maxSize) { + return CacheBuilder.newBuilder().maximumSize(maxSize); + } +} + + diff --git a/src/org/minima/system/network/base/LimitedSet.java b/src/org/minima/system/network/base/LimitedSet.java new file mode 100644 index 000000000..fba78c09d --- /dev/null +++ b/src/org/minima/system/network/base/LimitedSet.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Collections; +import java.util.Set; + +/** Helper that creates a set with a maximum capacity. */ +public final class LimitedSet { + + private LimitedSet() {} + + /** + * Creates a limited set. The returned set is safe for concurrent access and evicts the least + * recently used items. + * + * @param maxSize The maximum number of elements to keep in the set. + * @param The type of object held in the set. + * @return A set that will evict elements when the max size is exceeded. + */ + public static Set create(final int maxSize) { + return Collections.newSetFromMap(LimitedMap.create(maxSize)); + } +} + diff --git a/src/org/minima/system/network/base/SafeFuture.java b/src/org/minima/system/network/base/SafeFuture.java new file mode 100644 index 000000000..b09172fc7 --- /dev/null +++ b/src/org/minima/system/network/base/SafeFuture.java @@ -0,0 +1,637 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.time.Duration; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; + +public class SafeFuture extends CompletableFuture { + + public static SafeFuture COMPLETE = SafeFuture.completedFuture(null); + + public static void reportExceptions(final CompletionStage future) { + future.exceptionally( + error -> { + final Thread currentThread = Thread.currentThread(); + currentThread.getUncaughtExceptionHandler().uncaughtException(currentThread, error); + return null; + }); + } + + public static > Consumer reportExceptions( + final Function action) { + return value -> reportExceptions(action.apply(value)); + } + + public static SafeFuture completedFuture(U value) { + SafeFuture future = new SafeFuture<>(); + future.complete(value); + return future; + } + + public static SafeFuture failedFuture(Throwable ex) { + SafeFuture future = new SafeFuture<>(); + future.completeExceptionally(ex); + return future; + } + + public static SafeFuture of(final CompletionStage stage) { + if (stage instanceof SafeFuture) { + return (SafeFuture) stage; + } + final SafeFuture safeFuture = new SafeFuture<>(); + propagateResult(stage, safeFuture); + return safeFuture; + } + + public static SafeFuture of(final ExceptionThrowingFutureSupplier futureSupplier) { + try { + return SafeFuture.of(futureSupplier.get()); + } catch (Throwable e) { + return SafeFuture.failedFuture(e); + } + } + + public static SafeFuture of(final ExceptionThrowingSupplier supplier) { + try { + return SafeFuture.completedFuture(supplier.get()); + } catch (final Throwable e) { + return SafeFuture.failedFuture(e); + } + } + + /** + * Creates a completed {@link SafeFuture} instance if none of the supplied interruptors are + * completed, else creates an exceptionally completed {@link SafeFuture} instance + * + * @see #orInterrupt(Interruptor...) + */ + public static SafeFuture notInterrupted(Interruptor... interruptors) { + SafeFuture delayedFuture = new SafeFuture<>(); + SafeFuture ret = delayedFuture.orInterrupt(interruptors); + delayedFuture.complete(null); + return ret; + } + + /** + * Creates an {@link Interruptor} instance from the interrupting future and exception supplier for + * the case if interruption is triggered. + * + *

The key feature of {@link Interruptor} and {@link #orInterrupt(Interruptor...)} method is + * that {@code interruptFuture} doesn't hold the reference to dependent futures after they + * complete. It's desired to consider this for long living interrupting futures to avoid memory + * leaks + * + * @param interruptFuture the future which triggers interruption when completes (normally or + * exceptionally) + * @param exceptionSupplier creates a desired exception if interruption is triggered + * @see #notInterrupted(Interruptor...) + * @see #orInterrupt(Interruptor...) + */ + public static Interruptor createInterruptor( + CompletableFuture interruptFuture, Supplier exceptionSupplier) { + return new Interruptor(interruptFuture, exceptionSupplier); + } + + /** + * Repeatedly run the loop until it returns false or completes exceptionally + * + * @param loopBody A supplier for generating futures to be run in succession + * @return A future that will complete when looping terminates + */ + public static SafeFuture asyncDoWhile(ExceptionThrowingFutureSupplier loopBody) { + // Loop while futures complete immediately in order to avoid stack overflow due to recursion + SafeFuture loopFuture = SafeFuture.of(loopBody); + while (loopFuture.isCompletedNormally()) { + if (!loopFuture.join()) { + // Break if the result is false + break; + } + loopFuture = SafeFuture.of(loopBody); + } + + return loopFuture.thenCompose(res -> res ? asyncDoWhile(loopBody) : SafeFuture.COMPLETE); + } + + @SuppressWarnings("FutureReturnValueIgnored") + static void propagateResult(final CompletionStage stage, final SafeFuture safeFuture) { + stage.whenComplete( + (result, error) -> { + if (error != null) { + safeFuture.completeExceptionally(error); + } else { + safeFuture.complete(result); + } + }); + } + + public static SafeFuture fromRunnable(final ExceptionThrowingRunnable action) { + try { + action.run(); + return SafeFuture.COMPLETE; + } catch (Throwable t) { + return SafeFuture.failedFuture(t); + } + } + + public static SafeFuture allOf(final SafeFuture... futures) { + return of(CompletableFuture.allOf(futures)) + .catchAndRethrow(completionException -> addSuppressedErrors(completionException, futures)); + } + + /** + * Adds the {@link Throwable} from each future as a suppressed exception to completionException + * unless it is already set as the cause. + * + *

This ensures that when futures are combined with {@link #allOf(SafeFuture[])} that all + * failures are reported, not just the first one. + * + * @param completionException the exception reported by {@link + * CompletableFuture#allOf(CompletableFuture[])} + * @param futures the futures passed to allOf + */ + @SuppressWarnings("FutureReturnValueIgnored") + public static void addSuppressedErrors( + final Throwable completionException, final SafeFuture[] futures) { + Stream.of(futures) + .forEach( + future -> + future.exceptionally( + error -> { + if (completionException.getCause() != error) { + completionException.addSuppressed(error); + } + return null; + })); + } + + /** + * Returns a new SafeFuture that is completed when all of the given SafeFutures complete + * successfully or completes exceptionally immediately when any of the SafeFutures complete + * exceptionally. The results, if any, of the given SafeFutures are not reflected in the returned + * SafeFuture, but may be obtained by inspecting them individually. If no SafeFutures are + * provided, returns a SafeFuture completed with the value {@code null}. + * + *

Among the applications of this method is to await completion of a set of independent + * SafeFutures before continuing a program, as in: {@code SafeFuture.allOf(c1, c2, c3).join();}. + * + * @param futures the SafeFutures + * @return a new SafeFuture that is completed when all of the given SafeFutures complete + * @throws NullPointerException if the array or any of its elements are {@code null} + */ + public static SafeFuture allOfFailFast(final SafeFuture... futures) { + final SafeFuture complete = new SafeFuture<>(); + Stream.of(futures).forEach(future -> future.finish(() -> {}, complete::completeExceptionally)); + allOf(futures).propagateTo(complete); + return complete; + } + + public static SafeFuture anyOf(final SafeFuture... futures) { + return of(CompletableFuture.anyOf(futures)); + } + + public SafeFuture toVoid() { + return thenAccept(__ -> {}); + } + + public boolean isCompletedNormally() { + return isDone() && !isCompletedExceptionally() && !isCancelled(); + } + + @Override + public SafeFuture newIncompleteFuture() { + return new SafeFuture<>(); + } + + public void reportExceptions() { + reportExceptions(this); + } + + public void finish(final Runnable onSuccess, final Consumer onError) { + finish(result -> onSuccess.run(), onError); + } + + public void propagateTo(final SafeFuture target) { + propagateResult(this, target); + } + + public void propagateToAsync(final SafeFuture target, final AsyncRunner asyncRunner) { + finish( + result -> asyncRunner.runAsync(() -> target.complete(result)).reportExceptions(), + error -> + asyncRunner.runAsync(() -> target.completeExceptionally(error)).reportExceptions()); + } + + /** + * Completes the {@code target} exceptionally if and only if this future is completed + * exceptionally + */ + public void propagateExceptionTo(final SafeFuture target) { + finish(() -> {}, target::completeExceptionally); + } + + /** + * Run final logic on success or error + * + * @param onFinished Task to run when future completes successfully or exceptionally + */ + public void always(final Runnable onFinished) { + finish(res -> onFinished.run(), err -> onFinished.run()); + } + + public SafeFuture alwaysRun(final Runnable action) { + return exceptionallyCompose( + error -> { + action.run(); + return failedFuture(error); + }) + .thenPeek(value -> action.run()); + } + + public void finish(final Consumer onSuccess, final Consumer onError) { + handle( + (result, error) -> { + if (error != null) { + onError.accept(error); + } else { + onSuccess.accept(result); + } + return null; + }) + .reportExceptions(); + } + + public void finish(final Consumer onError) { + handle( + (result, error) -> { + if (error != null) { + onError.accept(error); + } + return null; + }) + .reportExceptions(); + } + + public void finishAsync(final Consumer onError, final Executor executor) { + finishAsync(__ -> {}, onError, executor); + } + + public void finishAsync( + final Runnable onSuccess, final Consumer onError, final Executor executor) { + finishAsync(__ -> onSuccess.run(), onError, executor); + } + + public void finishAsync( + final Consumer onSuccess, final Consumer onError, final Executor executor) { + handleAsync( + (result, error) -> { + if (error != null) { + onError.accept(error); + } else { + onSuccess.accept(result); + } + return null; + }, + executor) + .reportExceptions(); + } + + /** + * Returns a new CompletionStage that, when the provided stage completes exceptionally, is + * executed with the provided stage's exception as the argument to the supplied function. + * Otherwise the returned stage completes successfully with the same value as the provided stage. + * + *

This is the exceptional equivalent to {@link CompletionStage#thenCompose(Function)} + * + * @param errorHandler the function returning a new CompletionStage + * @return the SafeFuture + */ + @SuppressWarnings({"FutureReturnValueIgnored", "MissingOverride"}) + public SafeFuture exceptionallyCompose( + final Function> errorHandler) { + final SafeFuture result = new SafeFuture<>(); + whenComplete( + (value, error) -> { + try { + final CompletionStage nextStep = + error != null ? errorHandler.apply(error) : completedFuture(value); + propagateResult(nextStep, result); + } catch (final Throwable t) { + result.completeExceptionally(t); + } + }); + return result; + } + + /** + * Returns a new CompletionStage that, when the this stage completes exceptionally, executes the + * provided {@code ExceptionThrowingConsumer} with the exception as the argument. The returned + * stage will be exceptionally completed with the same exception if the consumer completes without + * exceptions. If the consumer throws exception then the returned stage will be completed with + * thrown exception. + * + *

This is equivalent to a catch block that performs some action and then either rethrows the + * original exception or throws a new one + * + * @param onError the function to executor when this stage completes exceptionally. + * @return a new SafeFuture which completes with the same successful result as this stage or + * exceptionally with original exception or a new one + */ + public SafeFuture catchAndRethrow(final ExceptionThrowingConsumer onError) { + return exceptionallyCompose( + error -> { + try { + onError.accept(error); + return failedFuture(error); + } catch (Throwable t) { + return failedFuture(t); + } + }); + } + + public static SafeFuture supplyAsync(final Supplier supplier) { + return SafeFuture.of(CompletableFuture.supplyAsync(supplier)); + } + + @SuppressWarnings("unchecked") + @Override + public SafeFuture thenApply(final Function fn) { + return (SafeFuture) super.thenApply(fn); + } + + public SafeFuture thenApplyChecked(final ExceptionThrowingFunction function) { + return thenCompose( + value -> { + try { + final U result = function.apply(value); + return SafeFuture.completedFuture(result); + } catch (final Throwable e) { + return SafeFuture.failedFuture(e); + } + }); + } + + /** Shortcut to process the value when complete and return the same future */ + public SafeFuture thenPeek(Consumer fn) { + return thenApply( + v -> { + fn.accept(v); + return v; + }); + } + + @Override + public SafeFuture thenRun(final Runnable action) { + return (SafeFuture) super.thenRun(action); + } + + @Override + public SafeFuture thenRunAsync(final Runnable action, final Executor executor) { + return (SafeFuture) super.thenRunAsync(action, executor); + } + + @Override + public SafeFuture thenAccept(final Consumer action) { + return (SafeFuture) super.thenAccept(action); + } + + @Override + public SafeFuture thenAcceptAsync( + final Consumer action, final Executor executor) { + return (SafeFuture) super.thenAcceptAsync(action, executor); + } + + @SuppressWarnings("unchecked") + @Override + public SafeFuture thenCombine( + final CompletionStage other, + final BiFunction fn) { + return (SafeFuture) super.thenCombine(other, fn); + } + + @Override + public SafeFuture thenCompose(final Function> fn) { + return (SafeFuture) super.thenCompose(fn); + } + + @Override + public SafeFuture thenComposeAsync( + final Function> fn, final Executor executor) { + return (SafeFuture) super.thenComposeAsync(fn, executor); + } + + @SuppressWarnings("unchecked") + @Override + public SafeFuture thenCombineAsync( + final CompletionStage other, + final BiFunction fn, + final Executor executor) { + return (SafeFuture) super.thenCombineAsync(other, fn, executor); + } + + @Override + public SafeFuture exceptionally(final Function fn) { + return (SafeFuture) super.exceptionally(fn); + } + + @SuppressWarnings("unchecked") + @Override + public SafeFuture handle(final BiFunction fn) { + return (SafeFuture) super.handle(fn); + } + + @SuppressWarnings("unchecked") + @Override + public SafeFuture handleAsync( + final BiFunction fn, final Executor executor) { + return (SafeFuture) super.handleAsync(fn, executor); + } + + /** + * Returns a new CompletionStage that, when this stage completes either normally or exceptionally, + * is executed with this stage's result and exception as arguments to the supplied function. + * + *

When this stage is complete, the given function is invoked with the result (or {@code null} + * if none) and the exception (or {@code null} if none) returning another `CompletionStage`. When + * that stage completes, the `SafeFuture` returned by this method is completed with the same value + * or exception. + * + * @param fn the function to use to compute another CompletionStage + * @param the function's return type + * @return the new SafeFuture + */ + @SuppressWarnings({"FutureReturnValueIgnored"}) + public SafeFuture handleComposed( + final BiFunction> fn) { + final SafeFuture result = new SafeFuture<>(); + whenComplete( + (value, error) -> { + try { + propagateResult(fn.apply(value, error), result); + } catch (final Throwable t) { + result.completeExceptionally(t); + } + }); + return result; + } + + @Override + public SafeFuture whenComplete(final BiConsumer action) { + return (SafeFuture) super.whenComplete(action); + } + + public SafeFuture orTimeout(final Duration timeout) { + return orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + } + + @Override + public SafeFuture orTimeout(final long timeout, final TimeUnit unit) { + return (SafeFuture) super.orTimeout(timeout, unit); + } + + /** + * Returns the future which completes with the same result or exception The consumer is invoked if + * this future completes exceptionally + */ + public SafeFuture whenException(final Consumer action) { + return (SafeFuture) + super.whenComplete( + (r, t) -> { + if (t != null) { + action.accept(t); + } + }); + } + + /** + * Returns a void future that completes successfully with null result. The consumer is invoked if + * this future completes exceptions and the returned future only completes once the consumer + * returns. + * + *

The returned future will only complete exceptionally if the consumer throws an exception. + * + * @param action the exception handler to invoke. + * @return a void future that completes successfully unless the consumer throws an exception. + */ + public SafeFuture handleException(final Consumer action) { + return handle( + (__, error) -> { + if (error != null) { + action.accept(error); + } + return null; + }); + } + + /** + * Returns the future which completes with the same result or exception as this one. The resulting + * future becomes complete when `waitForStage` completes. If the `waitForStage` completes + * exceptionally the resulting future also completes exceptionally with the same exception + */ + public SafeFuture thenWaitFor(Function> waitForStage) { + return thenCompose(t -> waitForStage.apply(t).thenApply(__ -> t)); + } + + @SafeVarargs + @SuppressWarnings("unchecked") + public final SafeFuture or(SafeFuture... others) { + SafeFuture[] futures = Arrays.copyOf(others, others.length + 1); + futures[others.length] = this; + return anyOf(futures).thenApply(o -> (T) o); + } + + /** + * Derives a {@link SafeFuture} which yields the same result as this {@link SafeFuture} if no + * {@link Interruptor} was triggered before this future is done. + * + *

If any of supplied {@link Interruptor}s is triggered the returned {@link SafeFuture} is + * completed exceptionally. The exception thrown depends on which specific Interruptor was + * triggered + * + *

The key feature of this method is that {@code interruptFuture} contained in Interruptor + * doesn't hold the reference to dependent futures after they complete. It's desired to consider + * this for long living interrupting futures to avoid memory leaks + * + * @param interruptors a set of interruptors which futures trigger interruption if complete + * (normally or exceptionally) + * @see #createInterruptor(CompletableFuture, Supplier) + */ + // The result of anyOf() future is ignored since it is used just to handle completion + // of any future. All possible outcomes are propagated to the returned future instance + @SuppressWarnings("FutureReturnValueIgnored") + public SafeFuture orInterrupt(Interruptor... interruptors) { + CompletableFuture[] allFuts = new CompletableFuture[interruptors.length + 1]; + allFuts[0] = this; + for (int i = 0; i < interruptors.length; i++) { + allFuts[i + 1] = interruptors[i].interruptFuture; + } + SafeFuture ret = new SafeFuture<>(); + anyOf(allFuts) + .whenComplete( + (res, err) -> { + if (this.isDone()) { + this.propagateTo(ret); + } else { + for (Interruptor interruptor : interruptors) { + if (interruptor.interruptFuture.isDone()) { + try { + interruptor.getInterruptFuture().get(); + ret.completeExceptionally(interruptor.getExceptionSupplier().get()); + } catch (Exception e) { + ret.completeExceptionally(e); + } + } + } + } + }); + return ret; + } + + /** + * Class containing an interrupting Future and exception supplier which produces exception if + * interrupting Future is triggered + * + * @see #createInterruptor(CompletableFuture, Supplier) + * @see #orInterrupt(Interruptor...) + * @see #notInterrupted(Interruptor...) + */ + public static class Interruptor { + private final CompletableFuture interruptFuture; + private final Supplier exceptionSupplier; + + private Interruptor( + CompletableFuture interruptFuture, Supplier exceptionSupplier) { + this.interruptFuture = interruptFuture; + this.exceptionSupplier = exceptionSupplier; + } + + private CompletableFuture getInterruptFuture() { + return interruptFuture; + } + + private Supplier getExceptionSupplier() { + return exceptionSupplier; + } + } +} diff --git a/src/org/minima/system/network/base/gossip/GossipNetwork.java b/src/org/minima/system/network/base/gossip/GossipNetwork.java new file mode 100644 index 000000000..1386f1718 --- /dev/null +++ b/src/org/minima/system/network/base/gossip/GossipNetwork.java @@ -0,0 +1,33 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip; + +import java.util.Collection; +import java.util.Map; +//import org.apache.tuweni.bytes.Bytes; +import org.minima.system.network.base.SafeFuture; +import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; +//import tech.pegasys.teku.networking.p2p.gossip.config.GossipTopicsScoringConfig; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; +import org.minima.system.network.base.peer.NodeId; + +public interface GossipNetwork { + SafeFuture gossip(String topic, byte[] data); + + TopicChannel subscribe(String topic, TopicHandler topicHandler); + + Map> getSubscribersByTopic(); + + void updateGossipTopicScoring(final GossipTopicsScoringConfig config); +} diff --git a/src/org/minima/system/network/base/gossip/PreparedGossipMessage.java b/src/org/minima/system/network/base/gossip/PreparedGossipMessage.java new file mode 100644 index 000000000..139199dad --- /dev/null +++ b/src/org/minima/system/network/base/gossip/PreparedGossipMessage.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + + +package org.minima.system.network.base.gossip; + +//import org.apache.tuweni.bytes.Bytes; + +/** + * Semi-processed raw gossip message which can supply Gossip 'message-id' + * + * @see TopicHandler#prepareMessage(Bytes) + */ +public interface PreparedGossipMessage { + + /** + * Returns the Gossip 'message-id' If the 'message-id' calculation is resource consuming operation + * is should performed lazily by implementation class + */ + byte[] getMessageId(); +} diff --git a/src/org/minima/system/network/base/gossip/PreparedGossipMessageFactory.java b/src/org/minima/system/network/base/gossip/PreparedGossipMessageFactory.java new file mode 100644 index 000000000..354b2bd39 --- /dev/null +++ b/src/org/minima/system/network/base/gossip/PreparedGossipMessageFactory.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip; + +//import org.apache.tuweni.bytes.Bytes; + +/** Factory for {@link PreparedGossipMessage} instances */ +public interface PreparedGossipMessageFactory { + + /** Creates a {@link PreparedGossipMessage} instance */ + PreparedGossipMessage create(String topic, byte[] payload); +} diff --git a/src/org/minima/system/network/base/gossip/TopicChannel.java b/src/org/minima/system/network/base/gossip/TopicChannel.java new file mode 100644 index 000000000..6c303d6c1 --- /dev/null +++ b/src/org/minima/system/network/base/gossip/TopicChannel.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + + +package org.minima.system.network.base.gossip; + +//import org.apache.tuweni.bytes.Bytes; + +public interface TopicChannel { + void gossip(byte[] data); + + void close(); +} diff --git a/src/org/minima/system/network/base/gossip/TopicHandler.java b/src/org/minima/system/network/base/gossip/TopicHandler.java new file mode 100644 index 000000000..08a07b8bf --- /dev/null +++ b/src/org/minima/system/network/base/gossip/TopicHandler.java @@ -0,0 +1,38 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip; + +import io.libp2p.core.pubsub.ValidationResult; +import org.minima.system.network.base.SafeFuture; + +//import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +public interface TopicHandler { + + /** + * Preprocess 'raw' Gossip message returning the instance which may calculate Gossip 'message-id' + * and cache intermediate data for later message handling with {@link + * #handleMessage(PreparedGossipMessage)} + */ + PreparedGossipMessage prepareMessage(byte[] payload); + + /** + * Validates and handles gossip message preprocessed earlier by {@link #prepareMessage(Bytes)} + * + * @param message The preprocessed gossip message + * @return Message validation promise + */ + SafeFuture handleMessage(PreparedGossipMessage message); +} diff --git a/src/org/minima/system/network/base/gossip/config/GossipConfig.java b/src/org/minima/system/network/base/gossip/config/GossipConfig.java new file mode 100644 index 000000000..4abed8fbb --- /dev/null +++ b/src/org/minima/system/network/base/gossip/config/GossipConfig.java @@ -0,0 +1,206 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip.config; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.time.Duration; +import java.util.function.Consumer; + +/** + * Gossip options + * https://github.com/ethereum/eth2.0-specs/blob/v0.11.1/specs/phase0/p2p-interface.md#the-gossip-domain-gossipsub + */ +public class GossipConfig { + public static final int DEFAULT_D = 6; + public static final int DEFAULT_D_LOW = 5; + public static final int DEFAULT_D_HIGH = 12; + public static final int DEFAULT_D_LAZY = 6; + public static final Duration DEFAULT_FANOUT_TTL = Duration.ofSeconds(60); + public static final int DEFAULT_ADVERTISE = 3; + public static final int DEFAULT_HISTORY = 6; + public static final Duration DEFAULT_HEARTBEAT_INTERVAL = Duration.ofMillis(700); + public static final Duration DEFAULT_SEEN_TTL = DEFAULT_HEARTBEAT_INTERVAL.multipliedBy(550); + + private final int d; + private final int dLow; + private final int dHigh; + private final int dLazy; + private final Duration fanoutTTL; + private final int advertise; + private final int history; + private final Duration heartbeatInterval; + private final Duration seenTTL; + private final GossipScoringConfig scoringConfig; + + private GossipConfig( + int d, + int dLow, + int dHigh, + int dLazy, + Duration fanoutTTL, + int advertise, + int history, + Duration heartbeatInterval, + Duration seenTTL, + final GossipScoringConfig scoringConfig) { + this.d = d; + this.dLow = dLow; + this.dHigh = dHigh; + this.dLazy = dLazy; + this.fanoutTTL = fanoutTTL; + this.advertise = advertise; + this.history = history; + this.heartbeatInterval = heartbeatInterval; + this.seenTTL = seenTTL; + this.scoringConfig = scoringConfig; + } + + public static Builder builder() { + return new Builder(); + } + + public static GossipConfig createDefault() { + return builder().build(); + } + + public int getD() { + return d; + } + + public int getDLow() { + return dLow; + } + + public int getDHigh() { + return dHigh; + } + + public int getDLazy() { + return dLazy; + } + + public Duration getFanoutTTL() { + return fanoutTTL; + } + + public int getAdvertise() { + return advertise; + } + + public int getHistory() { + return history; + } + + public Duration getHeartbeatInterval() { + return heartbeatInterval; + } + + public Duration getSeenTTL() { + return seenTTL; + } + + public GossipScoringConfig getScoringConfig() { + return scoringConfig; + } + + public static class Builder { + private final GossipScoringConfig.Builder scoringConfigBuilder = GossipScoringConfig.builder(); + + private Integer d = DEFAULT_D; + private Integer dLow = DEFAULT_D_LOW; + private Integer dHigh = DEFAULT_D_HIGH; + private Integer dLazy = DEFAULT_D_LAZY; + private Duration fanoutTTL = DEFAULT_FANOUT_TTL; + private Integer advertise = DEFAULT_ADVERTISE; + private Integer history = DEFAULT_HISTORY; + private Duration heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL; + private Duration seenTTL = DEFAULT_SEEN_TTL; + + private Builder() {} + + public GossipConfig build() { + return new GossipConfig( + d, + dLow, + dHigh, + dLazy, + fanoutTTL, + advertise, + history, + heartbeatInterval, + seenTTL, + scoringConfigBuilder.build()); + } + + public Builder scoring(final Consumer consumer) { + consumer.accept(scoringConfigBuilder); + return this; + } + + public Builder d(final Integer d) { + checkNotNull(d); + this.d = d; + return this; + } + + public Builder dLow(final Integer dLow) { + checkNotNull(dLow); + this.dLow = dLow; + return this; + } + + public Builder dHigh(final Integer dHigh) { + checkNotNull(dHigh); + this.dHigh = dHigh; + return this; + } + + public Builder dLazy(final Integer dLazy) { + checkNotNull(dLazy); + this.dLazy = dLazy; + return this; + } + + public Builder fanoutTTL(final Duration fanoutTTL) { + checkNotNull(fanoutTTL); + this.fanoutTTL = fanoutTTL; + return this; + } + + public Builder advertise(final Integer advertise) { + checkNotNull(advertise); + this.advertise = advertise; + return this; + } + + public Builder history(final Integer history) { + checkNotNull(history); + this.history = history; + return this; + } + + public Builder heartbeatInterval(final Duration heartbeatInterval) { + checkNotNull(heartbeatInterval); + this.heartbeatInterval = heartbeatInterval; + return this; + } + + public Builder seenTTL(final Duration seenTTL) { + checkNotNull(seenTTL); + this.seenTTL = seenTTL; + return this; + } + } +} diff --git a/src/org/minima/system/network/base/gossip/config/GossipPeerScoringConfig.java b/src/org/minima/system/network/base/gossip/config/GossipPeerScoringConfig.java new file mode 100644 index 000000000..37bb744cb --- /dev/null +++ b/src/org/minima/system/network/base/gossip/config/GossipPeerScoringConfig.java @@ -0,0 +1,246 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip.config; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.time.Duration; +import java.util.Optional; +import org.minima.system.network.base.peer.NodeId; + +public class GossipPeerScoringConfig { + private final double topicScoreCap; + private final double appSpecificWeight; + private final Optional peerScorer; + private final Optional whitelistManager; + private final Optional directPeerManager; + private final double ipColocationFactorWeight; + private final int ipColocationFactorThreshold; + private final double behaviourPenaltyWeight; + private final double behaviourPenaltyDecay; + private final double behaviourPenaltyThreshold; + private final Duration decayInterval; + private final double decayToZero; + private final Duration retainScore; + + private GossipPeerScoringConfig( + final double topicScoreCap, + final Optional directPeerManager, + final Optional peerScorer, + final double appSpecificWeight, + final Optional whitelistManager, + final double ipColocationFactorWeight, + final int ipColocationFactorThreshold, + final double behaviourPenaltyWeight, + final double behaviourPenaltyDecay, + final double behaviourPenaltyThreshold, + final Duration decayInterval, + final double decayToZero, + final Duration retainScore) { + this.topicScoreCap = topicScoreCap; + this.directPeerManager = directPeerManager; + this.peerScorer = peerScorer; + this.appSpecificWeight = appSpecificWeight; + this.whitelistManager = whitelistManager; + this.ipColocationFactorWeight = ipColocationFactorWeight; + this.ipColocationFactorThreshold = ipColocationFactorThreshold; + this.behaviourPenaltyWeight = behaviourPenaltyWeight; + this.behaviourPenaltyDecay = behaviourPenaltyDecay; + this.behaviourPenaltyThreshold = behaviourPenaltyThreshold; + this.decayInterval = decayInterval; + this.decayToZero = decayToZero; + this.retainScore = retainScore; + } + + public static Builder builder() { + return new Builder(); + } + + public double getTopicScoreCap() { + return topicScoreCap; + } + + public double getAppSpecificWeight() { + return appSpecificWeight; + } + + public Optional getAppSpecificScorer() { + return peerScorer; + } + + public Optional getWhitelistManager() { + return whitelistManager; + } + + public Optional getDirectPeerManager() { + return directPeerManager; + } + + public double getIpColocationFactorWeight() { + return ipColocationFactorWeight; + } + + public int getIpColocationFactorThreshold() { + return ipColocationFactorThreshold; + } + + public double getBehaviourPenaltyWeight() { + return behaviourPenaltyWeight; + } + + public double getBehaviourPenaltyDecay() { + return behaviourPenaltyDecay; + } + + public double getBehaviourPenaltyThreshold() { + return behaviourPenaltyThreshold; + } + + public Duration getDecayInterval() { + return decayInterval; + } + + public double getDecayToZero() { + return decayToZero; + } + + public Duration getRetainScore() { + return retainScore; + } + + public static class Builder { + private Double topicScoreCap = 0.0; + private Double appSpecificWeight = 0.0; + private Optional appSpecificScorer = Optional.empty(); + private Optional whitelistManager = Optional.empty(); + private Optional directPeerManager = Optional.empty(); + private Double ipColocationFactorWeight = 0.0; + private Integer ipColocationFactorThreshold = 0; + private Double behaviourPenaltyWeight = 0.0; + private Double behaviourPenaltyDecay = 0.9; + private Double behaviourPenaltyThreshold = 1.0; + private Duration decayInterval = Duration.ofMinutes(1); + private Double decayToZero = 0.0; + private Duration retainScore = Duration.ofMinutes(10); + + private Builder() {} + + public GossipPeerScoringConfig build() { + return new GossipPeerScoringConfig( + topicScoreCap, + directPeerManager, + appSpecificScorer, + appSpecificWeight, + whitelistManager, + ipColocationFactorWeight, + ipColocationFactorThreshold, + behaviourPenaltyWeight, + behaviourPenaltyDecay, + behaviourPenaltyThreshold, + decayInterval, + decayToZero, + retainScore); + } + + public Builder topicScoreCap(final Double topicScoreCap) { + checkNotNull(topicScoreCap); + this.topicScoreCap = topicScoreCap; + return this; + } + + public Builder appSpecificWeight(final Double appSpecificWeight) { + checkNotNull(appSpecificWeight); + this.appSpecificWeight = appSpecificWeight; + return this; + } + + public Builder appSpecificScorer(final Optional appSpecificScorer) { + checkNotNull(appSpecificScorer); + this.appSpecificScorer = appSpecificScorer; + return this; + } + + public Builder whitelistManager(final Optional whitelistManager) { + checkNotNull(whitelistManager); + this.whitelistManager = whitelistManager; + return this; + } + + public Builder directPeerManager(final Optional directPeerManager) { + checkNotNull(directPeerManager); + this.directPeerManager = directPeerManager; + return this; + } + + public Builder ipColocationFactorWeight(final Double ipColocationFactorWeight) { + checkNotNull(ipColocationFactorWeight); + this.ipColocationFactorWeight = ipColocationFactorWeight; + return this; + } + + public Builder ipColocationFactorThreshold(final Integer ipColocationFactorThreshold) { + checkNotNull(ipColocationFactorThreshold); + this.ipColocationFactorThreshold = ipColocationFactorThreshold; + return this; + } + + public Builder behaviourPenaltyWeight(final Double behaviourPenaltyWeight) { + checkNotNull(behaviourPenaltyWeight); + this.behaviourPenaltyWeight = behaviourPenaltyWeight; + return this; + } + + public Builder behaviourPenaltyDecay(final Double behaviourPenaltyDecay) { + checkNotNull(behaviourPenaltyDecay); + this.behaviourPenaltyDecay = behaviourPenaltyDecay; + return this; + } + + public Builder behaviourPenaltyThreshold(final Double behaviourPenaltyThreshold) { + checkNotNull(behaviourPenaltyThreshold); + this.behaviourPenaltyThreshold = behaviourPenaltyThreshold; + return this; + } + + public Builder decayInterval(final Duration decayInterval) { + checkNotNull(decayInterval); + this.decayInterval = decayInterval; + return this; + } + + public Builder decayToZero(final Double decayToZero) { + checkNotNull(decayToZero); + this.decayToZero = decayToZero; + return this; + } + + public Builder retainScore(final Duration retainScore) { + checkNotNull(retainScore); + this.retainScore = retainScore; + return this; + } + } + + public interface PeerScorer { + double scorePeer(final NodeId peer); + } + + public interface DirectPeerManager { + boolean isDirectPeer(final NodeId peer); + } + + public interface WhitelistedIpManager { + boolean isWhitelisted(final String ipAddress); + } +} diff --git a/src/org/minima/system/network/base/gossip/config/GossipScoringConfig.java b/src/org/minima/system/network/base/gossip/config/GossipScoringConfig.java new file mode 100644 index 000000000..10ba5590b --- /dev/null +++ b/src/org/minima/system/network/base/gossip/config/GossipScoringConfig.java @@ -0,0 +1,167 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip.config; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Map; +import java.util.function.Consumer; + +/** Gossip scoring config. Contains peer scoring and topic scoring configs. */ +public class GossipScoringConfig { + private final GossipPeerScoringConfig peerScoringConfig; + private final GossipTopicScoringConfig defaultTopicScoringConfig; + private final Map topicScoringConfig; + private final double gossipThreshold; + private final double publishThreshold; + private final double graylistThreshold; + private final double acceptPXThreshold; + private final double opportunisticGraftThreshold; + + private GossipScoringConfig( + final GossipPeerScoringConfig peerScoringConfig, + final GossipTopicScoringConfig defaultTopicScoringConfig, + final Map topicScoringConfig, + final double gossipThreshold, + final double publishThreshold, + final double graylistThreshold, + final double acceptPXThreshold, + final double opportunisticGraftThreshold) { + this.peerScoringConfig = peerScoringConfig; + this.defaultTopicScoringConfig = defaultTopicScoringConfig; + this.topicScoringConfig = topicScoringConfig; + this.gossipThreshold = gossipThreshold; + this.publishThreshold = publishThreshold; + this.graylistThreshold = graylistThreshold; + this.acceptPXThreshold = acceptPXThreshold; + this.opportunisticGraftThreshold = opportunisticGraftThreshold; + } + + public GossipPeerScoringConfig getPeerScoringConfig() { + return peerScoringConfig; + } + + public GossipTopicScoringConfig getDefaultTopicScoringConfig() { + return defaultTopicScoringConfig; + } + + public Map getTopicScoringConfig() { + return topicScoringConfig; + } + + public double getGossipThreshold() { + return gossipThreshold; + } + + public double getPublishThreshold() { + return publishThreshold; + } + + public double getGraylistThreshold() { + return graylistThreshold; + } + + public double getAcceptPXThreshold() { + return acceptPXThreshold; + } + + public double getOpportunisticGraftThreshold() { + return opportunisticGraftThreshold; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private final GossipPeerScoringConfig.Builder peerScoringConfigBuilder = + GossipPeerScoringConfig.builder(); + private final GossipTopicScoringConfig.Builder defaultTopicScoringConfigBuilder = + GossipTopicScoringConfig.builder(); + private final GossipTopicsScoringConfig.Builder topicsScoringBuilder = + GossipTopicsScoringConfig.builder(); + private Double gossipThreshold = 0.0; + private Double publishThreshold = 0.0; + private Double graylistThreshold = 0.0; + private Double acceptPXThreshold = 0.0; + private Double opportunisticGraftThreshold = 0.0; + + private Builder() {} + + public GossipScoringConfig build() { + final Map topicConfigs = + topicsScoringBuilder.build().getTopicConfigs(); + return new GossipScoringConfig( + peerScoringConfigBuilder.build(), + defaultTopicScoringConfigBuilder.build(), + topicConfigs, + gossipThreshold, + publishThreshold, + graylistThreshold, + acceptPXThreshold, + opportunisticGraftThreshold); + } + + public Builder peerScoring(final Consumer consumer) { + consumer.accept(peerScoringConfigBuilder); + return this; + } + + public Builder topicScoring( + final String topic, final Consumer consumer) { + topicsScoringBuilder.topicScoring(topic, consumer); + return this; + } + + public Builder topicScoring(final Consumer consumer) { + consumer.accept(topicsScoringBuilder); + return this; + } + + public Builder defaultTopicScoring(final Consumer consumer) { + consumer.accept(defaultTopicScoringConfigBuilder); + return this; + } + + public Builder gossipThreshold(final Double gossipThreshold) { + checkNotNull(gossipThreshold); + this.gossipThreshold = gossipThreshold; + return this; + } + + public Builder publishThreshold(final Double publishThreshold) { + checkNotNull(publishThreshold); + this.publishThreshold = publishThreshold; + return this; + } + + public Builder graylistThreshold(final Double graylistThreshold) { + checkNotNull(graylistThreshold); + this.graylistThreshold = graylistThreshold; + return this; + } + + public Builder acceptPXThreshold(final Double acceptPXThreshold) { + checkNotNull(acceptPXThreshold); + this.acceptPXThreshold = acceptPXThreshold; + return this; + } + + public Builder opportunisticGraftThreshold(final Double opportunisticGraftThreshold) { + checkNotNull(opportunisticGraftThreshold); + this.opportunisticGraftThreshold = opportunisticGraftThreshold; + return this; + } + } +} diff --git a/src/org/minima/system/network/base/gossip/config/GossipTopicScoringConfig.java b/src/org/minima/system/network/base/gossip/config/GossipTopicScoringConfig.java new file mode 100644 index 000000000..9593f575f --- /dev/null +++ b/src/org/minima/system/network/base/gossip/config/GossipTopicScoringConfig.java @@ -0,0 +1,293 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip.config; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.time.Duration; + +/** Scoring config for a single topic */ +public class GossipTopicScoringConfig { + private final double topicWeight; + private final double timeInMeshWeight; + private final Duration timeInMeshQuantum; + private final double timeInMeshCap; + private final double firstMessageDeliveriesWeight; + private final double firstMessageDeliveriesDecay; + private final double firstMessageDeliveriesCap; + private final double meshMessageDeliveriesWeight; + private final double meshMessageDeliveriesDecay; + private final double meshMessageDeliveriesThreshold; + private final double meshMessageDeliveriesCap; + private final Duration meshMessageDeliveriesActivation; + private final Duration meshMessageDeliveryWindow; + private final double meshFailurePenaltyWeight; + private final double meshFailurePenaltyDecay; + private final double invalidMessageDeliveriesWeight; + private final double invalidMessageDeliveriesDecay; + + private GossipTopicScoringConfig( + final double topicWeight, + final double timeInMeshWeight, + final Duration timeInMeshQuantum, + final double timeInMeshCap, + final double firstMessageDeliveriesWeight, + final double firstMessageDeliveriesDecay, + final double firstMessageDeliveriesCap, + final double meshMessageDeliveriesWeight, + final double meshMessageDeliveriesDecay, + final double meshMessageDeliveriesThreshold, + final double meshMessageDeliveriesCap, + final Duration meshMessageDeliveriesActivation, + final Duration meshMessageDeliveryWindow, + final double meshFailurePenaltyWeight, + final double meshFailurePenaltyDecay, + final double invalidMessageDeliveriesWeight, + final double invalidMessageDeliveriesDecay) { + this.topicWeight = topicWeight; + this.timeInMeshWeight = timeInMeshWeight; + this.timeInMeshQuantum = timeInMeshQuantum; + this.timeInMeshCap = timeInMeshCap; + this.firstMessageDeliveriesWeight = firstMessageDeliveriesWeight; + this.firstMessageDeliveriesDecay = firstMessageDeliveriesDecay; + this.firstMessageDeliveriesCap = firstMessageDeliveriesCap; + this.meshMessageDeliveriesWeight = meshMessageDeliveriesWeight; + this.meshMessageDeliveriesDecay = meshMessageDeliveriesDecay; + this.meshMessageDeliveriesThreshold = meshMessageDeliveriesThreshold; + this.meshMessageDeliveriesCap = meshMessageDeliveriesCap; + this.meshMessageDeliveriesActivation = meshMessageDeliveriesActivation; + this.meshMessageDeliveryWindow = meshMessageDeliveryWindow; + this.meshFailurePenaltyWeight = meshFailurePenaltyWeight; + this.meshFailurePenaltyDecay = meshFailurePenaltyDecay; + this.invalidMessageDeliveriesWeight = invalidMessageDeliveriesWeight; + this.invalidMessageDeliveriesDecay = invalidMessageDeliveriesDecay; + } + + public static Builder builder() { + return new Builder(); + } + + public double getTopicWeight() { + return topicWeight; + } + + public double getTimeInMeshWeight() { + return timeInMeshWeight; + } + + public Duration getTimeInMeshQuantum() { + return timeInMeshQuantum; + } + + public double getTimeInMeshCap() { + return timeInMeshCap; + } + + public double getFirstMessageDeliveriesWeight() { + return firstMessageDeliveriesWeight; + } + + public double getFirstMessageDeliveriesDecay() { + return firstMessageDeliveriesDecay; + } + + public double getFirstMessageDeliveriesCap() { + return firstMessageDeliveriesCap; + } + + public double getMeshMessageDeliveriesWeight() { + return meshMessageDeliveriesWeight; + } + + public double getMeshMessageDeliveriesDecay() { + return meshMessageDeliveriesDecay; + } + + public double getMeshMessageDeliveriesThreshold() { + return meshMessageDeliveriesThreshold; + } + + public double getMeshMessageDeliveriesCap() { + return meshMessageDeliveriesCap; + } + + public Duration getMeshMessageDeliveriesActivation() { + return meshMessageDeliveriesActivation; + } + + public Duration getMeshMessageDeliveryWindow() { + return meshMessageDeliveryWindow; + } + + public double getMeshFailurePenaltyWeight() { + return meshFailurePenaltyWeight; + } + + public double getMeshFailurePenaltyDecay() { + return meshFailurePenaltyDecay; + } + + public double getInvalidMessageDeliveriesWeight() { + return invalidMessageDeliveriesWeight; + } + + public double getInvalidMessageDeliveriesDecay() { + return invalidMessageDeliveriesDecay; + } + + public static class Builder { + private Double topicWeight = 0.0; + private Double timeInMeshWeight = 0.0; + private Duration timeInMeshQuantum = Duration.ofSeconds(1); + private Double timeInMeshCap = 0.0; + private Double firstMessageDeliveriesWeight = 0.0; + private Double firstMessageDeliveriesDecay = 0.0; + private Double firstMessageDeliveriesCap = 0.0; + private Double meshMessageDeliveriesWeight = 0.0; + private Double meshMessageDeliveriesDecay = 0.0; + private Double meshMessageDeliveriesThreshold = 0.0; + private Double meshMessageDeliveriesCap = 0.0; + private Duration meshMessageDeliveriesActivation = Duration.ofMinutes(1); + private Duration meshMessageDeliveryWindow = Duration.ofMillis(10); + private Double meshFailurePenaltyWeight = 0.0; + private Double meshFailurePenaltyDecay = 0.0; + private Double invalidMessageDeliveriesWeight = 0.0; + private Double invalidMessageDeliveriesDecay = 0.0; + + private Builder() {} + + public GossipTopicScoringConfig build() { + return new GossipTopicScoringConfig( + topicWeight, + timeInMeshWeight, + timeInMeshQuantum, + timeInMeshCap, + firstMessageDeliveriesWeight, + firstMessageDeliveriesDecay, + firstMessageDeliveriesCap, + meshMessageDeliveriesWeight, + meshMessageDeliveriesDecay, + meshMessageDeliveriesThreshold, + meshMessageDeliveriesCap, + meshMessageDeliveriesActivation, + meshMessageDeliveryWindow, + meshFailurePenaltyWeight, + meshFailurePenaltyDecay, + invalidMessageDeliveriesWeight, + invalidMessageDeliveriesDecay); + } + + public Builder topicWeight(final Double topicWeight) { + checkNotNull(topicWeight); + this.topicWeight = topicWeight; + return this; + } + + public Builder timeInMeshWeight(final Double timeInMeshWeight) { + checkNotNull(timeInMeshWeight); + this.timeInMeshWeight = timeInMeshWeight; + return this; + } + + public Builder timeInMeshQuantum(final Duration timeInMeshQuantum) { + checkNotNull(timeInMeshQuantum); + this.timeInMeshQuantum = timeInMeshQuantum; + return this; + } + + public Builder timeInMeshCap(final Double timeInMeshCap) { + checkNotNull(timeInMeshCap); + this.timeInMeshCap = timeInMeshCap; + return this; + } + + public Builder firstMessageDeliveriesWeight(final Double firstMessageDeliveriesWeight) { + checkNotNull(firstMessageDeliveriesWeight); + this.firstMessageDeliveriesWeight = firstMessageDeliveriesWeight; + return this; + } + + public Builder firstMessageDeliveriesDecay(final Double firstMessageDeliveriesDecay) { + checkNotNull(firstMessageDeliveriesDecay); + this.firstMessageDeliveriesDecay = firstMessageDeliveriesDecay; + return this; + } + + public Builder firstMessageDeliveriesCap(final Double firstMessageDeliveriesCap) { + checkNotNull(firstMessageDeliveriesCap); + this.firstMessageDeliveriesCap = firstMessageDeliveriesCap; + return this; + } + + public Builder meshMessageDeliveriesWeight(final Double meshMessageDeliveriesWeight) { + checkNotNull(meshMessageDeliveriesWeight); + this.meshMessageDeliveriesWeight = meshMessageDeliveriesWeight; + return this; + } + + public Builder meshMessageDeliveriesDecay(final Double meshMessageDeliveriesDecay) { + checkNotNull(meshMessageDeliveriesDecay); + this.meshMessageDeliveriesDecay = meshMessageDeliveriesDecay; + return this; + } + + public Builder meshMessageDeliveriesThreshold(final Double meshMessageDeliveriesThreshold) { + checkNotNull(meshMessageDeliveriesThreshold); + this.meshMessageDeliveriesThreshold = meshMessageDeliveriesThreshold; + return this; + } + + public Builder meshMessageDeliveriesCap(final Double meshMessageDeliveriesCap) { + checkNotNull(meshMessageDeliveriesCap); + this.meshMessageDeliveriesCap = meshMessageDeliveriesCap; + return this; + } + + public Builder meshMessageDeliveriesActivation(final Duration meshMessageDeliveriesActivation) { + checkNotNull(meshMessageDeliveriesActivation); + this.meshMessageDeliveriesActivation = meshMessageDeliveriesActivation; + return this; + } + + public Builder meshMessageDeliveryWindow(final Duration meshMessageDeliveryWindow) { + checkNotNull(meshMessageDeliveryWindow); + this.meshMessageDeliveryWindow = meshMessageDeliveryWindow; + return this; + } + + public Builder meshFailurePenaltyWeight(final Double meshFailurePenaltyWeight) { + checkNotNull(meshFailurePenaltyWeight); + this.meshFailurePenaltyWeight = meshFailurePenaltyWeight; + return this; + } + + public Builder meshFailurePenaltyDecay(final Double meshFailurePenaltyDecay) { + checkNotNull(meshFailurePenaltyDecay); + this.meshFailurePenaltyDecay = meshFailurePenaltyDecay; + return this; + } + + public Builder invalidMessageDeliveriesWeight(final Double invalidMessageDeliveriesWeight) { + checkNotNull(invalidMessageDeliveriesWeight); + this.invalidMessageDeliveriesWeight = invalidMessageDeliveriesWeight; + return this; + } + + public Builder invalidMessageDeliveriesDecay(final Double invalidMessageDeliveriesDecay) { + checkNotNull(invalidMessageDeliveriesDecay); + this.invalidMessageDeliveriesDecay = invalidMessageDeliveriesDecay; + return this; + } + } +} diff --git a/src/org/minima/system/network/base/gossip/config/GossipTopicsScoringConfig.java b/src/org/minima/system/network/base/gossip/config/GossipTopicsScoringConfig.java new file mode 100644 index 000000000..5235a69f4 --- /dev/null +++ b/src/org/minima/system/network/base/gossip/config/GossipTopicsScoringConfig.java @@ -0,0 +1,66 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.gossip.config; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** Scoring config for a collection a set of topics */ +public class GossipTopicsScoringConfig { + private final Map topicConfigs; + + private GossipTopicsScoringConfig(final Map topicConfigs) { + this.topicConfigs = topicConfigs; + } + + public boolean isEmpty() { + return topicConfigs.isEmpty(); + } + + public static Builder builder() { + return new Builder(); + } + + public Map getTopicConfigs() { + return topicConfigs; + } + + public static class Builder { + private final Map topicScoringConfigBuilders = + new HashMap<>(); + + public GossipTopicsScoringConfig build() { + final Map topicConfig = + topicScoringConfigBuilders.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().build())); + return new GossipTopicsScoringConfig(topicConfig); + } + + public Builder topicScoring( + final String topic, final Consumer consumer) { + GossipTopicScoringConfig.Builder builder = + topicScoringConfigBuilders.computeIfAbsent( + topic, __ -> GossipTopicScoringConfig.builder()); + consumer.accept(builder); + return this; + } + + public Builder clear() { + topicScoringConfigBuilders.clear(); + return this; + } + } +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/GossipHandler.java b/src/org/minima/system/network/base/libp2p/gossip/GossipHandler.java new file mode 100644 index 000000000..8fb06a972 --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/GossipHandler.java @@ -0,0 +1,119 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +import io.libp2p.core.pubsub.MessageApi; +import io.libp2p.core.pubsub.PubsubPublisherApi; +import io.libp2p.core.pubsub.Topic; +import io.libp2p.core.pubsub.ValidationResult; +import io.libp2p.pubsub.PubsubMessage; +import io.netty.buffer.Unpooled; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +//import org.apache.tuweni.bytes.Bytes; +//import org.hyperledger.besu.plugin.services.MetricsSystem; +//import org.hyperledger.besu.plugin.services.metrics.Counter; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; +//import tech.pegasys.teku.infrastructure.collections.LimitedSet; +//import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; +//import tech.pegasys.teku.networking.p2p.gossip.TopicHandler; +import org.minima.system.network.base.LimitedSet; +import org.minima.system.network.base.SafeFuture; +import org.minima.system.network.base.gossip.TopicHandler; +import org.minima.system.network.base.metrics.Counter; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.metrics.TekuMetricCategory; + +public class GossipHandler implements Function> { + private static final Logger LOG = LogManager.getLogger(); + + private static final int GOSSIP_MAX_SIZE = 1024; + + private static final SafeFuture VALIDATION_FAILED = + SafeFuture.completedFuture(ValidationResult.Invalid); + private static final SafeFuture VALIDATION_IGNORED = + SafeFuture.completedFuture(ValidationResult.Ignore); + + private static final int MAX_SENT_MESSAGES = 2048; + + private final Topic topic; + private final PubsubPublisherApi publisher; + private final TopicHandler handler; + private final Set processedMessages = LimitedSet.create(MAX_SENT_MESSAGES); + private final Counter messageCounter; + + public GossipHandler( + final MetricsSystem metricsSystem, + final Topic topic, + final PubsubPublisherApi publisher, + final TopicHandler handler) { + this.topic = topic; + this.publisher = publisher; + this.handler = handler; + this.messageCounter = + metricsSystem + .createLabelledCounter( + TekuMetricCategory.LIBP2P, + "gossip_messages_total", + "Total number of gossip messages received (avoid libp2p deduplication)", + "topic") + .labels(topic.getTopic()); + } + + @Override + public SafeFuture apply(final MessageApi message) { + messageCounter.inc(); + final int messageSize = message.getData().readableBytes(); + if (messageSize > GOSSIP_MAX_SIZE) { + LOG.trace( + "Rejecting gossip message of length {} which exceeds maximum size of {}", + messageSize, + GOSSIP_MAX_SIZE); + return VALIDATION_FAILED; + } + byte[] arr = new byte[message.getData().readableBytes()]; + message.getData().slice().readBytes(arr); + byte[] bytes = arr; // Bytes.wrap(arr); + if (!processedMessages.add(bytes)) { + // We've already seen this message, skip processing + LOG.trace("Ignoring duplicate message for topic {}: {} bytes", topic, bytes.length); + return VALIDATION_IGNORED; + } + LOG.trace("Received message for topic {}: {} bytes", topic, bytes.length); + + PubsubMessage pubsubMessage = message.getOriginalMessage(); + if (!(pubsubMessage instanceof PreparedPubsubMessage)) { + throw new IllegalArgumentException( + "Don't know this PubsubMessage implementation: " + pubsubMessage.getClass()); + } + PreparedPubsubMessage gossipPubsubMessage = (PreparedPubsubMessage) pubsubMessage; + return handler.handleMessage(gossipPubsubMessage.getPreparedMessage()); + } + + public void gossip(byte[] bytes) { + if (!processedMessages.add(bytes)) { + // We've already gossiped this data + return; + } + + LOG.trace("Gossiping {}: {} bytes", topic, bytes.length); + SafeFuture.of(publisher.publish(Unpooled.wrappedBuffer(bytes), topic)) + .finish( + () -> LOG.trace("Successfully gossiped message on {}", topic), + err -> LOG.debug("Failed to gossip message on " + topic, err)); + } +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/GossipTopicFilter.java b/src/org/minima/system/network/base/libp2p/gossip/GossipTopicFilter.java new file mode 100644 index 000000000..10ca351d6 --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/GossipTopicFilter.java @@ -0,0 +1,19 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +@FunctionalInterface +public interface GossipTopicFilter { + boolean isRelevantTopic(String topic); +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/GossipWireValidator.java b/src/org/minima/system/network/base/libp2p/gossip/GossipWireValidator.java new file mode 100644 index 000000000..111bc8bee --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/GossipWireValidator.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +import io.libp2p.pubsub.PubsubMessage; +import io.libp2p.pubsub.PubsubRouterMessageValidator; +import org.jetbrains.annotations.NotNull; +import pubsub.pb.Rpc; + +/** + * Validates Gossip messages at the level of Protobuf structures Rejects messages with prohibited + * Gossip fields: {@code from, signature, seqno} + */ +public class GossipWireValidator implements PubsubRouterMessageValidator { + + public static class InvalidGossipMessageException extends IllegalArgumentException { + public InvalidGossipMessageException(String s) { + super(s); + } + } + + @Override + public void validate(@NotNull PubsubMessage pubsubMessage) { + Rpc.Message message = pubsubMessage.getProtobufMessage(); + if (message.hasFrom()) { + throw new InvalidGossipMessageException("The message has prohibited 'from' field: "); + } + if (message.hasSignature()) { + throw new InvalidGossipMessageException("The message has prohibited 'signature' field"); + } + if (message.hasSeqno()) { + throw new InvalidGossipMessageException("The message has prohibited 'seqno' field"); + } + } +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java b/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java new file mode 100644 index 000000000..78d307809 --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java @@ -0,0 +1,242 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +import com.google.common.base.Preconditions; +import io.libp2p.core.PeerId; +import io.libp2p.core.pubsub.PubsubApi; +import io.libp2p.core.pubsub.PubsubApiKt; +import io.libp2p.core.pubsub.PubsubPublisherApi; +import io.libp2p.core.pubsub.PubsubSubscription; +import io.libp2p.core.pubsub.Topic; +import io.libp2p.core.pubsub.ValidationResult; +import io.libp2p.pubsub.FastIdSeenCache; +import io.libp2p.pubsub.MaxCountTopicSubscriptionFilter; +import io.libp2p.pubsub.PubsubProtocol; +import io.libp2p.pubsub.PubsubRouterMessageValidator; +import io.libp2p.pubsub.SeenCache; +import io.libp2p.pubsub.TTLSeenCache; +import io.libp2p.pubsub.TopicSubscriptionFilter; +import io.libp2p.pubsub.gossip.Gossip; +import io.libp2p.pubsub.gossip.GossipParams; +import io.libp2p.pubsub.gossip.GossipRouter; +import io.libp2p.pubsub.gossip.GossipScoreParams; +import io.libp2p.pubsub.gossip.GossipTopicScoreParams; +import io.netty.buffer.Unpooled; +import io.netty.channel.ChannelHandler; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import kotlin.jvm.functions.Function0; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.minima.system.network.base.LibP2PNodeId; +import org.minima.system.network.base.LibP2PParamsFactory; +import org.minima.system.network.base.SafeFuture; +// import org.apache.tuweni.bytes.Bytes; +// import org.apache.tuweni.crypto.Hash; +// import org.hyperledger.besu.plugin.services.MetricsSystem; +// import org.jetbrains.annotations.NotNull; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.gossip.GossipNetwork; +// import tech.pegasys.teku.networking.p2p.gossip.PreparedGossipMessage; +// import tech.pegasys.teku.networking.p2p.gossip.PreparedGossipMessageFactory; +// import tech.pegasys.teku.networking.p2p.gossip.TopicChannel; +// import tech.pegasys.teku.networking.p2p.gossip.TopicHandler; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipConfig; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipTopicsScoringConfig; +// import tech.pegasys.teku.networking.p2p.libp2p.LibP2PNodeId; +// import tech.pegasys.teku.networking.p2p.libp2p.config.LibP2PParamsFactory; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +import org.minima.system.network.base.gossip.GossipNetwork; +import org.minima.system.network.base.gossip.PreparedGossipMessage; +import org.minima.system.network.base.gossip.PreparedGossipMessageFactory; +import org.minima.system.network.base.gossip.TopicChannel; +import org.minima.system.network.base.gossip.TopicHandler; +import org.minima.system.network.base.gossip.config.GossipConfig; +import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.NodeId; +import org.minima.utils.Crypto; + +public class LibP2PGossipNetwork implements GossipNetwork { + + private static final Logger LOG = LogManager.getLogger(); + + private static final PubsubRouterMessageValidator STRICT_FIELDS_VALIDATOR = + new GossipWireValidator(); + private static final Function0 NULL_SEQNO_GENERATOR = () -> null; + + private final MetricsSystem metricsSystem; + private final Gossip gossip; + private final PubsubPublisherApi publisher; + private final TopicHandlers topicHandlers; + + public static LibP2PGossipNetwork create( + MetricsSystem metricsSystem, + GossipConfig gossipConfig, + PreparedGossipMessageFactory defaultMessageFactory, + GossipTopicFilter gossipTopicFilter, + boolean logWireGossip) { + + TopicHandlers topicHandlers = new TopicHandlers(); + Gossip gossip = + createGossip( + gossipConfig, logWireGossip, defaultMessageFactory, gossipTopicFilter, topicHandlers); + PubsubPublisherApi publisher = gossip.createPublisher(null, NULL_SEQNO_GENERATOR); + + return new LibP2PGossipNetwork(metricsSystem, gossip, publisher, topicHandlers); + } + + private static Gossip createGossip( + GossipConfig gossipConfig, + boolean gossipLogsEnabled, + PreparedGossipMessageFactory defaultMessageFactory, + GossipTopicFilter gossipTopicFilter, + TopicHandlers topicHandlers) { + final GossipParams gossipParams = LibP2PParamsFactory.createGossipParams(gossipConfig); + final GossipScoreParams scoreParams = + LibP2PParamsFactory.createGossipScoreParams(gossipConfig.getScoringConfig()); + + final TopicSubscriptionFilter subscriptionFilter = + new MaxCountTopicSubscriptionFilter(100, 200, gossipTopicFilter::isRelevantTopic); + GossipRouter router = + new GossipRouter( + gossipParams, scoreParams, PubsubProtocol.Gossip_V_1_1, subscriptionFilter) { + + final SeenCache> seenCache = + new TTLSeenCache<>( + new FastIdSeenCache<>( + msg -> + (new Crypto()).hashSHA2(msg.getProtobufMessage().getData().toByteArray())), + // Bytes.wrap( + // Hash.sha2_256(msg.getProtobufMessage().getData().toByteArray())) + // ), + gossipParams.getSeenTTL(), + getCurTimeMillis()); + + @NotNull + @Override + protected SeenCache> getSeenMessages() { + return seenCache; + } + }; + + router.setMessageFactory( + msg -> { + Preconditions.checkArgument( + msg.getTopicIDsCount() == 1, + "Unexpected number of topics for a single message: " + msg.getTopicIDsCount()); + String topic = msg.getTopicIDs(0); + byte[] payload = msg.getData().toByteArray(); + + PreparedGossipMessage preparedMessage = + topicHandlers + .getHandlerForTopic(topic) + .map(handler -> handler.prepareMessage(payload)) + .orElse(defaultMessageFactory.create(topic, payload)); + + return new PreparedPubsubMessage(msg, preparedMessage); + }); + router.setMessageValidator(STRICT_FIELDS_VALIDATOR); + + ChannelHandler debugHandler = + gossipLogsEnabled ? new LoggingHandler("wire.gossip", LogLevel.DEBUG) : null; + PubsubApi pubsubApi = PubsubApiKt.createPubsubApi(router); + + return new Gossip(router, pubsubApi, debugHandler); + } + + public LibP2PGossipNetwork( + MetricsSystem metricsSystem, + Gossip gossip, + PubsubPublisherApi publisher, + TopicHandlers topicHandlers) { + this.metricsSystem = metricsSystem; + this.gossip = gossip; + this.publisher = publisher; + this.topicHandlers = topicHandlers; + } + + @Override + public SafeFuture gossip(final String topic, final byte[] data) { + return SafeFuture.of( + publisher.publish(Unpooled.wrappedBuffer(data), new Topic(topic))); + } + + @Override + public TopicChannel subscribe(final String topic, final TopicHandler topicHandler) { + LOG.trace("Subscribe to topic: {}", topic); + topicHandlers.add(topic, topicHandler); + final Topic libP2PTopic = new Topic(topic); + final GossipHandler gossipHandler = + new GossipHandler(metricsSystem, libP2PTopic, publisher, topicHandler); + PubsubSubscription subscription = gossip.subscribe(gossipHandler, libP2PTopic); + return new LibP2PTopicChannel(gossipHandler, subscription); + } + + @Override + public Map> getSubscribersByTopic() { + Map> peerTopics = gossip.getPeerTopics().join(); + final Map> result = new HashMap<>(); + for (Map.Entry> peerTopic : peerTopics.entrySet()) { + final LibP2PNodeId nodeId = new LibP2PNodeId(peerTopic.getKey()); + peerTopic + .getValue() + .forEach( + topic -> result.computeIfAbsent(topic.getTopic(), __ -> new HashSet<>()).add(nodeId)); + } + return result; + } + + @Override + public void updateGossipTopicScoring(final GossipTopicsScoringConfig config) { + if (config.isEmpty()) { + return; + } + + final Map params = + config.getTopicConfigs().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> LibP2PParamsFactory.createTopicScoreParams(e.getValue()))); + gossip.updateTopicScoreParams(params); + } + + public Gossip getGossip() { + return gossip; + } + + private static class TopicHandlers { + + private final Map topicToHandlerMap = new ConcurrentHashMap<>(); + + public void add(String topic, TopicHandler handler) { + topicToHandlerMap.put(topic, handler); + } + + public Optional getHandlerForTopic(String topic) { + return Optional.ofNullable(topicToHandlerMap.get(topic)); + } + } +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/LibP2PTopicChannel.java b/src/org/minima/system/network/base/libp2p/gossip/LibP2PTopicChannel.java new file mode 100644 index 000000000..308c77b05 --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/LibP2PTopicChannel.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +import io.libp2p.core.pubsub.PubsubSubscription; +import java.util.concurrent.atomic.AtomicBoolean; +//import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.networking.p2p.gossip.TopicChannel; +import org.minima.system.network.base.gossip.TopicChannel; + +public class LibP2PTopicChannel implements TopicChannel { + private final GossipHandler topicHandler; + private final PubsubSubscription subscription; + private final AtomicBoolean closed = new AtomicBoolean(false); + + public LibP2PTopicChannel( + final GossipHandler topicHandler, final PubsubSubscription subscription) { + this.topicHandler = topicHandler; + this.subscription = subscription; + } + + @Override + public void gossip(final byte[] data) { + if (closed.get()) { + return; + } + topicHandler.gossip(data); + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + subscription.unsubscribe(); + } + } +} diff --git a/src/org/minima/system/network/base/libp2p/gossip/PreparedPubsubMessage.java b/src/org/minima/system/network/base/libp2p/gossip/PreparedPubsubMessage.java new file mode 100644 index 000000000..6041c222a --- /dev/null +++ b/src/org/minima/system/network/base/libp2p/gossip/PreparedPubsubMessage.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.libp2p.gossip; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import io.libp2p.core.pubsub.MessageApi; +import io.libp2p.etc.types.WBytes; +import io.libp2p.pubsub.AbstractPubsubMessage; +import io.libp2p.pubsub.PubsubMessage; +import io.libp2p.pubsub.gossip.GossipRouter; +import org.jetbrains.annotations.NotNull; +import pubsub.pb.Rpc.Message; +import org.minima.system.network.base.gossip.PreparedGossipMessage; + +/** + * The bridge class between outer Libp2p {@link PubsubMessage} and inner {@link + * PreparedGossipMessage} + * + *

The {@link PreparedGossipMessage} instance created during {@link + * GossipRouter#getMessageFactory()} invocation can later be accessed when the gossip message is + * handled: {@link MessageApi#getOriginalMessage()} + */ +public class PreparedPubsubMessage extends AbstractPubsubMessage { + + private final Message protobufMessage; + private final PreparedGossipMessage preparedMessage; + private final Supplier cachedMessageId; + + public PreparedPubsubMessage(Message protobufMessage, PreparedGossipMessage preparedMessage) { + this.protobufMessage = protobufMessage; + this.preparedMessage = preparedMessage; + cachedMessageId = + Suppliers.memoize(() -> new WBytes(preparedMessage.getMessageId())); + } + + @NotNull + @Override + public WBytes getMessageId() { + return cachedMessageId.get(); + } + + @NotNull + @Override + public Message getProtobufMessage() { + return protobufMessage; + } + + public PreparedGossipMessage getPreparedMessage() { + return preparedMessage; + } +} diff --git a/src/org/minima/system/network/base/metrics/Counter.java b/src/org/minima/system/network/base/metrics/Counter.java new file mode 100644 index 000000000..cf016beda --- /dev/null +++ b/src/org/minima/system/network/base/metrics/Counter.java @@ -0,0 +1,35 @@ + +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.minima.system.network.base.metrics; + +/** + * A counter is a metric to track counts of events or running totals etc. The value of the counter + * can only increase. + */ +public interface Counter { + + /** Increment the counter by 1. */ + void inc(); + + /** + * Increment the counter by a specified amount. + * + * @param amount The amount to increment the counter by. Must be greater than or equal to 0. + */ + void inc(long amount); +} + diff --git a/src/org/minima/system/network/base/metrics/LabelledMetric.java b/src/org/minima/system/network/base/metrics/LabelledMetric.java new file mode 100644 index 000000000..6dde9b139 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/LabelledMetric.java @@ -0,0 +1,33 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +/** + * A metric with labels associated. Values for the associated labels can be provided to access the + * underlying metric. + * + * @param The type of metric the labels are applied to. + */ +public interface LabelledMetric { + + /** + * Returns a metric tagged with the specified label values. + * + * @param labels An array of label values in the same order as the labels when creating this + * metric. The number of values provided must match the number of labels. + * @return A metric tagged with the specified labels. + */ + T labels(String... labels); +} diff --git a/src/org/minima/system/network/base/metrics/MetricCategory.java b/src/org/minima/system/network/base/metrics/MetricCategory.java new file mode 100644 index 000000000..977cba2bb --- /dev/null +++ b/src/org/minima/system/network/base/metrics/MetricCategory.java @@ -0,0 +1,45 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +import java.util.Optional; + +/** + * A MetricCategory is used to group related metrics. Every metric belongs to one and only one + * MetricCategory. + * + *

Categories must be registered with the {@link MetricCategoryRegistry} during plugin + * initialisation. + */ +public interface MetricCategory { + + /** + * Gets the name of this MetricCategory. + * + * @return The name of this MetricCategory. + */ + String getName(); + + /** + * Gets the application-specific MetricCategory prefix. An empty Optional may be returned if this + * category is not application specific. + * + *

The prefix, if present, is prepended to the category name when creating a single combined + * name for metrics. + * + * @return An optional application prefix. + */ + Optional getApplicationPrefix(); +} diff --git a/src/org/minima/system/network/base/metrics/MetricsSystem.java b/src/org/minima/system/network/base/metrics/MetricsSystem.java new file mode 100644 index 000000000..4537c747b --- /dev/null +++ b/src/org/minima/system/network/base/metrics/MetricsSystem.java @@ -0,0 +1,122 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +//package org.hyperledger.besu.plugin.services; +package org.minima.system.network.base.metrics; + +// import org.hyperledger.besu.plugin.services.metrics.Counter; +// import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +// import org.hyperledger.besu.plugin.services.metrics.MetricCategory; +// import org.hyperledger.besu.plugin.services.metrics.OperationTimer; + +import java.util.function.DoubleSupplier; +import java.util.function.IntSupplier; +import java.util.function.LongSupplier; + +/** An interface for creating various Metrics components. */ +public interface MetricsSystem { + + /** + * Creates a Counter. + * + * @param category The {@link MetricCategory} this counter is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @return The created Counter instance. + */ + default Counter createCounter( + final MetricCategory category, final String name, final String help) { + return createLabelledCounter(category, name, help, new String[0]).labels(); + } + + /** + * Creates a Counter with assigned labels. + * + * @param category The {@link MetricCategory} this counter is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @param labelNames An array of labels to assign to the Counter. + * @return The created LabelledMetric instance. + */ + LabelledMetric createLabelledCounter( + MetricCategory category, String name, String help, String... labelNames); + + /** + * Creates a Timer. + * + * @param category The {@link MetricCategory} this timer is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @return The created Timer instance. + */ + default OperationTimer createTimer( + final MetricCategory category, final String name, final String help) { + return createLabelledTimer(category, name, help, new String[0]).labels(); + } + + /** + * Creates a Timer with assigned labels. + * + * @param category The {@link MetricCategory} this timer is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @param labelNames An array of labels to assign to the Timer. + * @return The created LabelledMetric instance. + */ + LabelledMetric createLabelledTimer( + MetricCategory category, String name, String help, String... labelNames); + + /** + * Creates a gauge for displaying double vales. A gauge is a metric to report the current value. + * The metric value may go up or down. + * + * @param category The {@link MetricCategory} this gauge is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @param valueSupplier A supplier for the double value to be presented. + */ + void createGauge(MetricCategory category, String name, String help, DoubleSupplier valueSupplier); + + /** + * Creates a gauge for displaying integer values. + * + * @param category The {@link MetricCategory} this gauge is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @param valueSupplier A supplier for the integer value to be presented. + */ + default void createIntegerGauge( + final MetricCategory category, + final String name, + final String help, + final IntSupplier valueSupplier) { + createGauge(category, name, help, () -> (double) valueSupplier.getAsInt()); + } + + /** + * Creates a gauge for displaying long values. + * + * @param category The {@link MetricCategory} this gauge is assigned to. + * @param name A name for this metric. + * @param help A human readable description of the metric. + * @param valueSupplier A supplier for the long value to be presented. + */ + default void createLongGauge( + final MetricCategory category, + final String name, + final String help, + final LongSupplier valueSupplier) { + createGauge(category, name, help, () -> (double) valueSupplier.getAsLong()); + } +} diff --git a/src/org/minima/system/network/base/metrics/OperationTimer.java b/src/org/minima/system/network/base/metrics/OperationTimer.java new file mode 100644 index 000000000..ae8111166 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/OperationTimer.java @@ -0,0 +1,45 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +import java.io.Closeable; + +/** A timer metric that records duration of operations for metrics purposes. */ +public interface OperationTimer { + + /** + * Starts the timer. + * + * @return The produced TimingContext, which must be stopped or closed when the operation being + * timed has completed. + */ + TimingContext startTimer(); + + /** An interface for stopping the timer and returning elapsed time. */ + interface TimingContext extends Closeable { + + /** + * Stops the timer and returns the elapsed time. + * + * @return Elapsed time in seconds. + */ + double stopTimer(); + + @Override + default void close() { + stopTimer(); + } + } +} diff --git a/src/org/minima/system/network/base/metrics/TekuMetricCategory.java b/src/org/minima/system/network/base/metrics/TekuMetricCategory.java new file mode 100644 index 000000000..1b6f6db91 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/TekuMetricCategory.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.metrics; + +import java.util.Optional; + +public enum TekuMetricCategory implements MetricCategory { + BEACON("beacon"), + EVENTBUS("eventbus"), + EXECUTOR("executor"), + LIBP2P("libp2p"), + NETWORK("network"), + STORAGE("storage"), + STORAGE_HOT_DB("storage_hot"), + STORAGE_FINALIZED_DB("storage_finalized"), + REMOTE_VALIDATOR("remote_validator"), + VALIDATOR("validator"), + VALIDATOR_PERFORMANCE("validator_performance"); + + private final String name; + + TekuMetricCategory(final String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public Optional getApplicationPrefix() { + return Optional.empty(); + } +} diff --git a/src/org/minima/system/network/base/peer/NodeId.java b/src/org/minima/system/network/base/peer/NodeId.java new file mode 100644 index 000000000..e9469543b --- /dev/null +++ b/src/org/minima/system/network/base/peer/NodeId.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public abstract class NodeId { + + public abstract byte[] toBytes(); + + public abstract String toBase58(); + + @Override + public final int hashCode() { + return toBytes().hashCode(); + } + + @Override + public final boolean equals(final Object obj) { + if (!(obj instanceof NodeId)) { + return false; + } + return toBytes().equals(((NodeId) obj).toBytes()); + } + + @Override + public final String toString() { + return toBase58(); + } +} From cfed9eaa848c2e98292059673a3042638f3c626a Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 8 Apr 2021 17:05:40 +0100 Subject: [PATCH 05/55] ssz: added teku ssz fork within minima. --- .../base/ssz/AbstractSszCollection.java | 59 ++ .../base/ssz/AbstractSszCollectionSchema.java | 347 ++++++++++++ .../base/ssz/AbstractSszComposite.java | 150 +++++ .../base/ssz/AbstractSszContainerSchema.java | 325 +++++++++++ .../ssz/AbstractSszImmutableContainer.java | 103 ++++ .../ssz/AbstractSszMutableCollection.java | 69 +++ .../base/ssz/AbstractSszMutableComposite.java | 234 ++++++++ .../base/ssz/AbstractSszPrimitive.java | 75 +++ .../base/ssz/AbstractSszPrimitiveSchema.java | 119 ++++ .../base/ssz/AbstractSszVectorSchema.java | 187 ++++++ .../network/base/ssz/ArrayIntCache.java | 114 ++++ .../network/base/ssz/BitvectorImpl.java | 143 +++++ .../system/network/base/ssz/BranchNode.java | 118 ++++ .../system/network/base/ssz/Bytes4.java | 110 ++++ .../minima/system/network/base/ssz/Cache.java | 62 ++ .../system/network/base/ssz/Container3.java | 51 ++ .../network/base/ssz/ContainerSchema3.java | 72 +++ .../system/network/base/ssz/GIndexUtil.java | 311 ++++++++++ .../system/network/base/ssz/IntCache.java | 75 +++ .../base/ssz/InvalidValueSchemaException.java | 21 + .../system/network/base/ssz/LeafDataNode.java | 27 + .../system/network/base/ssz/LeafNode.java | 98 ++++ .../system/network/base/ssz/Merkleizable.java | 29 + .../system/network/base/ssz/NoopIntCache.java | 46 ++ .../system/network/base/ssz/SchemaUtils.java | 45 ++ .../base/ssz/SimpleOffsetSerializable.java | 35 ++ .../network/base/ssz/SimpleSszReader.java | 60 ++ .../system/network/base/ssz/SszBit.java | 37 ++ .../system/network/base/ssz/SszBitvector.java | 62 ++ .../network/base/ssz/SszBitvectorImpl.java | 105 ++++ .../network/base/ssz/SszBitvectorSchema.java | 40 ++ .../base/ssz/SszBitvectorSchemaImpl.java | 81 +++ .../system/network/base/ssz/SszByte.java | 32 ++ .../network/base/ssz/SszByteArrayWriter.java | 43 ++ .../network/base/ssz/SszByteVector.java | 43 ++ .../network/base/ssz/SszByteVectorImpl.java | 70 +++ .../network/base/ssz/SszByteVectorSchema.java | 29 + .../base/ssz/SszByteVectorSchemaImpl.java | 65 +++ .../system/network/base/ssz/SszBytes32.java | 29 + .../network/base/ssz/SszBytes32Vector.java | 23 + .../base/ssz/SszBytes32VectorImpl.java | 39 ++ .../base/ssz/SszBytes32VectorSchema.java | 27 + .../base/ssz/SszBytes32VectorSchemaImpl.java | 37 ++ .../system/network/base/ssz/SszBytes4.java | 29 + .../network/base/ssz/SszCollection.java | 56 ++ .../network/base/ssz/SszCollectionSchema.java | 57 ++ .../system/network/base/ssz/SszComposite.java | 42 ++ .../network/base/ssz/SszCompositeSchema.java | 88 +++ .../system/network/base/ssz/SszContainer.java | 33 ++ .../network/base/ssz/SszContainerImpl.java | 87 +++ .../network/base/ssz/SszContainerSchema.java | 97 ++++ .../system/network/base/ssz/SszData.java | 59 ++ .../base/ssz/SszDeserializeException.java | 21 + .../network/base/ssz/SszLengthBounds.java | 116 ++++ .../base/ssz/SszMutableBytes32Vector.java | 24 + .../base/ssz/SszMutableBytes32VectorImpl.java | 47 ++ .../network/base/ssz/SszMutableComposite.java | 67 +++ .../network/base/ssz/SszMutableContainer.java | 21 + .../base/ssz/SszMutableContainerImpl.java | 70 +++ .../network/base/ssz/SszMutableData.java | 45 ++ .../ssz/SszMutablePrimitiveCollection.java | 44 ++ .../base/ssz/SszMutablePrimitiveVector.java | 30 + .../ssz/SszMutablePrimitiveVectorImpl.java | 50 ++ .../base/ssz/SszMutableRefComposite.java | 31 + .../base/ssz/SszMutableRefContainer.java | 28 + .../network/base/ssz/SszMutableRefVector.java | 35 ++ .../network/base/ssz/SszMutableVector.java | 27 + .../base/ssz/SszMutableVectorImpl.java | 63 +++ .../network/base/ssz/SszNodeTemplate.java | 208 +++++++ .../system/network/base/ssz/SszPrimitive.java | 40 ++ .../base/ssz/SszPrimitiveCollection.java | 54 ++ .../ssz/SszPrimitiveCollectionSchema.java | 50 ++ .../network/base/ssz/SszPrimitiveSchema.java | 29 + .../network/base/ssz/SszPrimitiveSchemas.java | 237 ++++++++ .../network/base/ssz/SszPrimitiveVector.java | 25 + .../base/ssz/SszPrimitiveVectorImpl.java | 40 ++ .../base/ssz/SszPrimitiveVectorSchema.java | 53 ++ .../ssz/SszPrimitiveVectorSchemaImpl.java | 41 ++ .../system/network/base/ssz/SszReader.java | 53 ++ .../system/network/base/ssz/SszSchema.java | 92 +++ .../network/base/ssz/SszSchemaHints.java | 71 +++ .../system/network/base/ssz/SszSuperNode.java | 176 ++++++ .../system/network/base/ssz/SszType.java | 78 +++ .../system/network/base/ssz/SszUInt64.java | 33 ++ .../system/network/base/ssz/SszVector.java | 30 + .../network/base/ssz/SszVectorImpl.java | 76 +++ .../network/base/ssz/SszVectorSchema.java | 54 ++ .../network/base/ssz/SszVectorSchemaImpl.java | 42 ++ .../system/network/base/ssz/SszWriter.java | 29 + .../network/base/ssz/TillIndexVisitor.java | 46 ++ .../system/network/base/ssz/TreeNode.java | 157 ++++++ .../system/network/base/ssz/TreeNodeImpl.java | 135 +++++ .../system/network/base/ssz/TreeUpdates.java | 199 +++++++ .../system/network/base/ssz/TreeUtil.java | 200 +++++++ .../system/network/base/ssz/TreeVisitor.java | 25 + .../system/network/base/ssz/UInt64.java | 530 ++++++++++++++++++ 96 files changed, 7917 insertions(+) create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszCollection.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszCollectionSchema.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszComposite.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszContainerSchema.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszImmutableContainer.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszMutableCollection.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszMutableComposite.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszPrimitive.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszPrimitiveSchema.java create mode 100644 src/org/minima/system/network/base/ssz/AbstractSszVectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/ArrayIntCache.java create mode 100644 src/org/minima/system/network/base/ssz/BitvectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/BranchNode.java create mode 100644 src/org/minima/system/network/base/ssz/Bytes4.java create mode 100644 src/org/minima/system/network/base/ssz/Cache.java create mode 100644 src/org/minima/system/network/base/ssz/Container3.java create mode 100644 src/org/minima/system/network/base/ssz/ContainerSchema3.java create mode 100644 src/org/minima/system/network/base/ssz/GIndexUtil.java create mode 100644 src/org/minima/system/network/base/ssz/IntCache.java create mode 100644 src/org/minima/system/network/base/ssz/InvalidValueSchemaException.java create mode 100644 src/org/minima/system/network/base/ssz/LeafDataNode.java create mode 100644 src/org/minima/system/network/base/ssz/LeafNode.java create mode 100644 src/org/minima/system/network/base/ssz/Merkleizable.java create mode 100644 src/org/minima/system/network/base/ssz/NoopIntCache.java create mode 100644 src/org/minima/system/network/base/ssz/SchemaUtils.java create mode 100644 src/org/minima/system/network/base/ssz/SimpleOffsetSerializable.java create mode 100644 src/org/minima/system/network/base/ssz/SimpleSszReader.java create mode 100644 src/org/minima/system/network/base/ssz/SszBit.java create mode 100644 src/org/minima/system/network/base/ssz/SszBitvector.java create mode 100644 src/org/minima/system/network/base/ssz/SszBitvectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszBitvectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszBitvectorSchemaImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszByte.java create mode 100644 src/org/minima/system/network/base/ssz/SszByteArrayWriter.java create mode 100644 src/org/minima/system/network/base/ssz/SszByteVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszByteVectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszByteVectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszByteVectorSchemaImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes32.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes32Vector.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes32VectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes32VectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes32VectorSchemaImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszBytes4.java create mode 100644 src/org/minima/system/network/base/ssz/SszCollection.java create mode 100644 src/org/minima/system/network/base/ssz/SszCollectionSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszComposite.java create mode 100644 src/org/minima/system/network/base/ssz/SszCompositeSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszContainer.java create mode 100644 src/org/minima/system/network/base/ssz/SszContainerImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszContainerSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszData.java create mode 100644 src/org/minima/system/network/base/ssz/SszDeserializeException.java create mode 100644 src/org/minima/system/network/base/ssz/SszLengthBounds.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableBytes32Vector.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableBytes32VectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableComposite.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableContainer.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableContainerImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableData.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutablePrimitiveCollection.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutablePrimitiveVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutablePrimitiveVectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableRefComposite.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableRefContainer.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableRefVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszMutableVectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszNodeTemplate.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitive.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveCollection.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveCollectionSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveSchemas.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveVectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchemaImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszReader.java create mode 100644 src/org/minima/system/network/base/ssz/SszSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszSchemaHints.java create mode 100644 src/org/minima/system/network/base/ssz/SszSuperNode.java create mode 100644 src/org/minima/system/network/base/ssz/SszType.java create mode 100644 src/org/minima/system/network/base/ssz/SszUInt64.java create mode 100644 src/org/minima/system/network/base/ssz/SszVector.java create mode 100644 src/org/minima/system/network/base/ssz/SszVectorImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszVectorSchema.java create mode 100644 src/org/minima/system/network/base/ssz/SszVectorSchemaImpl.java create mode 100644 src/org/minima/system/network/base/ssz/SszWriter.java create mode 100644 src/org/minima/system/network/base/ssz/TillIndexVisitor.java create mode 100644 src/org/minima/system/network/base/ssz/TreeNode.java create mode 100644 src/org/minima/system/network/base/ssz/TreeNodeImpl.java create mode 100644 src/org/minima/system/network/base/ssz/TreeUpdates.java create mode 100644 src/org/minima/system/network/base/ssz/TreeUtil.java create mode 100644 src/org/minima/system/network/base/ssz/TreeVisitor.java create mode 100644 src/org/minima/system/network/base/ssz/UInt64.java diff --git a/src/org/minima/system/network/base/ssz/AbstractSszCollection.java b/src/org/minima/system/network/base/ssz/AbstractSszCollection.java new file mode 100644 index 000000000..e302be743 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszCollection.java @@ -0,0 +1,59 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Supplier; +// import tech.pegasys.teku.ssz.SszCollection; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszCollectionSchema; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public abstract class AbstractSszCollection + extends AbstractSszComposite implements SszCollection { + + protected AbstractSszCollection( + SszCompositeSchema schema, Supplier lazyBackingNode) { + super(schema, lazyBackingNode); + } + + protected AbstractSszCollection(SszCompositeSchema schema, TreeNode backingNode) { + super(schema, backingNode); + } + + protected AbstractSszCollection( + SszCompositeSchema schema, TreeNode backingNode, IntCache cache) { + super(schema, backingNode, cache); + } + + @SuppressWarnings("unchecked") + @Override + public SszCollectionSchema getSchema() { + return (SszCollectionSchema) super.getSchema(); + } + + @SuppressWarnings("unchecked") + @Override + protected SszElementT getImpl(int index) { + SszCollectionSchema type = + (SszCollectionSchema) this.getSchema(); + SszSchema elementType = type.getElementSchema(); + TreeNode node = + getBackingNode().get(type.getChildGeneralizedIndex(index / type.getElementsPerChunk())); + return (SszElementT) + elementType.createFromBackingNode(node, index % type.getElementsPerChunk()); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszCollectionSchema.java b/src/org/minima/system/network/base/ssz/AbstractSszCollectionSchema.java new file mode 100644 index 000000000..44913ca43 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszCollectionSchema.java @@ -0,0 +1,347 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.lang.Integer.min; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.tuweni.bytes.Bytes; + +import org.minima.system.network.base.ssz.SszSchemaHints.SszSuperNodeHint; + +// import tech.pegasys.teku.ssz.SszCollection; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints.SszSuperNodeHint; +// import tech.pegasys.teku.ssz.schema.SszType; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.SszNodeTemplate; +// import tech.pegasys.teku.ssz.tree.SszSuperNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +/** Type of homogeneous collections (like List and Vector) */ +public abstract class AbstractSszCollectionSchema< + SszElementT extends SszData, SszCollectionT extends SszCollection> + implements SszCompositeSchema { + + private final long maxLength; + private final SszSchema elementSchema; + private final SszSchemaHints hints; + protected final Supplier elementSszSupernodeTemplate = + Suppliers.memoize(() -> SszNodeTemplate.createFromType(getElementSchema())); + private volatile TreeNode defaultTree; + + protected AbstractSszCollectionSchema( + long maxLength, SszSchema elementSchema, SszSchemaHints hints) { + checkArgument(maxLength >= 0); + this.maxLength = maxLength; + this.elementSchema = elementSchema; + this.hints = hints; + } + + protected abstract TreeNode createDefaultTree(); + + @Override + public TreeNode getDefaultTree() { + if (defaultTree == null) { + this.defaultTree = createDefaultTree(); + } + return defaultTree; + } + + @Override + public long getMaxLength() { + return maxLength; + } + + public SszSchema getElementSchema() { + return elementSchema; + } + + @Override + public SszSchema getChildSchema(int index) { + if (index >= maxLength) { + throw new IndexOutOfBoundsException("Child index > maxLength"); + } + return getElementSchema(); + } + + @Override + public int getElementsPerChunk() { + SszSchema elementSchema = getElementSchema(); + return elementSchema.isPrimitive() + ? (256 / ((SszPrimitiveSchema) elementSchema).getBitsSize()) + : 1; + } + + protected int getSszElementBitSize() { + SszSchema elementSchema = getElementSchema(); + return elementSchema.isPrimitive() + ? ((SszPrimitiveSchema) elementSchema).getBitsSize() + : elementSchema.getSszFixedPartSize() * 8; + } + + protected int getVariablePartSize(TreeNode vectorNode, int length) { + if (isFixedSize()) { + return 0; + } else { + int size = 0; + for (int i = 0; i < length; i++) { + size += getElementSchema().getSszSize(vectorNode.get(getChildGeneralizedIndex(i))); + } + return size; + } + } + + /** + * Serializes {@code elementsCount} from the content of this collection + * + * @param vectorNode for a {@link SszVectorSchemaImpl} type - the node itself, for a {@link + * SszListSchemaImpl} - the left sibling node of list size node + */ + public int sszSerializeVector(TreeNode vectorNode, SszWriter writer, int elementsCount) { + if (getElementSchema().isFixedSize()) { + return sszSerializeFixedVectorFast(vectorNode, writer, elementsCount); + } else { + return sszSerializeVariableVector(vectorNode, writer, elementsCount); + } + } + + private int sszSerializeFixedVectorFast( + TreeNode vectorNode, SszWriter writer, int elementsCount) { + if (elementsCount == 0) { + return 0; + } + int nodesCount = getChunks(elementsCount); + int[] bytesCnt = new int[1]; + TreeUtil.iterateLeavesData( + vectorNode, + getChildGeneralizedIndex(0), + getChildGeneralizedIndex(nodesCount - 1), + leafData -> { + writer.write(leafData); + bytesCnt[0] += leafData.size(); + }); + return bytesCnt[0]; + } + + private int sszSerializeVariableVector(TreeNode vectorNode, SszWriter writer, int elementsCount) { + SszSchema elementType = getElementSchema(); + int variableOffset = SSZ_LENGTH_SIZE * elementsCount; + for (int i = 0; i < elementsCount; i++) { + TreeNode childSubtree = vectorNode.get(getChildGeneralizedIndex(i)); + int childSize = elementType.getSszSize(childSubtree); + writer.write(SszType.sszLengthToBytes(variableOffset)); + variableOffset += childSize; + } + for (int i = 0; i < elementsCount; i++) { + TreeNode childSubtree = vectorNode.get(getChildGeneralizedIndex(i)); + elementType.sszSerializeTree(childSubtree, writer); + } + return variableOffset; + } + + protected DeserializedData sszDeserializeVector(SszReader reader) { + if (getElementSchema().isFixedSize()) { + Optional sszSuperNodeHint = getHints().getHint(SszSuperNodeHint.class); + if (sszSuperNodeHint.isPresent()) { + return sszDeserializeSupernode(reader, sszSuperNodeHint.get().getDepth()); + } else { + return sszDeserializeFixed(reader); + } + } else { + return sszDeserializeVariable(reader); + } + } + + private DeserializedData sszDeserializeSupernode(SszReader reader, int supernodeDepth) { + SszNodeTemplate template = elementSszSupernodeTemplate.get(); + int sszSize = reader.getAvailableBytes(); + if (sszSize % template.getSszLength() != 0) { + throw new SszDeserializeException("Ssz length is not multiple of element length"); + } + int elementsCount = sszSize / template.getSszLength(); + int chunkSize = (1 << supernodeDepth) * template.getSszLength(); + int bytesRemain = sszSize; + List sszNodes = new ArrayList<>(bytesRemain / chunkSize + 1); + while (bytesRemain > 0) { + int toRead = min(bytesRemain, chunkSize); + bytesRemain -= toRead; + Bytes bytes = reader.read(toRead); + SszSuperNode node = new SszSuperNode(supernodeDepth, template, bytes); + sszNodes.add(node); + } + TreeNode tree = + TreeUtil.createTree( + sszNodes, + new SszSuperNode(supernodeDepth, template, Bytes.EMPTY), + treeDepth() - supernodeDepth); + return new DeserializedData(tree, elementsCount); + } + + private DeserializedData sszDeserializeFixed(SszReader reader) { + int bytesSize = reader.getAvailableBytes(); + checkSsz( + bytesSize % getElementSchema().getSszFixedPartSize() == 0, + "SSZ sequence length is not multiple of fixed element size"); + int elementBitSize = getSszElementBitSize(); + if (elementBitSize >= 8) { + checkSsz( + bytesSize * 8 / elementBitSize <= getMaxLength(), + "SSZ sequence length exceeds max type length"); + } else { + // preliminary rough check + checkSsz( + (bytesSize - 1) * 8 / elementBitSize <= getMaxLength(), + "SSZ sequence length exceeds max type length"); + } + if (getElementSchema() instanceof AbstractSszPrimitiveSchema) { + int bytesRemain = bytesSize; + List childNodes = new ArrayList<>(bytesRemain / LeafNode.MAX_BYTE_SIZE + 1); + while (bytesRemain > 0) { + int toRead = min(bytesRemain, LeafNode.MAX_BYTE_SIZE); + bytesRemain -= toRead; + Bytes bytes = reader.read(toRead); + LeafNode node = LeafNode.create(bytes); + childNodes.add(node); + } + + Optional lastByte; + if (childNodes.isEmpty()) { + lastByte = Optional.empty(); + } else { + Bytes lastNodeData = childNodes.get(childNodes.size() - 1).getData(); + lastByte = Optional.of(lastNodeData.get(lastNodeData.size() - 1)); + } + return new DeserializedData( + TreeUtil.createTree(childNodes, treeDepth()), bytesSize * 8 / elementBitSize, lastByte); + } else { + int elementsCount = bytesSize / getElementSchema().getSszFixedPartSize(); + List childNodes = new ArrayList<>(); + for (int i = 0; i < elementsCount; i++) { + try (SszReader sszReader = reader.slice(getElementSchema().getSszFixedPartSize())) { + TreeNode childNode = getElementSchema().sszDeserializeTree(sszReader); + childNodes.add(childNode); + } + } + return new DeserializedData(TreeUtil.createTree(childNodes, treeDepth()), elementsCount); + } + } + + private DeserializedData sszDeserializeVariable(SszReader reader) { + final int endOffset = reader.getAvailableBytes(); + final List childNodes = new ArrayList<>(); + if (endOffset > 0) { + int firstElementOffset = SszType.sszBytesToLength(reader.read(SSZ_LENGTH_SIZE)); + checkSsz(firstElementOffset % SSZ_LENGTH_SIZE == 0, "Invalid first element offset"); + int elementsCount = firstElementOffset / SSZ_LENGTH_SIZE; + checkSsz(elementsCount <= getMaxLength(), "SSZ sequence length exceeds max type length"); + List elementOffsets = new ArrayList<>(elementsCount + 1); + elementOffsets.add(firstElementOffset); + for (int i = 1; i < elementsCount; i++) { + int offset = SszType.sszBytesToLength(reader.read(SSZ_LENGTH_SIZE)); + elementOffsets.add(offset); + } + elementOffsets.add(endOffset); + + List elementSizes = + IntStream.range(0, elementOffsets.size() - 1) + .map(i -> elementOffsets.get(i + 1) - elementOffsets.get(i)) + .boxed() + .collect(Collectors.toList()); + + if (elementSizes.stream().anyMatch(s -> s < 0)) { + throw new SszDeserializeException("Invalid SSZ: wrong child offsets"); + } + + for (int elementSize : elementSizes) { + try (SszReader sszReader = reader.slice(elementSize)) { + childNodes.add(getElementSchema().sszDeserializeTree(sszReader)); + } + } + } + return new DeserializedData(TreeUtil.createTree(childNodes, treeDepth()), childNodes.size()); + } + + protected static void checkSsz(boolean condition, String error) { + if (!condition) { + throw new SszDeserializeException(error); + } + } + + public SszSchemaHints getHints() { + return hints; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AbstractSszCollectionSchema that = (AbstractSszCollectionSchema) o; + return maxLength == that.maxLength && elementSchema.equals(that.elementSchema); + } + + @Override + public int hashCode() { + return Objects.hash(maxLength, elementSchema); + } + + protected static class DeserializedData { + + private final TreeNode dataTree; + private final int childrenCount; + private final Optional lastSszByte; + + public DeserializedData(TreeNode dataTree, int childrenCount) { + this(dataTree, childrenCount, Optional.empty()); + } + + public DeserializedData(TreeNode dataTree, int childrenCount, Optional lastSszByte) { + this.dataTree = dataTree; + this.childrenCount = childrenCount; + this.lastSszByte = lastSszByte; + } + + public TreeNode getDataTree() { + return dataTree; + } + + public int getChildrenCount() { + return childrenCount; + } + + public Optional getLastSszByte() { + return lastSszByte; + } + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszComposite.java b/src/org/minima/system/network/base/ssz/AbstractSszComposite.java new file mode 100644 index 000000000..36d52f235 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszComposite.java @@ -0,0 +1,150 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Optional; +import java.util.function.Supplier; +// import tech.pegasys.teku.ssz.SszComposite; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.cache.ArrayIntCache; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** + * Base backing class for immutable composite ssz structures (lists, vectors, containers) + * + *

It caches it's child ssz instances so that if the underlying tree nodes are not changed (in + * the corresponding mutable classes) the instances are not recreated from tree nodes on later + * access. + * + *

Though internally this class has a mutable cache it may be thought of as immutable instance + * and used safely across threads + * + * @param the type of children. For heterogeneous composites (like container) this type + * would be just generic {@link SszData} + */ +public abstract class AbstractSszComposite + implements SszComposite { + + private final IntCache childrenViewCache; + private int sizeCache = -1; + private final SszCompositeSchema schema; + private final Supplier backingNode; + + /** Creates an instance from a schema and a backing node */ + protected AbstractSszComposite(SszCompositeSchema schema, Supplier lazyBackingNode) { + this(schema, lazyBackingNode, Optional.empty()); + } + + protected AbstractSszComposite(SszCompositeSchema schema, TreeNode backingNode) { + this(schema, () -> backingNode, Optional.empty()); + } + + /** + * Creates an instance from a schema and a backing node. + * + *

{@link SszData} instances cache is supplied for optimization to shortcut children creation + * from backing nodes. The cache should correspond to the supplied backing tree. + */ + protected AbstractSszComposite( + SszCompositeSchema schema, TreeNode backingNode, IntCache cache) { + this(schema, () -> backingNode, Optional.of(cache)); + } + + protected AbstractSszComposite( + SszCompositeSchema schema, + Supplier lazyBackingNode, + Optional> cache) { + this.schema = schema; + this.backingNode = lazyBackingNode; + this.childrenViewCache = cache.orElseGet(this::createCache); + } + + /** + * 'Transfers' the cache to a new Cache instance eliminating all the cached values from the + * current view cache. This is made under assumption that the ssz data instance this cache is + * transferred to would be used further with high probability and this ssz data instance would be + * either GCed or used with lower probability + */ + IntCache transferCache() { + return childrenViewCache.transfer(); + } + + /** + * Creates a new empty children cache. Could be overridden by subclasses for fine tuning of the + * initial cache size + */ + protected IntCache createCache() { + return new ArrayIntCache<>(); + } + + @Override + public final SszChildT get(int index) { + return childrenViewCache.getInt(index, this::getImplWithIndexCheck); + } + + private SszChildT getImplWithIndexCheck(int index) { + checkIndex(index); + return getImpl(index); + } + + /** Cache miss fallback child getter. This is where child is created from the backing tree node */ + protected abstract SszChildT getImpl(int index); + + @Override + public SszCompositeSchema getSchema() { + return schema; + } + + @Override + public TreeNode getBackingNode() { + return backingNode.get(); + } + + @Override + public final int size() { + if (sizeCache == -1) { + sizeCache = sizeImpl(); + } + return sizeCache; + } + + /** Size value is normally cached. This method calculates the size from backing tree */ + protected abstract int sizeImpl(); + + /** + * Checks the child index + * + * @throws IndexOutOfBoundsException if index is invalid + */ + protected abstract void checkIndex(int index); + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SszComposite)) { + return false; + } + SszComposite that = (SszComposite) o; + return getSchema().equals(that.getSchema()) && hashTreeRoot().equals(that.hashTreeRoot()); + } + + @Override + public int hashCode() { + return hashTreeRoot().slice(28).toInt(); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszContainerSchema.java b/src/org/minima/system/network/base/ssz/AbstractSszContainerSchema.java new file mode 100644 index 000000000..b396d7209 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszContainerSchema.java @@ -0,0 +1,325 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Queue; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +// import tech.pegasys.teku.ssz.SszContainer; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.schema.SszContainerSchema; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.schema.SszType; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszLengthBounds; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +public abstract class AbstractSszContainerSchema + implements SszContainerSchema { + + public static class NamedSchema { + private final String name; + private final SszSchema schema; + + public static NamedSchema of(String name, SszSchema schema) { + return new NamedSchema<>(name, schema); + } + + private NamedSchema(String name, SszSchema schema) { + this.name = name; + this.schema = schema; + } + + public String getName() { + return name; + } + + public SszSchema getSchema() { + return schema; + } + } + + protected static NamedSchema namedSchema( + String fieldName, SszSchema schema) { + return new NamedSchema<>(fieldName, schema); + } + + private final String containerName; + private final List childrenNames = new ArrayList<>(); + private final Map childrenNamesToFieldIndex = new HashMap<>(); + private final List> childrenSchemas; + private final TreeNode defaultTree; + private final long treeWidth; + + protected AbstractSszContainerSchema(String name, List> childrenSchemas) { + this.containerName = name; + for (int i = 0; i < childrenSchemas.size(); i++) { + final NamedSchema childSchema = childrenSchemas.get(i); + if (childrenNamesToFieldIndex.put(childSchema.getName(), i) != null) { + throw new IllegalArgumentException( + "Duplicate field name detected for field " + childSchema.getName() + " at index " + i); + } + childrenNames.add(childSchema.getName()); + } + this.childrenSchemas = + childrenSchemas.stream().map(NamedSchema::getSchema).collect(Collectors.toList()); + this.defaultTree = createDefaultTree(); + this.treeWidth = SszContainerSchema.super.treeWidth(); + } + + protected AbstractSszContainerSchema(List> childrenSchemas) { + this.containerName = ""; + for (int i = 0; i < childrenSchemas.size(); i++) { + final String name = "field-" + i; + childrenNamesToFieldIndex.put(name, i); + childrenNames.add(name); + } + this.childrenSchemas = childrenSchemas; + this.defaultTree = createDefaultTree(); + this.treeWidth = SszContainerSchema.super.treeWidth(); + } + + @Override + public TreeNode createTreeFromFieldValues(List fieldValues) { + checkArgument(fieldValues.size() == getFieldsCount(), "Wrong number of filed values"); + return TreeUtil.createTree( + fieldValues.stream().map(SszData::getBackingNode).collect(Collectors.toList())); + } + + @Override + public C getDefault() { + return createFromBackingNode(getDefaultTree()); + } + + @Override + public TreeNode getDefaultTree() { + return defaultTree; + } + + @Override + public long treeWidth() { + return treeWidth; + } + + private TreeNode createDefaultTree() { + List defaultChildren = new ArrayList<>((int) getMaxLength()); + for (int i = 0; i < getFieldsCount(); i++) { + defaultChildren.add(getChildSchema(i).getDefault().getBackingNode()); + } + return TreeUtil.createTree(defaultChildren); + } + + @Override + public SszSchema getChildSchema(int index) { + return childrenSchemas.get(index); + } + + /** + * Get the index of a field by name + * + * @param fieldName + * @return The index if it exists, otherwise -1 + */ + @Override + public int getFieldIndex(String fieldName) { + final Integer index = childrenNamesToFieldIndex.get(fieldName); + return index == null ? -1 : index; + } + + @Override + public abstract C createFromBackingNode(TreeNode node); + + @Override + public long getMaxLength() { + return childrenSchemas.size(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AbstractSszContainerSchema that = (AbstractSszContainerSchema) o; + return childrenSchemas.equals(that.childrenSchemas); + } + + @Override + public int hashCode() { + return Objects.hash(childrenSchemas); + } + + @Override + public boolean isFixedSize() { + for (int i = 0; i < getFieldsCount(); i++) { + if (!getChildSchema(i).isFixedSize()) { + return false; + } + } + return true; + } + + @Override + public int getSszFixedPartSize() { + int size = 0; + for (int i = 0; i < getFieldsCount(); i++) { + SszSchema childType = getChildSchema(i); + size += childType.isFixedSize() ? childType.getSszFixedPartSize() : SSZ_LENGTH_SIZE; + } + return size; + } + + @Override + public int getSszVariablePartSize(TreeNode node) { + int size = 0; + for (int i = 0; i < getFieldsCount(); i++) { + SszSchema childType = getChildSchema(i); + if (!childType.isFixedSize()) { + size += childType.getSszSize(node.get(getChildGeneralizedIndex(i))); + } + } + return size; + } + + @Override + public List> getFieldSchemas() { + return childrenSchemas; + } + + @Override + public int sszSerializeTree(TreeNode node, SszWriter writer) { + int variableChildOffset = getSszFixedPartSize(); + int[] variableSizes = new int[getFieldsCount()]; + for (int i = 0; i < getFieldsCount(); i++) { + TreeNode childSubtree = node.get(getChildGeneralizedIndex(i)); + SszSchema childType = getChildSchema(i); + if (childType.isFixedSize()) { + int size = childType.sszSerializeTree(childSubtree, writer); + assert size == childType.getSszFixedPartSize(); + } else { + writer.write(SszType.sszLengthToBytes(variableChildOffset)); + int childSize = childType.getSszSize(childSubtree); + variableSizes[i] = childSize; + variableChildOffset += childSize; + } + } + for (int i = 0; i < getMaxLength(); i++) { + SszSchema childType = getChildSchema(i); + if (!childType.isFixedSize()) { + TreeNode childSubtree = node.get(getChildGeneralizedIndex(i)); + int size = childType.sszSerializeTree(childSubtree, writer); + assert size == variableSizes[i]; + } + } + return variableChildOffset; + } + + @Override + public TreeNode sszDeserializeTree(SszReader reader) { + int endOffset = reader.getAvailableBytes(); + int childCount = getFieldsCount(); + Queue fixedChildrenSubtrees = new ArrayDeque<>(childCount); + List variableChildrenOffsets = new ArrayList<>(childCount); + for (int i = 0; i < childCount; i++) { + SszSchema childType = getChildSchema(i); + if (childType.isFixedSize()) { + try (SszReader sszReader = reader.slice(childType.getSszFixedPartSize())) { + TreeNode childNode = childType.sszDeserializeTree(sszReader); + fixedChildrenSubtrees.add(childNode); + } + } else { + int childOffset = SszType.sszBytesToLength(reader.read(SSZ_LENGTH_SIZE)); + variableChildrenOffsets.add(childOffset); + } + } + + if (variableChildrenOffsets.isEmpty()) { + if (reader.getAvailableBytes() > 0) { + throw new SszDeserializeException("Invalid SSZ: unread bytes for fixed size container"); + } + } else { + if (variableChildrenOffsets.get(0) != endOffset - reader.getAvailableBytes()) { + throw new SszDeserializeException( + "First variable element offset doesn't match the end of fixed part"); + } + } + + variableChildrenOffsets.add(endOffset); + + ArrayDeque variableChildrenSizes = + new ArrayDeque<>(variableChildrenOffsets.size() - 1); + for (int i = 0; i < variableChildrenOffsets.size() - 1; i++) { + variableChildrenSizes.add( + variableChildrenOffsets.get(i + 1) - variableChildrenOffsets.get(i)); + } + + if (variableChildrenSizes.stream().anyMatch(s -> s < 0)) { + throw new SszDeserializeException("Invalid SSZ: wrong child offsets"); + } + + List childrenSubtrees = new ArrayList<>(childCount); + for (int i = 0; i < childCount; i++) { + SszSchema childType = getChildSchema(i); + if (childType.isFixedSize()) { + childrenSubtrees.add(fixedChildrenSubtrees.remove()); + } else { + try (SszReader sszReader = reader.slice(variableChildrenSizes.remove())) { + TreeNode childNode = childType.sszDeserializeTree(sszReader); + childrenSubtrees.add(childNode); + } + } + } + + return TreeUtil.createTree(childrenSubtrees); + } + + @Override + public SszLengthBounds getSszLengthBounds() { + return IntStream.range(0, getFieldsCount()) + .mapToObj(this::getChildSchema) + // dynamic sized children need 4-byte offset + .map(t -> t.getSszLengthBounds().addBytes((t.isFixedSize() ? 0 : SSZ_LENGTH_SIZE))) + // elements are not packed in containers + .map(SszLengthBounds::ceilToBytes) + .reduce(SszLengthBounds.ZERO, SszLengthBounds::add); + } + + @Override + public String getContainerName() { + return !containerName.isEmpty() ? containerName : getClass().getName(); + } + + @Override + public List getFieldNames() { + return childrenNames; + } + + @Override + public String toString() { + return getContainerName(); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszImmutableContainer.java b/src/org/minima/system/network/base/ssz/AbstractSszImmutableContainer.java new file mode 100644 index 000000000..d6af20152 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszImmutableContainer.java @@ -0,0 +1,103 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Preconditions; +import java.util.Arrays; +import java.util.Objects; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableContainer; +// import tech.pegasys.teku.ssz.cache.ArrayIntCache; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszContainerSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** Handy base class for immutable containers */ +public abstract class AbstractSszImmutableContainer extends SszContainerImpl { + + protected AbstractSszImmutableContainer( + AbstractSszContainerSchema schema) { + this(schema, schema.getDefaultTree()); + } + + protected AbstractSszImmutableContainer( + SszContainerSchema schema, TreeNode backingNode) { + super(schema, backingNode); + } + + protected AbstractSszImmutableContainer( + SszContainerSchema schema, SszData... memberValues) { + super( + schema, + schema.createTreeFromFieldValues(Arrays.asList(memberValues)), + createCache(memberValues)); + checkArgument( + memberValues.length == this.getSchema().getMaxLength(), + "Wrong number of member values: %s", + memberValues.length); + for (int i = 0; i < memberValues.length; i++) { + Preconditions.checkArgument( + memberValues[i].getSchema().equals(schema.getChildSchema(i)), + "Wrong child schema at index %s. Expected: %s, was %s", + i, + schema.getChildSchema(i), + memberValues[i].getSchema()); + } + } + + private static IntCache createCache(SszData... memberValues) { + ArrayIntCache cache = new ArrayIntCache<>(memberValues.length); + for (int i = 0; i < memberValues.length; i++) { + cache.invalidateWithNewValue(i, memberValues[i]); + } + return cache; + } + + @Override + public SszMutableContainer createWritableCopy() { + throw new UnsupportedOperationException( + "This container doesn't support mutable structure: " + getClass().getName()); + } + + @Override + public boolean isWritableSupported() { + return false; + } + + @Override + public boolean equals(Object obj) { + if (Objects.isNull(obj)) { + return false; + } + + if (this == obj) { + return true; + } + + if (!(obj instanceof AbstractSszImmutableContainer)) { + return false; + } + + AbstractSszImmutableContainer other = (AbstractSszImmutableContainer) obj; + return hashTreeRoot().equals(other.hashTreeRoot()); + } + + @Override + public int hashCode() { + return hashTreeRoot().slice(0, 4).toInt(); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszMutableCollection.java b/src/org/minima/system/network/base/ssz/AbstractSszMutableCollection.java new file mode 100644 index 000000000..5b20c471a --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszMutableCollection.java @@ -0,0 +1,69 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.schema.SszCollectionSchema; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUpdates; + +public abstract class AbstractSszMutableCollection< + SszElementT extends SszData, SszMutableElementT extends SszElementT> + extends AbstractSszMutableComposite { + + protected AbstractSszMutableCollection(AbstractSszComposite backingImmutableData) { + super(backingImmutableData); + } + + @Override + public SszCollectionSchema getSchema() { + return (SszCollectionSchema) super.getSchema(); + } + + @Override + protected TreeUpdates packChanges( + List> newChildValues, TreeNode original) { + SszCollectionSchema type = getSchema(); + SszSchema elementType = type.getElementSchema(); + int elementsPerChunk = type.getElementsPerChunk(); + + return newChildValues.stream() + .collect(Collectors.groupingBy(e -> e.getKey() / elementsPerChunk)) + .entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map( + e -> { + int nodeIndex = e.getKey(); + List> nodeVals = e.getValue(); + long gIndex = type.getChildGeneralizedIndex(nodeIndex); + // optimization: when all packed values changed no need to retrieve original node to + // merge with + TreeNode node = + nodeVals.size() == elementsPerChunk ? LeafNode.EMPTY_LEAF : original.get(gIndex); + for (Map.Entry entry : nodeVals) { + node = + elementType.updateBackingNode( + node, entry.getKey() % elementsPerChunk, entry.getValue()); + } + return new TreeUpdates.Update(gIndex, node); + }) + .collect(TreeUpdates.collector()); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszMutableComposite.java b/src/org/minima/system/network/base/ssz/AbstractSszMutableComposite.java new file mode 100644 index 000000000..1738f04b9 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszMutableComposite.java @@ -0,0 +1,234 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +// import tech.pegasys.teku.ssz.InvalidValueSchemaException; +// import tech.pegasys.teku.ssz.SszComposite; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableComposite; +// import tech.pegasys.teku.ssz.SszMutableData; +// import tech.pegasys.teku.ssz.SszMutableRefComposite; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUpdates; + +/** + * Base backing {@link SszMutableData} class for mutable composite ssz structures (lists, vectors, + * containers) + * + *

It has corresponding backing immutable {@link SszData} and the set of changed children. When + * the {@link #commitChanges()} is called a new immutable {@link SszData} instance is created where + * changes accumulated in this instance are merged with cached backing {@link SszData} instance + * which weren't changed. + * + *

If this ssz data is get by reference from its parent composite view ({@link + * SszMutableRefComposite#getByRef(int)} then all the changes are notified to the parent view (see + * {@link SszMutableComposite#setInvalidator(Consumer)} + * + *

The mutable structures based on this class are inherently NOT thread safe + */ +public abstract class AbstractSszMutableComposite< + SszChildT extends SszData, SszMutableChildT extends SszChildT> + implements SszMutableRefComposite { + + protected AbstractSszComposite backingImmutableData; + private Consumer invalidator; + private final Map childrenChanges = new HashMap<>(); + private final Map childrenRefs = new HashMap<>(); + private final Set childrenRefsChanged = new HashSet<>(); + private Integer sizeCache; + + /** Creates a new mutable instance with backing immutable data */ + protected AbstractSszMutableComposite(AbstractSszComposite backingImmutableData) { + this.backingImmutableData = backingImmutableData; + sizeCache = backingImmutableData.size(); + } + + @Override + public void set(int index, SszChildT value) { + checkIndex(index, true); + checkNotNull(value); + if (!value.getSchema().equals(getSchema().getChildSchema(index))) { + throw new InvalidValueSchemaException( + "Expected child to have schema " + + getSchema().getChildSchema(index) + + ", but value has schema " + + value.getSchema()); + } + if (childrenRefs.containsKey(index)) { + throw new IllegalStateException( + "A child couldn't be simultaneously modified by value and accessed by ref"); + } + childrenChanges.put(index, value); + sizeCache = index >= sizeCache ? index + 1 : sizeCache; + invalidate(); + } + + @Override + public SszChildT get(int index) { + checkIndex(index, false); + SszChildT ret = childrenChanges.get(index); + if (ret != null) { + return ret; + } else if (childrenRefs.containsKey(index)) { + return childrenRefs.get(index); + } else { + return backingImmutableData.get(index); + } + } + + @Override + public SszMutableChildT getByRef(int index) { + SszMutableChildT ret = childrenRefs.get(index); + if (ret == null) { + SszChildT readView = get(index); + childrenChanges.remove(index); + @SuppressWarnings("unchecked") + SszMutableChildT w = (SszMutableChildT) readView.createWritableCopy(); + if (w instanceof SszMutableComposite) { + ((SszMutableComposite) w) + .setInvalidator( + viewWrite -> { + childrenRefsChanged.add(index); + invalidate(); + }); + } + childrenRefs.put(index, w); + ret = w; + } + return ret; + } + + @Override + public SszCompositeSchema getSchema() { + return backingImmutableData.getSchema(); + } + + @Override + @SuppressWarnings("unchecked") + public void clear() { + backingImmutableData = (AbstractSszComposite) getSchema().getDefault(); + childrenChanges.clear(); + childrenRefs.clear(); + childrenRefsChanged.clear(); + sizeCache = backingImmutableData.size(); + invalidate(); + } + + @Override + public int size() { + return sizeCache; + } + + @Override + @SuppressWarnings("unchecked") + public SszComposite commitChanges() { + if (childrenChanges.isEmpty() && childrenRefsChanged.isEmpty()) { + return backingImmutableData; + } else { + IntCache cache = backingImmutableData.transferCache(); + List> changesList = + Stream.concat( + childrenChanges.entrySet().stream(), + childrenRefsChanged.stream() + .map( + idx -> + new SimpleImmutableEntry<>( + idx, + (SszChildT) + ((SszMutableData) childrenRefs.get(idx)).commitChanges()))) + .sorted(Map.Entry.comparingByKey()) + .collect(Collectors.toList()); + // pre-fill the read cache with changed values + changesList.forEach(e -> cache.invalidateWithNewValue(e.getKey(), e.getValue())); + TreeNode originalBackingTree = backingImmutableData.getBackingNode(); + TreeUpdates changes = changesToNewNodes(changesList, originalBackingTree); + TreeNode newBackingTree = originalBackingTree.updated(changes); + TreeNode finalBackingTree = doFinalTreeUpdates(newBackingTree); + return createImmutableSszComposite(finalBackingTree, cache); + } + } + + protected TreeNode doFinalTreeUpdates(TreeNode updatedTree) { + return updatedTree; + } + + /** Converts a set of changed view with their indexes to the {@link TreeUpdates} instance */ + protected TreeUpdates changesToNewNodes( + List> newChildValues, TreeNode original) { + SszCompositeSchema type = getSchema(); + int elementsPerChunk = type.getElementsPerChunk(); + if (elementsPerChunk == 1) { + return newChildValues.stream() + .map( + e -> + new TreeUpdates.Update( + type.getChildGeneralizedIndex(e.getKey()), e.getValue().getBackingNode())) + .collect(TreeUpdates.collector()); + } else { + return packChanges(newChildValues, original); + } + } + + /** + * Converts a set of changed view with their indexes to the {@link TreeUpdates} instance for views + * which support packed values (i.e. several child views per backing tree node) + */ + protected abstract TreeUpdates packChanges( + List> newChildValues, TreeNode original); + + /** + * Should be implemented by subclasses to create respectful immutable view with backing tree and + * views cache + */ + protected abstract AbstractSszComposite createImmutableSszComposite( + TreeNode backingNode, IntCache viewCache); + + @Override + public void setInvalidator(Consumer listener) { + invalidator = listener; + } + + protected void invalidate() { + if (invalidator != null) { + invalidator.accept(this); + } + } + + /** Creating nested mutable copies is not supported yet */ + @Override + public SszMutableComposite createWritableCopy() { + throw new UnsupportedOperationException( + "createWritableCopy() is now implemented for immutable SszData only"); + } + + /** + * Checks the child index for get or set + * + * @throws IndexOutOfBoundsException is index is not valid + */ + protected abstract void checkIndex(int index, boolean set); +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszPrimitive.java b/src/org/minima/system/network/base/ssz/AbstractSszPrimitive.java new file mode 100644 index 000000000..8af5e6cbb --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszPrimitive.java @@ -0,0 +1,75 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Objects; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszPrimitiveSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public abstract class AbstractSszPrimitive> + implements SszPrimitive { + private final AbstractSszPrimitiveSchema schema; + private final C value; + + protected AbstractSszPrimitive(C value, AbstractSszPrimitiveSchema schema) { + checkNotNull(value); + this.schema = schema; + this.value = value; + } + + @Override + public C get() { + return value; + } + + @Override + public AbstractSszPrimitiveSchema getSchema() { + return schema; + } + + @Override + public TreeNode getBackingNode() { + return getSchema().createBackingNode(getThis()); + } + + @SuppressWarnings("unchecked") + protected V getThis() { + return (V) this; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AbstractSszPrimitive that = (AbstractSszPrimitive) o; + return Objects.equals(get(), that.get()); + } + + @Override + public int hashCode() { + return Objects.hash(get()); + } + + @Override + public String toString() { + return get().toString(); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszPrimitiveSchema.java b/src/org/minima/system/network/base/ssz/AbstractSszPrimitiveSchema.java new file mode 100644 index 000000000..1513e1b0e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszPrimitiveSchema.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +//import static tech.pegasys.teku.ssz.tree.TreeUtil.bitsCeilToBytes; +import static org.minima.system.network.base.ssz.TreeUtil.bitsCeilToBytes; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszLengthBounds; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.LeafDataNode; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** + * Represents primitive view type + * + * @param Class of the basic view of this type + */ +public abstract class AbstractSszPrimitiveSchema< + DataT, SszDataT extends SszPrimitive> + implements SszPrimitiveSchema { + + private final int bitsSize; + + protected AbstractSszPrimitiveSchema(int bitsSize) { + checkArgument( + bitsSize > 0 && bitsSize <= 256 && 256 % bitsSize == 0, "Invalid bitsize: %s", bitsSize); + this.bitsSize = bitsSize; + } + + @Override + public int getBitsSize() { + return bitsSize; + } + + @Override + public SszDataT createFromBackingNode(TreeNode node) { + return createFromBackingNode(node, 0); + } + + @Override + public final SszDataT createFromBackingNode(TreeNode node, int internalIndex) { + assert node instanceof LeafDataNode; + return createFromLeafBackingNode((LeafDataNode) node, internalIndex); + } + + public abstract SszDataT createFromLeafBackingNode(LeafDataNode node, int internalIndex); + + public TreeNode createBackingNode(SszDataT newValue) { + return updateBackingNode(LeafNode.EMPTY_LEAF, 0, newValue); + } + + @Override + public abstract TreeNode updateBackingNode(TreeNode srcNode, int internalIndex, SszData newValue); + + private int getSSZBytesSize() { + return bitsCeilToBytes(getBitsSize()); + } + + @Override + public boolean isFixedSize() { + return true; + } + + @Override + public int getSszFixedPartSize() { + return getSSZBytesSize(); + } + + @Override + public int getSszVariablePartSize(TreeNode node) { + return 0; + } + + @Override + public int sszSerializeTree(TreeNode node, SszWriter writer) { + int sszBytesSize = getSSZBytesSize(); + final Bytes nodeData; + if (node instanceof LeafDataNode) { + // small perf optimization + nodeData = ((LeafDataNode) node).getData(); + } else { + nodeData = node.hashTreeRoot(); + } + writer.write(nodeData.toArrayUnsafe(), 0, sszBytesSize); + return sszBytesSize; + } + + @Override + public TreeNode sszDeserializeTree(SszReader reader) { + Bytes bytes = reader.read(getSSZBytesSize()); + if (reader.getAvailableBytes() > 0) { + throw new SszDeserializeException("Extra " + reader.getAvailableBytes() + " bytes found"); + } + return LeafNode.create(bytes); + } + + @Override + public SszLengthBounds getSszLengthBounds() { + return SszLengthBounds.ofBits(getBitsSize()); + } +} diff --git a/src/org/minima/system/network/base/ssz/AbstractSszVectorSchema.java b/src/org/minima/system/network/base/ssz/AbstractSszVectorSchema.java new file mode 100644 index 000000000..ea9141e0d --- /dev/null +++ b/src/org/minima/system/network/base/ssz/AbstractSszVectorSchema.java @@ -0,0 +1,187 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static java.util.Collections.emptyList; +import static org.minima.system.network.base.ssz.TreeUtil.bitsCeilToBytes; +import static org.minima.system.network.base.ssz.SszSchemaHints.SszSuperNodeHint; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszVector; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints.SszSuperNodeHint; +// import tech.pegasys.teku.ssz.schema.SszVectorSchema; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszLengthBounds; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.SszSuperNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +public abstract class AbstractSszVectorSchema< + SszElementT extends SszData, SszVectorT extends SszVector> + extends AbstractSszCollectionSchema + implements SszVectorSchema { + + private final boolean isListBacking; + + protected AbstractSszVectorSchema(SszSchema elementType, long vectorLength) { + this(elementType, vectorLength, false); + } + + protected AbstractSszVectorSchema( + SszSchema elementType, long vectorLength, boolean isListBacking) { + this(elementType, vectorLength, isListBacking, SszSchemaHints.none()); + } + + protected AbstractSszVectorSchema( + SszSchema elementSchema, + long vectorLength, + boolean isListBacking, + SszSchemaHints hints) { + super(vectorLength, elementSchema, hints); + this.isListBacking = isListBacking; + } + + @Override + public SszVectorT getDefault() { + return createFromBackingNode(getDefaultTree()); + } + + @Override + protected TreeNode createDefaultTree() { + if (isListBacking) { + Optional sszSuperNodeHint = getHints().getHint(SszSuperNodeHint.class); + if (sszSuperNodeHint.isPresent()) { + int superNodeDepth = sszSuperNodeHint.get().getDepth(); + SszSuperNode defaultSuperSszNode = + new SszSuperNode(superNodeDepth, elementSszSupernodeTemplate.get(), Bytes.EMPTY); + int binaryDepth = treeDepth() - superNodeDepth; + return TreeUtil.createTree(emptyList(), defaultSuperSszNode, binaryDepth); + } else { + return TreeUtil.createDefaultTree(maxChunks(), LeafNode.EMPTY_LEAF); + } + } else if (getElementsPerChunk() == 1) { + return TreeUtil.createDefaultTree(maxChunks(), getElementSchema().getDefaultTree()); + } else { + // packed vector + int fullZeroNodesCount = getLength() / getElementsPerChunk(); + int lastNodeElementCount = getLength() % getElementsPerChunk(); + int lastNodeSizeBytes = bitsCeilToBytes(lastNodeElementCount * getSszElementBitSize()); + Stream fullZeroNodes = + Stream.generate(() -> LeafNode.ZERO_LEAVES[32]).limit(fullZeroNodesCount); + Stream lastZeroNode = + lastNodeSizeBytes > 0 + ? Stream.of(LeafNode.ZERO_LEAVES[lastNodeSizeBytes]) + : Stream.empty(); + return TreeUtil.createTree( + Stream.concat(fullZeroNodes, lastZeroNode).collect(Collectors.toList())); + } + } + + @Override + public abstract SszVectorT createFromBackingNode(TreeNode node); + + @Override + public int getLength() { + long maxLength = getMaxLength(); + if (maxLength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Vector size too large: " + maxLength); + } + return (int) maxLength; + } + + public int getChunksCount() { + long maxChunks = maxChunks(); + if (maxChunks > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Vector size too large: " + maxChunks); + } + return (int) maxChunks; + } + + @Override + public boolean isFixedSize() { + return getElementSchema().isFixedSize(); + } + + @Override + public int getSszVariablePartSize(TreeNode node) { + return getVariablePartSize(node, getLength()); + } + + @Override + public int getSszFixedPartSize() { + int bitsPerChild = isFixedSize() ? getSszElementBitSize() : SSZ_LENGTH_SIZE * 8; + return bitsCeilToBytes(getLength() * bitsPerChild); + } + + @Override + public int sszSerializeTree(TreeNode node, SszWriter writer) { + return sszSerializeVector(node, writer, getLength()); + } + + @Override + public TreeNode sszDeserializeTree(SszReader reader) { + if (getElementSchema() == SszPrimitiveSchemas.BIT_SCHEMA) { + throw new UnsupportedOperationException( + "Bitvector deserialization is only supported by SszBitvectorSchema"); + } + + DeserializedData data = sszDeserializeVector(reader); + if (data.getChildrenCount() != getLength()) { + throw new SszDeserializeException("Invalid Vector ssz"); + } + return data.getDataTree(); + } + + @Override + public SszLengthBounds getSszLengthBounds() { + return getElementSchema() + .getSszLengthBounds() + // if elements are of dynamic size the offset size should be added for every element + .addBytes(getElementSchema().isFixedSize() ? 0 : SSZ_LENGTH_SIZE) + .mul(getLength()) + .ceilToBytes(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof AbstractSszVectorSchema)) { + return false; + } + AbstractSszVectorSchema that = (AbstractSszVectorSchema) o; + return getElementSchema().equals(that.getElementSchema()) + && getMaxLength() == that.getMaxLength(); + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + public String toString() { + return "Vector[" + getElementSchema() + ", " + getLength() + "]"; + } +} diff --git a/src/org/minima/system/network/base/ssz/ArrayIntCache.java b/src/org/minima/system/network/base/ssz/ArrayIntCache.java new file mode 100644 index 000000000..e5984d06e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/ArrayIntCache.java @@ -0,0 +1,114 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Arrays; +import java.util.Optional; +import java.util.function.IntFunction; + +/** + * Thread-safe int indexed cache + * + *

CAUTION: though the class is thread-safe it contains no synchronisation for performance + * reasons When accessed concurrently the cache may result in extra cache misses and extra backing + * array copying but this should be safe. In optimistic scenarios such overhead could be neglected + * + *

Modify this class carefully to not violate thread safety! + */ +public final class ArrayIntCache implements IntCache { + private static final int DEFAULT_INITIAL_CACHE_SIZE = 16; + private volatile V[] values; + private final int initSize; + + public ArrayIntCache() { + this(16); + } + + public ArrayIntCache(int initialSize) { + this.initSize = initialSize; + this.values = createArray(initialSize); + } + + private ArrayIntCache(V[] values, int initSize) { + this.values = values; + this.initSize = initSize; + } + + @SuppressWarnings("unchecked") + private V[] createArray(int size) { + return (V[]) new Object[size]; + } + + private V[] extend(int index) { + V[] valuesLocal = this.values; + int newSize = valuesLocal.length; + if (index >= newSize) { + while (index >= newSize) { + newSize <<= 1; + } + V[] newValues = Arrays.copyOf(valuesLocal, newSize); + this.values = newValues; + return newValues; + } + return valuesLocal; + } + + @Override + public V getInt(int key, IntFunction fallback) { + V[] valuesLocal = this.values; + V val = key >= valuesLocal.length ? null : valuesLocal[key]; + if (val == null) { + val = fallback.apply(key); + extend(key)[key] = val; + } + return val; + } + + @Override + public Optional getCached(Integer key) { + V[] valuesLocal = this.values; + return key >= valuesLocal.length ? Optional.empty() : Optional.ofNullable(valuesLocal[key]); + } + + @Override + public IntCache copy() { + V[] valuesLocal = this.values; + return new ArrayIntCache<>(Arrays.copyOf(valuesLocal, valuesLocal.length), initSize); + } + + @Override + public IntCache transfer() { + IntCache copy = copy(); + this.values = createArray(DEFAULT_INITIAL_CACHE_SIZE); + return copy; + } + + @Override + public void invalidateWithNewValueInt(int key, V newValue) { + extend(key)[key] = newValue; + } + + @Override + public void invalidateInt(int key) { + V[] valuesLocal = this.values; + if (key < valuesLocal.length) { + valuesLocal[key] = null; + } + } + + @Override + public void clear() { + values = createArray(initSize); + } +} diff --git a/src/org/minima/system/network/base/ssz/BitvectorImpl.java b/src/org/minima/system/network/base/ssz/BitvectorImpl.java new file mode 100644 index 000000000..59bd26d0a --- /dev/null +++ b/src/org/minima/system/network/base/ssz/BitvectorImpl.java @@ -0,0 +1,143 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkElementIndex; +import static java.util.stream.Collectors.toList; +import static org.minima.system.network.base.ssz.TreeUtil.bitsCeilToBytes; + +import com.google.common.base.Objects; +import java.util.Arrays; +import java.util.BitSet; +import java.util.List; +import java.util.stream.IntStream; +import org.apache.tuweni.bytes.Bytes; + +class BitvectorImpl { + + public static BitvectorImpl fromBytes(Bytes bytes, int size) { + checkArgument( + bytes.size() == sszSerializationLength(size), + "Incorrect data size (%s) for Bitvector of size %s", + bytes.size(), + size); + BitSet bitset = new BitSet(size); + + for (int i = size - 1; i >= 0; i--) { + if (((bytes.get(i / 8) >>> (i % 8)) & 0x01) == 1) { + bitset.set(i); + } + } + + return new BitvectorImpl(bitset, size); + } + + public static int sszSerializationLength(final int size) { + return bitsCeilToBytes(size); + } + + private final BitSet data; + private final int size; + + private BitvectorImpl(BitSet bitSet, int size) { + this.data = bitSet; + this.size = size; + } + + public BitvectorImpl(int size) { + this.data = new BitSet(size); + this.size = size; + } + + public BitvectorImpl(int size, Iterable indicesToSet) { + this(size); + for (int i : indicesToSet) { + checkElementIndex(i, size); + data.set(i); + } + } + + public BitvectorImpl(int size, int... indicesToSet) { + this(size, Arrays.stream(indicesToSet).boxed().collect(toList())); + } + + public List getSetBitIndexes() { + return data.stream().boxed().collect(toList()); + } + + public BitvectorImpl withBit(int i) { + checkElementIndex(i, size); + BitSet newSet = (BitSet) data.clone(); + newSet.set(i); + return new BitvectorImpl(newSet, size); + } + + public int getBitCount() { + return data.cardinality(); + } + + public boolean getBit(int i) { + checkElementIndex(i, size); + return data.get(i); + } + + public int getSize() { + return size; + } + + public IntStream streamAllSetBits() { + return data.stream(); + } + + @SuppressWarnings("NarrowingCompoundAssignment") + public Bytes serialize() { + byte[] array = new byte[sszSerializationLength(size)]; + IntStream.range(0, size).forEach(i -> array[i / 8] |= ((data.get(i) ? 1 : 0) << (i % 8))); + return Bytes.wrap(array); + } + + public BitvectorImpl rightShift(int i) { + int length = this.getSize(); + BitSet newData = new BitSet(getSize()); + for (int j = 0; j < length - i; j++) { + if (this.getBit(j)) { + newData.set(j + i); + } + } + return new BitvectorImpl(newData, getSize()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof BitvectorImpl)) return false; + BitvectorImpl bitvector = (BitvectorImpl) o; + return getSize() == bitvector.getSize() && Objects.equal(data, bitvector.data); + } + + @Override + public int hashCode() { + return Objects.hashCode(data, getSize()); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < size; i++) { + sb.append(getBit(i) ? 1 : 0); + } + return sb.toString(); + } +} diff --git a/src/org/minima/system/network/base/ssz/BranchNode.java b/src/org/minima/system/network/base/ssz/BranchNode.java new file mode 100644 index 000000000..094554955 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/BranchNode.java @@ -0,0 +1,118 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.function.Function; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.jetbrains.annotations.NotNull; +import org.minima.system.network.base.ssz.GIndexUtil.NodeRelation; +import org.minima.system.network.base.ssz.TreeNodeImpl.BranchNodeImpl; +import org.minima.utils.Crypto; + +/** + * Branch node of a tree. This node type corresponds to the 'Commit' node in the spec: + * https://github.com/protolambda/eth-merkle-trees/blob/master/typing_partials.md#structure + */ +public interface BranchNode extends TreeNode { + + /** + * Creates a basic binary Branch node with left and right child + * + * @param left Non-null left child + * @param right Non-null right child + */ + static BranchNode create(TreeNode left, TreeNode right) { + checkNotNull(left); + checkNotNull(right); + return new BranchNodeImpl(left, right); + } + + /** + * Returns left child node. It can be either a default or non-default node. Note that both left + * and right child may be the same default instance + */ + @NotNull + TreeNode left(); + + /** + * Returns right child node. It can be either a default or non-default node. Note that both left + * and right child may be the same default instance + */ + @NotNull + TreeNode right(); + + /** + * Rebind 'sets' a new left/right child of this node. Rebind doesn't modify this instance but + * creates and returns a new one which contains a new assigned and old unmodified child + */ + BranchNode rebind(boolean left, TreeNode newNode); + + @Override + default Bytes32 hashTreeRoot() { + byte[] hash = (new Crypto()).hashSHA2(Bytes.concatenate(left().hashTreeRoot(), right().hashTreeRoot()).toArray()); + return Bytes32.wrap(hash); + //return Hash.sha2_256(Bytes.concatenate(left().hashTreeRoot(), right().hashTreeRoot())); + } + + @NotNull + @Override + default TreeNode get(long target) { + checkArgument(target >= 1, "Invalid index: %s", target); + if (GIndexUtil.gIdxIsSelf(target)) { + return this; + } else { + long relativeGIndex = GIndexUtil.gIdxGetRelativeGIndex(target, 1); + return GIndexUtil.gIdxGetChildIndex(target, 1) == 0 + ? left().get(relativeGIndex) + : right().get(relativeGIndex); + } + } + + @Override + default boolean iterate( + long thisGeneralizedIndex, long startGeneralizedIndex, TreeVisitor visitor) { + + if (GIndexUtil.gIdxCompare(thisGeneralizedIndex, startGeneralizedIndex) == NodeRelation.Left) { + return true; + } else { + return visitor.visit(this, thisGeneralizedIndex) + && left() + .iterate( + GIndexUtil.gIdxLeftGIndex(thisGeneralizedIndex), startGeneralizedIndex, visitor) + && right() + .iterate( + GIndexUtil.gIdxRightGIndex(thisGeneralizedIndex), startGeneralizedIndex, visitor); + } + } + + @Override + default TreeNode updated(long target, Function nodeUpdater) { + if (GIndexUtil.gIdxIsSelf(target)) { + return nodeUpdater.apply(this); + } else { + long relativeGIndex = GIndexUtil.gIdxGetRelativeGIndex(target, 1); + if (GIndexUtil.gIdxGetChildIndex(target, 1) == 0) { + TreeNode newLeftChild = left().updated(relativeGIndex, nodeUpdater); + return rebind(true, newLeftChild); + } else { + TreeNode newRightChild = right().updated(relativeGIndex, nodeUpdater); + return rebind(false, newRightChild); + } + } + } +} diff --git a/src/org/minima/system/network/base/ssz/Bytes4.java b/src/org/minima/system/network/base/ssz/Bytes4.java new file mode 100644 index 000000000..b8c824996 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/Bytes4.java @@ -0,0 +1,110 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Objects; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.MutableBytes; + +public class Bytes4 { + public static final int SIZE = 4; + + private Bytes bytes; + + public Bytes4(Bytes bytes) { + checkArgument(bytes.size() == 4, "Bytes4 should be 4 bytes, but was %s bytes.", bytes.size()); + this.bytes = bytes; + } + + public static Bytes4 fromHexString(String value) { + return new Bytes4(Bytes.fromHexString(value)); + } + + public String toHexString() { + return bytes.toHexString(); + } + + /** + * Left pad a {@link Bytes} value with zero bytes to create a {@link Bytes4}. + * + * @param value The bytes value pad. + * @return A {@link Bytes4} that exposes the left-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 4}. + */ + public static Bytes4 leftPad(Bytes value) { + checkNotNull(value); + if (value instanceof Bytes4) { + return (Bytes4) value; + } + checkArgument(value.size() <= 4, "Expected at most %s bytes but got %s", 4, value.size()); + MutableBytes result = MutableBytes.create(4); + value.copyTo(result, 4 - value.size()); + return new Bytes4(result); + } + + /** + * Right pad a {@link Bytes} value with zero bytes to create a {@link Bytes4}. + * + * @param value The bytes value pad. + * @return A {@link Bytes4} that exposes the right-padded bytes of {@code value}. + * @throws IllegalArgumentException if {@code value.size() > 4}. + */ + public static Bytes4 rightPad(Bytes value) { + checkNotNull(value); + if (value instanceof Bytes4) { + return (Bytes4) value; + } + checkArgument(value.size() <= 4, "Expected at most %s bytes but got %s", 4, value.size()); + MutableBytes result = MutableBytes.create(4); + value.copyTo(result, 0); + return new Bytes4(result); + } + + public Bytes getWrappedBytes() { + return bytes; + } + + public Bytes4 copy() { + return new Bytes4(bytes.copy()); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Bytes4 bytes4 = (Bytes4) o; + return bytes.equals(bytes4.bytes); + } + + @Override + public int hashCode() { + return Objects.hash(bytes); + } + + public String toUnprefixedHexString() { + return bytes.toUnprefixedHexString(); + } + + @Override + public String toString() { + return bytes.toString(); + } +} diff --git a/src/org/minima/system/network/base/ssz/Cache.java b/src/org/minima/system/network/base/ssz/Cache.java new file mode 100644 index 000000000..33c2e4234 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/Cache.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Optional; +import java.util.function.Function; + +/** + * Cache + * + * @param type of keys + * @param type of values + */ +public interface Cache { + /** + * Queries value from the cache. If it's not found there, fallback function is used to calculate + * value. After calculation result is put in cache and returned. + * + * @param key Key to query + * @param fallback Fallback function for calculation of the result in case of missed cache entry + * @return expected value result for provided key + */ + V get(K key, Function fallback); + + /** + * Optionally returns the value corresponding to the passed key is it's in the cache + */ + Optional getCached(K key); + + /** Creates independent copy of this Cache instance */ + Cache copy(); + + /** Creates independent copy of this Cache instance while possibly clearing this cache content */ + default Cache transfer() { + return copy(); + } + + /** Removes cache entry */ + void invalidate(K key); + + default void invalidateWithNewValue(K key, V newValue) { + invalidate(key); + get(key, k -> newValue); + } + + /** Clears all cached values */ + void clear(); + + /** Returns the current number of items in the cache */ + int size(); +} diff --git a/src/org/minima/system/network/base/ssz/Container3.java b/src/org/minima/system/network/base/ssz/Container3.java new file mode 100644 index 000000000..888bcb6b0 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/Container3.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.impl.AbstractSszImmutableContainer; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** Autogenerated by tech.pegasys.teku.ssz.backing.ContainersGenerator */ +public class Container3< + C extends Container3, + V0 extends SszData, + V1 extends SszData, + V2 extends SszData> + extends AbstractSszImmutableContainer { + + protected Container3(ContainerSchema3 schema) { + super(schema); + } + + protected Container3(ContainerSchema3 schema, TreeNode backingNode) { + super(schema, backingNode); + } + + protected Container3(ContainerSchema3 schema, V0 arg0, V1 arg1, V2 arg2) { + super(schema, arg0, arg1, arg2); + } + + protected V0 getField0() { + return getAny(0); + } + + protected V1 getField1() { + return getAny(1); + } + + protected V2 getField2() { + return getAny(2); + } +} diff --git a/src/org/minima/system/network/base/ssz/ContainerSchema3.java b/src/org/minima/system/network/base/ssz/ContainerSchema3.java new file mode 100644 index 000000000..3d9bb707e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/ContainerSchema3.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.List; +import java.util.function.BiFunction; +// import tech.pegasys.teku.ssz.SszContainer; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** Autogenerated by tech.pegasys.teku.ssz.backing.ContainersGenerator */ +public abstract class ContainerSchema3< + C extends SszContainer, V0 extends SszData, V1 extends SszData, V2 extends SszData> + extends AbstractSszContainerSchema { + + public static + ContainerSchema3 create( + SszSchema fieldSchema0, + SszSchema fieldSchema1, + SszSchema fieldSchema2, + BiFunction, TreeNode, C> instanceCtor) { + return new ContainerSchema3<>(fieldSchema0, fieldSchema1, fieldSchema2) { + @Override + public C createFromBackingNode(TreeNode node) { + return instanceCtor.apply(this, node); + } + }; + } + + protected ContainerSchema3( + SszSchema fieldSchema0, SszSchema fieldSchema1, SszSchema fieldSchema2) { + + super(List.of(fieldSchema0, fieldSchema1, fieldSchema2)); + } + + protected ContainerSchema3( + String containerName, + NamedSchema fieldNamedSchema0, + NamedSchema fieldNamedSchema1, + NamedSchema fieldNamedSchema2) { + + super(containerName, List.of(fieldNamedSchema0, fieldNamedSchema1, fieldNamedSchema2)); + } + + @SuppressWarnings("unchecked") + public SszSchema getFieldSchema0() { + return (SszSchema) getChildSchema(0); + } + + @SuppressWarnings("unchecked") + public SszSchema getFieldSchema1() { + return (SszSchema) getChildSchema(1); + } + + @SuppressWarnings("unchecked") + public SszSchema getFieldSchema2() { + return (SszSchema) getChildSchema(2); + } +} diff --git a/src/org/minima/system/network/base/ssz/GIndexUtil.java b/src/org/minima/system/network/base/ssz/GIndexUtil.java new file mode 100644 index 000000000..33ef96e1d --- /dev/null +++ b/src/org/minima/system/network/base/ssz/GIndexUtil.java @@ -0,0 +1,311 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static java.lang.Integer.min; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Util methods for binary tree generalized indexes manipulations See + * https://github.com/ethereum/eth2.0-specs/blob/v1.0.0/ssz/merkle-proofs.md#generalized-merkle-tree-index + * for more info on generalized indexes + * + *

Here the general index is represented by long which is treated as unsigned uint64 + * Thus the only illegal generalized index value is 0 + */ +public class GIndexUtil { + + /** See {@link #gIdxCompare(long, long)} */ + public enum NodeRelation { + Left, + Right, + Successor, + Predecessor, + Same; + + /** gIdxCompare(idx1, idx2) == gIdxCompare(idx2, idx1).inverse() */ + public NodeRelation inverse() { + switch (this) { + case Left: + return Right; + case Right: + return Left; + case Predecessor: + return Successor; + case Successor: + return Predecessor; + case Same: + return Same; + default: + throw new IllegalArgumentException("Unknown: " + this); + } + } + } + + /** Maximal depth this generalized index implementation can handle */ + // with the depth 64 positive long would overflow and we don't want to handle it here + public static final int MAX_DEPTH = 63; + + /** + * The generalized index of either a root tree node or an index of a node relative to the node + * itself. Effectively this is 1L + */ + public static final long SELF_G_INDEX = 1; + + /** The generalized index of the left child. Effectively 0b10 */ + public static final long LEFT_CHILD_G_INDEX = gIdxLeftGIndex(SELF_G_INDEX); + + /** The generalized index of the right child. Effectively 0b11 */ + public static final long RIGHT_CHILD_G_INDEX = gIdxRightGIndex(SELF_G_INDEX); + + /** + * The generalized index (normally an index of non-existing node) of the leftmost possible node + * Effectively this is {@link Long#MIN_VALUE} or 0b10000...000L in binary form + */ + static final long LEFTMOST_G_INDEX = gIdxLeftmostFrom(SELF_G_INDEX); + /** + * The generalized index (normally an index of non-existing node) of the rightmost possible node + * Effectively this is -1L or 0b11111...111L in binary form + */ + static final long RIGHTMOST_G_INDEX = gIdxRightmostFrom(SELF_G_INDEX); + + /** + * Indicates that a relative generalized index refers to the node itself + * + * @see #SELF_G_INDEX + */ + public static boolean gIdxIsSelf(long generalizedIndex) { + checkGIndex(generalizedIndex); + return generalizedIndex == SELF_G_INDEX; + } + + /** + * Indicates how the node with generalized index idx1 relates to the node with + * generalized index idx2: + * + *

    + *
  • {@link NodeRelation#Left}: idx1 is to the left of idx2 + *
  • {@link NodeRelation#Right}: idx1 is to the right of idx2 + *
  • {@link NodeRelation#Successor}: idx1 is the successor of idx2 + *
  • {@link NodeRelation#Predecessor}: idx1 is the predecessor of idx2 + *
  • {@link NodeRelation#Same}: idx1 is equal to idx2 + *
+ */ + public static NodeRelation gIdxCompare(long idx1, long idx2) { + checkGIndex(idx1); + checkGIndex(idx2); + long anchor1 = Long.highestOneBit(idx1); + long anchor2 = Long.highestOneBit(idx2); + int depth1 = Long.bitCount(anchor1 - 1); + int depth2 = Long.bitCount(anchor2 - 1); + int minDepth = min(depth1, depth2); + long minDepthIdx1 = idx1 >>> (depth1 - minDepth); + long minDepthIdx2 = idx2 >>> (depth2 - minDepth); + if (minDepthIdx1 == minDepthIdx2) { + if (depth1 < depth2) { + return NodeRelation.Predecessor; + } else if (depth1 > depth2) { + return NodeRelation.Successor; + } else { + return NodeRelation.Same; + } + } else { + if (minDepthIdx1 < minDepthIdx2) { + return NodeRelation.Left; + } else { + return NodeRelation.Right; + } + } + } + + /** + * Returns the depth of the node denoted by the supplied generalized index. E.g. the depth of the + * {@link #SELF_G_INDEX} would be 0 + */ + public static int gIdxGetDepth(long generalizedIndex) { + checkGIndex(generalizedIndex); + long anchor = Long.highestOneBit(generalizedIndex); + return Long.bitCount(anchor - 1); + } + + /** + * Returns the generalized index of the left child of the node with specified generalized index + * E.g. the result when passing {@link #SELF_G_INDEX} would be 10 + */ + public static long gIdxLeftGIndex(long generalizedIndex) { + return gIdxChildGIndex(generalizedIndex, 0, 1); + } + + /** + * Returns the generalized index of the right child of the node with specified generalized index + * E.g. the result when passing {@link #SELF_G_INDEX} would be 11 + */ + public static long gIdxRightGIndex(long generalizedIndex) { + return gIdxChildGIndex(generalizedIndex, 1, 1); + } + + /** + * More generic variant of methods {@link #gIdxLeftGIndex(long)} {@link #gIdxRightGIndex(long)} + * Calculates the generalized index of a node's childIdx successor at depth + * childDepth (depth relative to the original node). Note that childIdx is not + * the generalized index but index number of child. + * + *

For example: + * + *

    + *
  • gIdxChildGIndex(SELF_G_INDEX, 0, 2) == 100 + *
  • gIdxChildGIndex(SELF_G_INDEX, 1, 2) == 101 + *
  • gIdxChildGIndex(SELF_G_INDEX, 2, 2) == 110 + *
  • gIdxChildGIndex(SELF_G_INDEX, 3, 2) == 111 + *
  • gIdxChildGIndex(SELF_G_INDEX, 4, 2) is invalid cause there are just 4 successors + * at depth 2 + *
  • gIdxChildGIndex(anyIndex, 0, 1) == gIdxLeftGIndex(anyIndex) + *
  • gIdxChildGIndex(anyIndex, 1, 1) == gIdxRightGIndex(anyIndex) + *
+ */ + public static long gIdxChildGIndex(long generalizedIndex, long childIdx, int childDepth) { + checkGIndex(generalizedIndex); + assert childDepth >= 0 && childDepth <= MAX_DEPTH; + assert childIdx >= 0 && childIdx < (1L << childDepth); + assert gIdxGetDepth(generalizedIndex) + childDepth <= MAX_DEPTH; + return (generalizedIndex << childDepth) | childIdx; + } + + /** + * Compose absolute generalized index, where childGeneralizedIndex is relative to the + * node at parentGeneralizedIndex + * + *

For example: + * + *

    + *
  • gIdxCompose(0b1111, 0b1000) == 0b1111000 + *
  • gIdxCompose(0b1000, 0b1111) == 0b1000111 + *
+ */ + public static long gIdxCompose(long parentGeneralizedIndex, long childGeneralizedIndex) { + checkGIndex(parentGeneralizedIndex); + checkGIndex(childGeneralizedIndex); + assert gIdxGetDepth(parentGeneralizedIndex) + gIdxGetDepth(childGeneralizedIndex) <= MAX_DEPTH; + + long childAnchor = Long.highestOneBit(childGeneralizedIndex); + int childDepth = Long.bitCount(childAnchor - 1); + return (parentGeneralizedIndex << childDepth) | (childGeneralizedIndex ^ childAnchor); + } + + /** + * Returns the generalized index (normally an index of non-existing node) of the leftmost possible + * successor of this node + * + *

For example: + * + *

    + *
  • gIdxLeftmostFrom(0b1100) == 0b110000000...00L + *
  • gIdxLeftmostFrom(0b1101) == 0b110100000...00L + *
+ */ + public static long gIdxLeftmostFrom(long fromGeneralizedIndex) { + checkGIndex(fromGeneralizedIndex); + long highestOneBit = Long.highestOneBit(fromGeneralizedIndex); + if (highestOneBit < 0) { + return fromGeneralizedIndex; + } else { + int nodeDepth = Long.bitCount(highestOneBit - 1); + return fromGeneralizedIndex << (MAX_DEPTH - nodeDepth); + } + } + + /** + * Returns the generalized index (normally an index of non-existing node) of the rightmost + * possible successor of this node + * + *

For example: + * + *

    + *
  • gIdxRightmostFrom(0b1100) == 0b110011111...11L + *
  • gIdxRightmostFrom(0b1101) == 0b110111111...11L + *
+ */ + public static long gIdxRightmostFrom(long fromGeneralizedIndex) { + checkGIndex(fromGeneralizedIndex); + long highestOneBit = Long.highestOneBit(fromGeneralizedIndex); + if (highestOneBit < 0) { + return fromGeneralizedIndex; + } else { + int nodeDepth = Long.bitCount(highestOneBit - 1); + int shiftN = MAX_DEPTH - nodeDepth; + return (fromGeneralizedIndex << shiftN) | ((1L << shiftN) - 1); + } + } + + /** + * Returns the index number (not a generalized index) of a node at depth childDepth + * which is a predecessor of or equal to the node at generalizedIndex + * + *

For example: + * + *

    + *
  • gIdxGetChildIndex(LEFTMOST_G_INDEX, anyDepth) == 0 + *
  • gIdxGetChildIndex(0b1100, 2) == 2 + *
  • gIdxGetChildIndex(0b1101, 2) == 2 + *
  • gIdxGetChildIndex(0b1110, 2) == 3 + *
  • gIdxGetChildIndex(0b1111, 2) == 3 + *
  • gIdxGetChildIndex(0b11, 2) call would be invalid cause node with index 0b11 + * is at depth 1 + *
+ */ + public static int gIdxGetChildIndex(long generalizedIndex, int childDepth) { + checkGIndex(generalizedIndex); + assert childDepth >= 0 && childDepth <= MAX_DEPTH; + + long anchor = Long.highestOneBit(generalizedIndex); + int indexBitCount = Long.bitCount(anchor - 1); + assert indexBitCount >= childDepth; + long generalizedIndexWithoutAnchor = generalizedIndex ^ anchor; + return (int) (generalizedIndexWithoutAnchor >>> (indexBitCount - childDepth)); + } + + /** + * Returns the generalized index of the node at generalizedIndex relative to its + * predecessor at depth childDepth For example: + * + *
    + *
  • gIdxGetRelativeGIndex(0b1100, 2) == 0b10 + *
  • gIdxGetChildIndex(0b1101, 2) == 0b11 + *
  • gIdxGetChildIndex(0b1110, 2) == 0b10 + *
  • gIdxGetChildIndex(0b1111, 3) == SELF_G_INDEX + *
  • gIdxGetChildIndex(0b11, 2) call would be invalid cause node with index 0b11 + * is at depth 1 + *
+ */ + public static long gIdxGetRelativeGIndex(long generalizedIndex, int childDepth) { + checkGIndex(generalizedIndex); + assert childDepth >= 0 && childDepth <= MAX_DEPTH; + + long anchor = Long.highestOneBit(generalizedIndex); + long pivot = anchor >>> childDepth; + assert pivot != 0; + return (generalizedIndex & (pivot - 1)) | pivot; + } + + @VisibleForTesting + static long gIdxGetParent(long generalizedIndex) { + checkGIndex(generalizedIndex); + return generalizedIndex >>> 1; + } + + private static void checkGIndex(long index) { + assert index != 0; + } +} diff --git a/src/org/minima/system/network/base/ssz/IntCache.java b/src/org/minima/system/network/base/ssz/IntCache.java new file mode 100644 index 000000000..f5cf5a320 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/IntCache.java @@ -0,0 +1,75 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Function; +import java.util.function.IntFunction; + +/** + * Optimized int keys cache. Eliminate int boxing/unboxing + * + * @param type of values + */ +public interface IntCache extends Cache { + + @SuppressWarnings("unchecked") + static IntCache noop() { + return (IntCache) NoopIntCache.INSTANCE; + } + + /** + * Queries value from the cache. If it's not found there, fallback function is used to calculate + * value. After calculation result is put in cache and returned. + * + * @param key Key to query + * @param fallback Fallback function for calculation of the result in case of missed cache entry + * @return expected value result for provided key + */ + V getInt(int key, IntFunction fallback); + + @Override + default V get(Integer key, Function fallback) { + return getInt(key, value -> fallback.apply(key)); + } + + @Override + IntCache copy(); + + @Override + default IntCache transfer() { + return copy(); + } + + /** Removes cache entry */ + void invalidateInt(int key); + + @Override + default void invalidate(Integer key) { + invalidateInt(key); + } + + default void invalidateWithNewValueInt(int key, V newValue) { + invalidateInt(key); + getInt(key, k -> newValue); + } + + @Override + default void invalidateWithNewValue(Integer key, V newValue) { + invalidateWithNewValueInt(key, newValue); + } + + /** Clears all cached values */ + @Override + void clear(); +} diff --git a/src/org/minima/system/network/base/ssz/InvalidValueSchemaException.java b/src/org/minima/system/network/base/ssz/InvalidValueSchemaException.java new file mode 100644 index 000000000..b8880dbc1 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/InvalidValueSchemaException.java @@ -0,0 +1,21 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +public class InvalidValueSchemaException extends RuntimeException { + + public InvalidValueSchemaException(String message) { + super(message); + } +} diff --git a/src/org/minima/system/network/base/ssz/LeafDataNode.java b/src/org/minima/system/network/base/ssz/LeafDataNode.java new file mode 100644 index 000000000..66d167827 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/LeafDataNode.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; + +/** Represents a tree node which can supply its leaves data */ +public interface LeafDataNode extends TreeNode { + + /** + * Returns the merged leaf data of this node's leaf descendants (see {@link SszSuperNode} for + * example) or just a leaf data if this node represents a single leaf (see {@link + * LeafNode#getData()} + */ + Bytes getData(); +} diff --git a/src/org/minima/system/network/base/ssz/LeafNode.java b/src/org/minima/system/network/base/ssz/LeafNode.java new file mode 100644 index 000000000..feabeb15c --- /dev/null +++ b/src/org/minima/system/network/base/ssz/LeafNode.java @@ -0,0 +1,98 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.function.Function; +import java.util.stream.IntStream; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.jetbrains.annotations.NotNull; +import org.minima.system.network.base.ssz.GIndexUtil.NodeRelation; +//import tech.pegasys.teku.ssz.tree.TreeNodeImpl.LeafNodeImpl; +//import tech.pegasys.teku.ssz.tree.TreeUtil.ZeroLeafNode; +import org.minima.system.network.base.ssz.TreeUtil.ZeroLeafNode; +import org.minima.system.network.base.ssz.TreeNodeImpl.LeafNodeImpl; + +/** + * Leaf node of a tree which contains 'bytes32' value. This node type corresponds to the 'Root' node + * in the spec: + * https://github.com/protolambda/eth-merkle-trees/blob/master/typing_partials.md#structure + */ +public interface LeafNode extends TreeNode, LeafDataNode { + + int MAX_BYTE_SIZE = 32; + int MAX_BIT_SIZE = MAX_BYTE_SIZE * 8; + + /** + * Pre-allocated leaf nodes with the data consisting of 0, 1, 2, ..., 32 zero bytes Worth to + * mention that {@link TreeNode#hashTreeRoot()} for all these nodes return the same value {@link + * Bytes32#ZERO} + */ + LeafNode[] ZERO_LEAVES = + IntStream.rangeClosed(0, MAX_BYTE_SIZE).mapToObj(ZeroLeafNode::new).toArray(LeafNode[]::new); + + /** The {@link LeafNode} with empty data */ + LeafNode EMPTY_LEAF = ZERO_LEAVES[0]; + + /** Creates a basic Leaf node instance with the data <= 32 bytes */ + static LeafNode create(Bytes data) { + return new LeafNodeImpl(data); + } + + /** + * Returns only data bytes without zero right padding (unlike {@link #hashTreeRoot()}) E.g. if a + * {@code LeafNode} corresponds to a contained UInt64 field, then {@code getData()} returns only 8 + * bytes corresponding to the field value If a {@code Vector[Byte, 48]} is stored across two + * {@code LeafNode}s then the second node {@code getData} would return just the last 16 bytes of + * the vector (while {@link #hashTreeRoot()} would return zero padded 32 bytes) + */ + @Override + Bytes getData(); + + /** LeafNode hash tree root is the leaf data right padded to 32 bytes */ + @Override + default Bytes32 hashTreeRoot() { + return Bytes32.rightPad(getData()); + } + + /** + * @param target generalized index. Should be equal to 1 + * @return this node if 'target' == 1 + * @throws IllegalArgumentException if 'target' != 1 + */ + @NotNull + @Override + default TreeNode get(long target) { + checkArgument(target == 1, "Invalid root index: %s", target); + return this; + } + + @Override + default boolean iterate( + long thisGeneralizedIndex, long startGeneralizedIndex, TreeVisitor visitor) { + if (GIndexUtil.gIdxCompare(thisGeneralizedIndex, startGeneralizedIndex) == NodeRelation.Left) { + return true; + } else { + return visitor.visit(this, thisGeneralizedIndex); + } + } + + @Override + default TreeNode updated(long target, Function nodeUpdater) { + checkArgument(target == 1, "Invalid root index: %s", target); + return nodeUpdater.apply(this); + } +} diff --git a/src/org/minima/system/network/base/ssz/Merkleizable.java b/src/org/minima/system/network/base/ssz/Merkleizable.java new file mode 100644 index 000000000..ebe203b72 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/Merkleizable.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; + +/** + * Returns `hash_tree_root` conforming to SSZ spec: + * https://github.com/ethereum/eth2.0-specs/blob/dev/ssz/simple-serialize.md#merkleization + */ +public interface Merkleizable { + + /** + * Returns `hash_tree_root` conforming to SSZ spec: + * https://github.com/ethereum/eth2.0-specs/blob/dev/ssz/simple-serialize.md#merkleization + */ + Bytes32 hashTreeRoot(); +} diff --git a/src/org/minima/system/network/base/ssz/NoopIntCache.java b/src/org/minima/system/network/base/ssz/NoopIntCache.java new file mode 100644 index 000000000..6fad4afb9 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/NoopIntCache.java @@ -0,0 +1,46 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Optional; +import java.util.function.IntFunction; + +public class NoopIntCache implements IntCache { + + static final IntCache INSTANCE = new NoopIntCache<>(); + + private NoopIntCache() {} + + @Override + public V getInt(int key, IntFunction fallback) { + return fallback.apply(key); + } + + @Override + public IntCache copy() { + return new NoopIntCache<>(); + } + + @Override + public void invalidateInt(int key) {} + + @Override + public void clear() {} + + @Override + public Optional getCached(Integer key) { + return Optional.empty(); + } + +} diff --git a/src/org/minima/system/network/base/ssz/SchemaUtils.java b/src/org/minima/system/network/base/ssz/SchemaUtils.java new file mode 100644 index 000000000..4f5785c74 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SchemaUtils.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +class SchemaUtils { + + public static TreeNode createTreeFromBytes(Bytes bytes, int treeDepth) { + return TreeUtil.createTree( + split(bytes, LeafNode.MAX_BYTE_SIZE).stream() + .map(LeafNode::create) + .collect(Collectors.toList()), + treeDepth); + } + + public static List split(Bytes bytes, int chunkSize) { + List ret = new ArrayList<>(); + int off = 0; + int size = bytes.size(); + while (off < size) { + Bytes leafData = bytes.slice(off, Integer.min(chunkSize, size - off)); + ret.add(leafData); + off += chunkSize; + } + return ret; + } +} diff --git a/src/org/minima/system/network/base/ssz/SimpleOffsetSerializable.java b/src/org/minima/system/network/base/ssz/SimpleOffsetSerializable.java new file mode 100644 index 000000000..da865dcba --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SimpleOffsetSerializable.java @@ -0,0 +1,35 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.ssz.sos.SszWriter; + +/** + * Represent the data which can be SSZ serialized + * + *

SSZ spec: https://github.com/protolambda/eth2.0-ssz + */ +public interface SimpleOffsetSerializable { + + /** Returns this data SSZ serialization */ + Bytes sszSerialize(); + + /** + * SSZ serializes this data to supplied {@code writer} + * + * @return number of bytes written + */ + int sszSerialize(SszWriter writer); +} diff --git a/src/org/minima/system/network/base/ssz/SimpleSszReader.java b/src/org/minima/system/network/base/ssz/SimpleSszReader.java new file mode 100644 index 000000000..fe577f89a --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SimpleSszReader.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; + +public class SimpleSszReader implements SszReader { + + private final Bytes bytes; + protected int offset = 0; + + public SimpleSszReader(Bytes bytes) { + this.bytes = bytes; + } + + @Override + public int getAvailableBytes() { + return bytes.size() - offset; + } + + @Override + public SszReader slice(int size) { + checkIfAvailable(size); + SimpleSszReader ret = new SimpleSszReader(bytes.slice(offset, size)); + offset += size; + return ret; + } + + @Override + public Bytes read(int length) { + checkIfAvailable(length); + Bytes ret = bytes.slice(offset, length); + offset += length; + return ret; + } + + private void checkIfAvailable(int size) { + if (getAvailableBytes() < size) { + throw new SszDeserializeException("Invalid SSZ: trying to read more bytes than available"); + } + } + + @Override + public void close() { + if (getAvailableBytes() > 0) { + throw new SszDeserializeException("Invalid SSZ: unread bytes remain: " + getAvailableBytes()); + } + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBit.java b/src/org/minima/system/network/base/ssz/SszBit.java new file mode 100644 index 000000000..a29ec7e6a --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBit.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.impl.AbstractSszPrimitive; +//import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; + +public class SszBit extends AbstractSszPrimitive { + + private static final SszBit TRUE_VIEW = new SszBit(true); + private static final SszBit FALSE_VIEW = new SszBit(false); + + public static SszBit of(boolean value) { + return value ? TRUE_VIEW : FALSE_VIEW; + } + + private SszBit(Boolean value) { + super(value, SszPrimitiveSchemas.BIT_SCHEMA); + } + + @Override + @SuppressWarnings("ReferenceEquality") + public String toString() { + return this == TRUE_VIEW ? "1" : "0"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBitvector.java b/src/org/minima/system/network/base/ssz/SszBitvector.java new file mode 100644 index 000000000..d2e5b0b2e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBitvector.java @@ -0,0 +1,62 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.List; +import java.util.stream.IntStream; +// import tech.pegasys.teku.ssz.primitive.SszBit; +// import tech.pegasys.teku.ssz.schema.collections.SszBitvectorSchema; + +/** Specialized implementation of {@code SszVector} */ +public interface SszBitvector extends SszPrimitiveVector { + + @Override + default SszMutablePrimitiveVector createWritableCopy() { + throw new UnsupportedOperationException("SszBitlist is immutable structure"); + } + + @Override + default boolean isWritableSupported() { + return false; + } + + @Override + SszBitvectorSchema getSchema(); + + // Bitlist methods + + SszBitvector withBit(int i); + + /** Returns individual bit value */ + boolean getBit(int i); + + /** Returns the number of bits set to {@code true} in this {@code SszBitlist}. */ + int getBitCount(); + + /** Returns new vector with bits shifted to the right by {@code n} positions */ + SszBitvector rightShift(int n); + + /** Returns indexes of all bits set in this {@link SszBitvector} */ + List getAllSetBits(); + + /** Streams indexes of all bits set in this {@link SszBitvector} */ + default IntStream streamAllSetBits() { + return getAllSetBits().stream().mapToInt(i -> i); + } + + @Override + default Boolean getElement(int index) { + return getBit(index); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBitvectorImpl.java b/src/org/minima/system/network/base/ssz/SszBitvectorImpl.java new file mode 100644 index 000000000..2b1317905 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBitvectorImpl.java @@ -0,0 +1,105 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.List; +import java.util.stream.Collectors; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.collections.SszBitvector; +// import tech.pegasys.teku.ssz.collections.SszMutablePrimitiveVector; +// import tech.pegasys.teku.ssz.impl.SszVectorImpl; +// import tech.pegasys.teku.ssz.primitive.SszBit; +// import tech.pegasys.teku.ssz.schema.SszVectorSchema; +// import tech.pegasys.teku.ssz.schema.collections.SszBitvectorSchema; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszBitvectorImpl extends SszVectorImpl implements SszBitvector { + + public static SszBitvectorImpl ofBits(SszBitvectorSchema schema, int... bits) { + return new SszBitvectorImpl(schema, new BitvectorImpl(schema.getLength(), bits)); + } + + private final BitvectorImpl value; + + public SszBitvectorImpl(SszVectorSchema schema, TreeNode backingNode) { + super(schema, backingNode); + value = BitvectorImpl.fromBytes(sszSerialize(), size()); + } + + public SszBitvectorImpl(SszBitvectorSchema schema, BitvectorImpl value) { + super(schema, () -> schema.sszDeserializeTree(SszReader.fromBytes(value.serialize()))); + checkNotNull(value); + this.value = value; + } + + @SuppressWarnings("unchecked") + @Override + public SszBitvectorSchema getSchema() { + return (SszBitvectorSchema) super.getSchema(); + } + + @Override + protected IntCache createCache() { + // BitvectorImpl is far more effective cache than caching individual bits + return IntCache.noop(); + } + + @Override + public boolean getBit(int i) { + return value.getBit(i); + } + + @Override + public int getBitCount() { + return value.getBitCount(); + } + + @Override + public SszBitvector rightShift(int n) { + return new SszBitvectorImpl(getSchema(), value.rightShift(n)); + } + + @Override + public List getAllSetBits() { + return value.streamAllSetBits().boxed().collect(Collectors.toList()); + } + + @Override + public SszBitvector withBit(int i) { + return new SszBitvectorImpl(getSchema(), value.withBit(i)); + } + + @Override + protected int sizeImpl() { + return getSchema().getLength(); + } + + @Override + public SszMutablePrimitiveVector createWritableCopy() { + throw new UnsupportedOperationException("SszBitlist is immutable structure"); + } + + @Override + public boolean isWritableSupported() { + return false; + } + + @Override + public String toString() { + return "SszBitvector{size=" + this.size() + ", " + value.toString() + "}"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBitvectorSchema.java b/src/org/minima/system/network/base/ssz/SszBitvectorSchema.java new file mode 100644 index 000000000..4bf61831f --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBitvectorSchema.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.stream.StreamSupport; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.collections.SszBitvector; +// import tech.pegasys.teku.ssz.primitive.SszBit; +// import tech.pegasys.teku.ssz.schema.collections.impl.SszBitvectorSchemaImpl; + +public interface SszBitvectorSchema + extends SszPrimitiveVectorSchema { + + static SszBitvectorSchema create(long length) { + return new SszBitvectorSchemaImpl(length); + } + + SszBitvectorT ofBits(int... setBitIndexes); + + default SszBitvectorT fromBytes(Bytes bivectorBytes) { + return sszDeserialize(bivectorBytes); + } + + default SszBitvectorT ofBits(Iterable setBitIndexes) { + int[] indexesArray = + StreamSupport.stream(setBitIndexes.spliterator(), false).mapToInt(i -> i).toArray(); + return ofBits(indexesArray); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBitvectorSchemaImpl.java b/src/org/minima/system/network/base/ssz/SszBitvectorSchemaImpl.java new file mode 100644 index 000000000..579a480a8 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBitvectorSchemaImpl.java @@ -0,0 +1,81 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.List; +import java.util.stream.IntStream; +// import tech.pegasys.teku.ssz.collections.SszBitvector; +// import tech.pegasys.teku.ssz.collections.impl.SszBitvectorImpl; +// import tech.pegasys.teku.ssz.primitive.SszBit; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.schema.collections.SszBitvectorSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszVectorSchema; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +public class SszBitvectorSchemaImpl extends AbstractSszVectorSchema + implements SszBitvectorSchema { + + public SszBitvectorSchemaImpl(long length) { + super(SszPrimitiveSchemas.BIT_SCHEMA, length); + checkArgument(length > 0, "Invalid Bitlist length"); + } + + @Override + public SszBitvector createFromBackingNode(TreeNode node) { + return new SszBitvectorImpl(this, node); + } + + @Override + public SszBitvector ofBits(int... setBitIndexes) { + return SszBitvectorImpl.ofBits(this, setBitIndexes); + } + + @Override + public SszBitvector createFromElements(List elements) { + return ofBits(IntStream.range(0, elements.size()).filter(i -> elements.get(i).get()).toArray()); + } + + @Override + public int sszSerializeTree(TreeNode node, SszWriter writer) { + return sszSerializeVector(node, writer, getLength()); + } + + @Override + public TreeNode sszDeserializeTree(SszReader reader) { + checkSsz( + reader.getAvailableBytes() == TreeUtil.bitsCeilToBytes(getLength()), + "SSZ length doesn't match Bitvector size"); + + DeserializedData data = sszDeserializeVector(reader); + if (getLength() % 8 > 0) { + // for BitVector we need to check that all 'unused' bits in the last byte are 0 + int usedBitCount = getLength() % 8; + if (data.getLastSszByte().orElseThrow() >>> usedBitCount != 0) { + throw new SszDeserializeException("Invalid Bitvector ssz: trailing bits are not 0"); + } + } + return data.getDataTree(); + } + + @Override + public String toString() { + return "Bitvector[" + getMaxLength() + "]"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByte.java b/src/org/minima/system/network/base/ssz/SszByte.java new file mode 100644 index 000000000..5e70157f5 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByte.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.impl.AbstractSszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; + +public class SszByte extends AbstractSszPrimitive { + + public static SszByte of(int value) { + return new SszByte((byte) value); + } + + public static SszByte of(byte value) { + return new SszByte(value); + } + + private SszByte(Byte value) { + super(value, SszPrimitiveSchemas.BYTE_SCHEMA); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByteArrayWriter.java b/src/org/minima/system/network/base/ssz/SszByteArrayWriter.java new file mode 100644 index 000000000..c3e7b5a50 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByteArrayWriter.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; + +public class SszByteArrayWriter implements SszWriter { + private final byte[] bytes; + private int size = 0; + + public SszByteArrayWriter(int maxSize) { + bytes = new byte[maxSize]; + } + + @Override + public void write(byte[] bytes, int offset, int length) { + System.arraycopy(bytes, offset, this.bytes, this.size, length); + this.size += length; + } + + public byte[] getBytesArray() { + return bytes; + } + + public int getLength() { + return size; + } + + public Bytes toBytes() { + return Bytes.wrap(bytes, 0, size); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByteVector.java b/src/org/minima/system/network/base/ssz/SszByteVector.java new file mode 100644 index 000000000..26c6102e7 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByteVector.java @@ -0,0 +1,43 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.primitive.SszByte; +// import tech.pegasys.teku.ssz.schema.collections.SszByteVectorSchema; + +public interface SszByteVector extends SszPrimitiveVector { + + static SszByteVector fromBytes(Bytes byteVector) { + return SszByteVectorSchema.create(byteVector.size()).fromBytes(byteVector); + } + + static Bytes32 computeHashTreeRoot(Bytes byteVector) { + return fromBytes(byteVector).hashTreeRoot(); + } + + default Bytes getBytes() { + byte[] data = new byte[size()]; + for (int i = 0; i < size(); i++) { + data[i] = getElement(i); + } + return Bytes.wrap(data); + } + + @Override + default SszMutablePrimitiveVector createWritableCopy() { + throw new UnsupportedOperationException("Not supported here"); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByteVectorImpl.java b/src/org/minima/system/network/base/ssz/SszByteVectorImpl.java new file mode 100644 index 000000000..9e482d255 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByteVectorImpl.java @@ -0,0 +1,70 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.collections.SszByteVector; +// import tech.pegasys.teku.ssz.collections.SszMutablePrimitiveVector; +// import tech.pegasys.teku.ssz.primitive.SszByte; +// import tech.pegasys.teku.ssz.schema.collections.SszByteVectorSchema; +// import tech.pegasys.teku.ssz.schema.collections.impl.SszByteVectorSchemaImpl; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszByteVectorImpl extends SszPrimitiveVectorImpl + implements SszByteVector { + + private final Bytes data; + + public SszByteVectorImpl(SszByteVectorSchema schema, Bytes bytes) { + super(schema, () -> SszByteVectorSchemaImpl.fromBytesToTree(schema, bytes)); + this.data = bytes; + } + + public SszByteVectorImpl(SszByteVectorSchema schema, TreeNode backingTree) { + super(schema, backingTree); + this.data = SszByteVectorSchemaImpl.fromTreeToBytes(schema, backingTree); + } + + @Override + public Bytes getBytes() { + return data; + } + + @Override + protected IntCache createCache() { + // caching with Bytes in this class + return IntCache.noop(); + } + + @Override + public SszByteVectorSchemaImpl getSchema() { + return (SszByteVectorSchemaImpl) super.getSchema(); + } + + @Override + public SszMutablePrimitiveVector createWritableCopy() { + throw new UnsupportedOperationException("SszBitlist is immutable structure"); + } + + @Override + public boolean isWritableSupported() { + return false; + } + + @Override + public String toString() { + return "SszByteVector{" + data + '}'; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByteVectorSchema.java b/src/org/minima/system/network/base/ssz/SszByteVectorSchema.java new file mode 100644 index 000000000..481999117 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByteVectorSchema.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.collections.SszByteVector; +// import tech.pegasys.teku.ssz.primitive.SszByte; +// import tech.pegasys.teku.ssz.schema.collections.impl.SszByteVectorSchemaImpl; + +public interface SszByteVectorSchema + extends SszPrimitiveVectorSchema { + + SszVectorT fromBytes(Bytes bytes); + + static SszByteVectorSchema create(int length) { + return new SszByteVectorSchemaImpl<>(length); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszByteVectorSchemaImpl.java b/src/org/minima/system/network/base/ssz/SszByteVectorSchemaImpl.java new file mode 100644 index 000000000..602177028 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszByteVectorSchemaImpl.java @@ -0,0 +1,65 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.List; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.collections.SszByteVector; +// import tech.pegasys.teku.ssz.collections.impl.SszByteVectorImpl; +// import tech.pegasys.teku.ssz.primitive.SszByte; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.schema.collections.SszByteVectorSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszVectorSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +public class SszByteVectorSchemaImpl + extends AbstractSszVectorSchema + implements SszByteVectorSchema { + + public SszByteVectorSchemaImpl(long vectorLength) { + super(SszPrimitiveSchemas.BYTE_SCHEMA, vectorLength); + } + + @Override + @SuppressWarnings("unchecked") + public SszVectorT createFromBackingNode(TreeNode node) { + return (SszVectorT) new SszByteVectorImpl(this, node); + } + + @Override + @SuppressWarnings("unchecked") + public SszVectorT fromBytes(Bytes bytes) { + return (SszVectorT) new SszByteVectorImpl(this, bytes); + } + + public static TreeNode fromBytesToTree(SszByteVectorSchema schema, Bytes bytes) { + checkArgument(bytes.size() == schema.getLength(), "Bytes size doesn't match vector length"); + return SchemaUtils.createTreeFromBytes(bytes, schema.treeDepth()); + } + + public static Bytes fromTreeToBytes(SszByteVectorSchema schema, TreeNode tree) { + Bytes bytes = TreeUtil.concatenateLeavesData(tree); + checkArgument(bytes.size() == schema.getLength(), "Tree doesn't match vector schema"); + return bytes; + } + + @Override + public SszVectorT createFromElements(List elements) { + Bytes bytes = Bytes.of(elements.stream().mapToInt(sszByte -> 0xFF & sszByte.get()).toArray()); + return fromBytes(bytes); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes32.java b/src/org/minima/system/network/base/ssz/SszBytes32.java new file mode 100644 index 000000000..0ce481212 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes32.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.impl.AbstractSszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; + +public class SszBytes32 extends AbstractSszPrimitive { + + public static SszBytes32 of(Bytes32 val) { + return new SszBytes32(val); + } + + private SszBytes32(Bytes32 val) { + super(val, SszPrimitiveSchemas.BYTES32_SCHEMA); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes32Vector.java b/src/org/minima/system/network/base/ssz/SszBytes32Vector.java new file mode 100644 index 000000000..c7068db6f --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes32Vector.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; +//import tech.pegasys.teku.ssz.primitive.SszBytes32; + +public interface SszBytes32Vector extends SszPrimitiveVector { + + @Override + SszMutableBytes32Vector createWritableCopy(); +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes32VectorImpl.java b/src/org/minima/system/network/base/ssz/SszBytes32VectorImpl.java new file mode 100644 index 000000000..edd6609b7 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes32VectorImpl.java @@ -0,0 +1,39 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Supplier; +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.collections.SszBytes32Vector; +// import tech.pegasys.teku.ssz.collections.SszMutableBytes32Vector; +// import tech.pegasys.teku.ssz.primitive.SszBytes32; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszBytes32VectorImpl extends SszPrimitiveVectorImpl + implements SszBytes32Vector { + + public SszBytes32VectorImpl(SszCompositeSchema schema, Supplier lazyBackingNode) { + super(schema, lazyBackingNode); + } + + public SszBytes32VectorImpl(SszCompositeSchema schema, TreeNode backingNode) { + super(schema, backingNode); + } + + @Override + public SszMutableBytes32Vector createWritableCopy() { + return new SszMutableBytes32VectorImpl(this); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes32VectorSchema.java b/src/org/minima/system/network/base/ssz/SszBytes32VectorSchema.java new file mode 100644 index 000000000..b68cb17c6 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes32VectorSchema.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.collections.SszBytes32Vector; +// import tech.pegasys.teku.ssz.primitive.SszBytes32; +// import tech.pegasys.teku.ssz.schema.collections.impl.SszBytes32VectorSchemaImpl; + +public interface SszBytes32VectorSchema + extends SszPrimitiveVectorSchema { + + static SszBytes32VectorSchema create(int length) { + return new SszBytes32VectorSchemaImpl<>(length); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes32VectorSchemaImpl.java b/src/org/minima/system/network/base/ssz/SszBytes32VectorSchemaImpl.java new file mode 100644 index 000000000..54f065683 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes32VectorSchemaImpl.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.collections.SszBytes32Vector; +// import tech.pegasys.teku.ssz.collections.impl.SszBytes32VectorImpl; +// import tech.pegasys.teku.ssz.primitive.SszBytes32; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.schema.collections.SszBytes32VectorSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszVectorSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszBytes32VectorSchemaImpl + extends AbstractSszVectorSchema + implements SszBytes32VectorSchema { + + public SszBytes32VectorSchemaImpl(long vectorLength) { + super(SszPrimitiveSchemas.BYTES32_SCHEMA, vectorLength); + } + + @Override + @SuppressWarnings("unchecked") + public SszVectorT createFromBackingNode(TreeNode node) { + return (SszVectorT) new SszBytes32VectorImpl(this, node); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszBytes4.java b/src/org/minima/system/network/base/ssz/SszBytes4.java new file mode 100644 index 000000000..01bb49be3 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszBytes4.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.impl.AbstractSszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.type.Bytes4; + +public class SszBytes4 extends AbstractSszPrimitive { + + public static SszBytes4 of(Bytes4 val) { + return new SszBytes4(val); + } + + private SszBytes4(Bytes4 val) { + super(val, SszPrimitiveSchemas.BYTES4_SCHEMA); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszCollection.java b/src/org/minima/system/network/base/ssz/SszCollection.java new file mode 100644 index 000000000..fa0768e17 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszCollection.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.AbstractList; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; +import org.jetbrains.annotations.NotNull; +//import tech.pegasys.teku.ssz.schema.SszCollectionSchema; + +public interface SszCollection + extends SszComposite, Iterable { + + default boolean isEmpty() { + return size() == 0; + } + + default List asList() { + return new AbstractList<>() { + @Override + public ElementT get(int index) { + return SszCollection.this.get(index); + } + + @Override + public int size() { + return SszCollection.this.size(); + } + }; + } + + @NotNull + @Override + default Iterator iterator() { + return asList().iterator(); + } + + default Stream stream() { + return asList().stream(); + } + + @Override + SszCollectionSchema getSchema(); +} diff --git a/src/org/minima/system/network/base/ssz/SszCollectionSchema.java b/src/org/minima/system/network/base/ssz/SszCollectionSchema.java new file mode 100644 index 000000000..dde2c6e31 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszCollectionSchema.java @@ -0,0 +1,57 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; +// import tech.pegasys.teku.ssz.SszCollection; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableComposite; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public interface SszCollectionSchema< + SszElementT extends SszData, SszCollectionT extends SszCollection> + extends SszCompositeSchema { + + SszSchema getElementSchema(); + + @SuppressWarnings("unchecked") + default SszCollectionT of(SszElementT... elements) { + return createFromElements(Arrays.asList(elements)); + } + + @SuppressWarnings("unchecked") + default SszCollectionT createFromElements(List elements) { + checkArgument(elements.size() <= getMaxLength(), "Too many elements for this collection type"); + SszMutableComposite writableCopy = getDefault().createWritableCopy(); + int idx = 0; + for (SszElementT element : elements) { + writableCopy.set(idx++, element); + } + return (SszCollectionT) writableCopy.commitChanges(); + } + + default TreeNode createTreeFromElements(List elements) { + // TODO: suboptimal + return createFromElements(elements).getBackingNode(); + } + + default Collector collector() { + return Collectors.collectingAndThen(Collectors.toList(), this::createFromElements); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszComposite.java b/src/org/minima/system/network/base/ssz/SszComposite.java new file mode 100644 index 000000000..707c954f6 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszComposite.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.schema.SszCompositeSchema; + +/** + * Represents composite immutable ssz structure which has descendant ssz structures + * + * @param the type of children {@link SszData} + */ +public interface SszComposite extends SszData { + + /** Returns number of children in this structure */ + default int size() { + return (int) getSchema().getMaxLength(); + } + + /** + * Returns the child at index + * + * @throws IndexOutOfBoundsException if index >= size() + */ + SszChildT get(int index); + + @Override + SszCompositeSchema getSchema(); + + @Override + SszMutableComposite createWritableCopy(); +} diff --git a/src/org/minima/system/network/base/ssz/SszCompositeSchema.java b/src/org/minima/system/network/base/ssz/SszCompositeSchema.java new file mode 100644 index 000000000..9d1b7b7d3 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszCompositeSchema.java @@ -0,0 +1,88 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszComposite; +// import tech.pegasys.teku.ssz.tree.GIndexUtil; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUtil; + +/** Abstract schema of {@link SszComposite} subclasses */ +public interface SszCompositeSchema> + extends SszSchema { + + /** + * Returns the maximum number of elements in ssz structures of this scheme. For structures with + * fixed number of children (like Containers and Vectors) their size should always be equal to + * maxLength + */ + long getMaxLength(); + + /** + * Returns the child schema at index. For homogeneous structures (like Vector, List) the returned + * schema is the same for any index For heterogeneous structures (like Container) each child has + * individual schema + * + * @throws IndexOutOfBoundsException if index >= getMaxLength + */ + SszSchema getChildSchema(int index); + + /** + * Return the number of elements that may be stored in a single tree node This value is 1 for all + * types except of packed basic lists/vectors + */ + default int getElementsPerChunk() { + return 1; + } + + /** + * Returns the maximum number of this ssz structure backed subtree 'leaf' nodes required to store + * maxLength elements + */ + default long maxChunks() { + return (getMaxLength() - 1) / getElementsPerChunk() + 1; + } + + /** + * Returns then number of chunks (i.e. leaf nodes) to store {@code elementCount} child elements + * Returns a number lower than {@code elementCode} only in case of packed basic types collection + */ + default int getChunks(int elementCount) { + return (elementCount - 1) / getElementsPerChunk() + 1; + } + + /** Returns the backed binary tree depth to store maxLength elements */ + default int treeDepth() { + return Long.bitCount(treeWidth() - 1); + } + + /** Returns the backed binary tree width to store maxLength elements */ + default long treeWidth() { + return TreeUtil.nextPowerOf2(maxChunks()); + } + + /** + * Returns binary backing tree generalized index corresponding to child element index + * + * @see TreeNode#get(long) + */ + default long getChildGeneralizedIndex(long elementIndex) { + return GIndexUtil.gIdxChildGIndex(GIndexUtil.SELF_G_INDEX, elementIndex, treeDepth()); + } + + @Override + default boolean isPrimitive() { + return false; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszContainer.java b/src/org/minima/system/network/base/ssz/SszContainer.java new file mode 100644 index 000000000..c0ce199fe --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszContainer.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.schema.SszContainerSchema; + +/** + * Base class for immutable containers. Since containers are heterogeneous their generic child type + * is {@link SszData} + */ +public interface SszContainer extends SszComposite { + + @Override + SszContainerSchema getSchema(); + + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) + // container is heterogeneous by its nature so making unsafe cast here + // is more convenient and is not less safe + default C getAny(int index) { + return (C) get(index); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszContainerImpl.java b/src/org/minima/system/network/base/ssz/SszContainerImpl.java new file mode 100644 index 000000000..7d9142430 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszContainerImpl.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.stream.Collectors; +import java.util.stream.IntStream; +// import tech.pegasys.teku.ssz.SszContainer; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableContainer; +// import tech.pegasys.teku.ssz.cache.ArrayIntCache; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.schema.SszContainerSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszContainerImpl extends AbstractSszComposite implements SszContainer { + + public SszContainerImpl(SszContainerSchema type) { + this(type, type.getDefaultTree()); + } + + public SszContainerImpl(SszContainerSchema type, TreeNode backingNode) { + super(type, backingNode); + } + + public SszContainerImpl( + SszCompositeSchema type, TreeNode backingNode, IntCache cache) { + super(type, backingNode, cache); + } + + @Override + protected SszData getImpl(int index) { + SszCompositeSchema type = this.getSchema(); + TreeNode node = getBackingNode().get(type.getChildGeneralizedIndex(index)); + return type.getChildSchema(index).createFromBackingNode(node); + } + + @Override + public AbstractSszContainerSchema getSchema() { + return (AbstractSszContainerSchema) super.getSchema(); + } + + @Override + public SszMutableContainer createWritableCopy() { + return new SszMutableContainerImpl(this); + } + + @Override + protected int sizeImpl() { + return (int) this.getSchema().getMaxLength(); + } + + @Override + protected IntCache createCache() { + return new ArrayIntCache<>(size()); + } + + @Override + protected void checkIndex(int index) { + if (index >= size()) { + throw new IndexOutOfBoundsException( + "Invalid index " + index + " for container with size " + size()); + } + } + + @Override + public String toString() { + return this.getSchema().getContainerName() + + "{" + + IntStream.range(0, this.getSchema().getFieldsCount()) + .mapToObj(idx -> this.getSchema().getFieldNames().get(idx) + "=" + get(idx)) + .collect(Collectors.joining(", ")) + + "}"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszContainerSchema.java b/src/org/minima/system/network/base/ssz/SszContainerSchema.java new file mode 100644 index 000000000..b9f0160a6 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszContainerSchema.java @@ -0,0 +1,97 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.List; +import java.util.function.BiFunction; +// import tech.pegasys.teku.ssz.SszContainer; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema.NamedSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +import org.minima.system.network.base.ssz.AbstractSszContainerSchema.NamedSchema; + +/** + * {@link SszSchema} for an Ssz Container structure + * + * @param the type of actual container class + */ +public interface SszContainerSchema extends SszCompositeSchema { + + /** + * Creates a new {@link SszContainer} schema with specified field schemas and container instance + * constructor + */ + static SszContainerSchema create( + List> childrenSchemas, + BiFunction, TreeNode, C> instanceCtor) { + return new AbstractSszContainerSchema(childrenSchemas) { + @Override + public C createFromBackingNode(TreeNode node) { + return instanceCtor.apply(this, node); + } + }; + } + + static SszContainerSchema create( + String containerName, + List> childrenSchemas, + BiFunction, TreeNode, C> instanceCtor) { + return new AbstractSszContainerSchema(containerName, childrenSchemas) { + @Override + public C createFromBackingNode(TreeNode node) { + return instanceCtor.apply(this, node); + } + }; + } + + /** + * Get the index of a field by name + * + * @param fieldName + * @return The index if it exists, otherwise -1 + */ + int getFieldIndex(String fieldName); + + /** + * Creates the backing tree from container field values + * + * @throws IllegalArgumentException if value types doesn't match this scheme field types + */ + TreeNode createTreeFromFieldValues(List fieldValues); + + /** + * Creates an {@link SszContainer} instance from field values + * + * @throws IllegalArgumentException if value types doesn't match this scheme field types + */ + default C createFromFieldValues(List fieldValues) { + return createFromBackingNode(createTreeFromFieldValues(fieldValues)); + } + + /** Returns the number of fields in ssz containers of this type */ + default int getFieldsCount() { + return (int) getMaxLength(); + } + + /** Returns this container name */ + String getContainerName(); + + /** Return this container field names */ + List getFieldNames(); + + /** Return this container field schemas */ + List> getFieldSchemas(); +} diff --git a/src/org/minima/system/network/base/ssz/SszData.java b/src/org/minima/system/network/base/ssz/SszData.java new file mode 100644 index 000000000..822c81a8f --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszData.java @@ -0,0 +1,59 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** + * Base class of immutable views over Binary Backing Tree ({@link TreeNode}) Overlay views concept + * described here: + * https://github.com/protolambda/eth-merkle-trees/blob/master/typing_partials.md#views + */ +public interface SszData extends Merkleizable, SimpleOffsetSerializable { + + /** + * Creates a corresponding writeable copy of this immutable structure Any modifications made to + * the returned copy affect neither this structure nor its descendant structures + */ + SszMutableData createWritableCopy(); + + default boolean isWritableSupported() { + return true; + } + + /** Gets the schema of this structure */ + SszSchema getSchema(); + + /** Returns Backing Tree this structure is backed by */ + TreeNode getBackingNode(); + + @Override + default Bytes32 hashTreeRoot() { + return getBackingNode().hashTreeRoot(); + } + + @Override + default Bytes sszSerialize() { + return getSchema().sszSerializeTree(getBackingNode()); + } + + @Override + default int sszSerialize(SszWriter writer) { + return getSchema().sszSerializeTree(getBackingNode(), writer); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszDeserializeException.java b/src/org/minima/system/network/base/ssz/SszDeserializeException.java new file mode 100644 index 000000000..9e122bd0b --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszDeserializeException.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +public class SszDeserializeException extends IllegalArgumentException { + + public SszDeserializeException(String s) { + super(s); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszLengthBounds.java b/src/org/minima/system/network/base/ssz/SszLengthBounds.java new file mode 100644 index 000000000..e39be4e9d --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszLengthBounds.java @@ -0,0 +1,116 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static org.minima.system.network.base.ssz.TreeUtil.bitsCeilToBytes; + +import com.google.common.base.MoreObjects; +import java.util.Objects; + +public class SszLengthBounds { + public static final SszLengthBounds ZERO = new SszLengthBounds(0, 0); + private final long min; + private final long max; + + public static SszLengthBounds ofBits(long fixedSize) { + return new SszLengthBounds(fixedSize, fixedSize); + } + + public static SszLengthBounds ofBits(long min, long max) { + return new SszLengthBounds(min, max); + } + + public static SszLengthBounds ofBytes(long fixedSize) { + return new SszLengthBounds(fixedSize * 8, fixedSize * 8); + } + + public static SszLengthBounds ofBytes(long min, long max) { + return new SszLengthBounds(min * 8, max * 8); + } + + private SszLengthBounds(final long min, final long max) { + this.min = min; + this.max = max; + } + + public long getMinBits() { + return min; + } + + public long getMaxBits() { + return max; + } + + public long getMinBytes() { + return bitsCeilToBytes(min); + } + + public long getMaxBytes() { + return bitsCeilToBytes(max); + } + + public SszLengthBounds ceilToBytes() { + return SszLengthBounds.ofBytes(getMinBytes(), getMaxBytes()); + } + + public SszLengthBounds add(final SszLengthBounds other) { + return new SszLengthBounds(this.min + other.min, this.max + other.max); + } + + public SszLengthBounds addBytes(final int moreBytes) { + return addBits(moreBytes * 8); + } + + public SszLengthBounds addBits(final int moreBits) { + return new SszLengthBounds(this.min + moreBits, this.max + moreBits); + } + + public SszLengthBounds mul(final long factor) { + return new SszLengthBounds(this.min * factor, this.max * factor); + } + + public boolean isWithinBounds(final long lengthBytes) { + return lengthBytes >= getMinBytes() && lengthBytes <= getMaxBytes(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final SszLengthBounds that = (SszLengthBounds) o; + return getMinBits() == that.getMinBits() && getMaxBits() == that.getMaxBits(); + } + + @Override + public int hashCode() { + return Objects.hash(getMinBits(), getMaxBits()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("min", fromBits(getMinBits())) + .add("max", fromBits(getMaxBits())) + .toString(); + } + + private static String fromBits(long bits) { + long bytes = bits / 8; + return "" + bytes + ((bits & 7) == 0 ? "" : "(+" + (bits - bytes * 8) + " bits)"); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableBytes32Vector.java b/src/org/minima/system/network/base/ssz/SszMutableBytes32Vector.java new file mode 100644 index 000000000..e65d915ff --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableBytes32Vector.java @@ -0,0 +1,24 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; +//import tech.pegasys.teku.ssz.primitive.SszBytes32; + +public interface SszMutableBytes32Vector + extends SszMutablePrimitiveVector, SszBytes32Vector { + + @Override + SszBytes32Vector commitChanges(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableBytes32VectorImpl.java b/src/org/minima/system/network/base/ssz/SszMutableBytes32VectorImpl.java new file mode 100644 index 000000000..63f7b2b66 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableBytes32VectorImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes32; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.collections.SszBytes32Vector; +// import tech.pegasys.teku.ssz.collections.SszMutableBytes32Vector; +// import tech.pegasys.teku.ssz.impl.AbstractSszComposite; +// import tech.pegasys.teku.ssz.primitive.SszBytes32; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszMutableBytes32VectorImpl extends SszMutablePrimitiveVectorImpl + implements SszMutableBytes32Vector { + + public SszMutableBytes32VectorImpl(AbstractSszComposite backingImmutableData) { + super(backingImmutableData); + } + + @Override + public SszBytes32Vector commitChanges() { + return (SszBytes32Vector) super.commitChanges(); + } + + @Override + protected SszBytes32VectorImpl createImmutableSszComposite( + TreeNode backingNode, IntCache childrenCache) { + return new SszBytes32VectorImpl(getSchema(), backingNode); + } + + @Override + public SszMutableBytes32VectorImpl createWritableCopy() { + throw new UnsupportedOperationException( + "Creating a writable copy from writable instance is not supported"); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableComposite.java b/src/org/minima/system/network/base/ssz/SszMutableComposite.java new file mode 100644 index 000000000..6f7ca681c --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableComposite.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * Represents composite mutable ssz structure which has descendant ssz structures + * + * @param the type of children + */ +public interface SszMutableComposite + extends SszMutableData, SszComposite { + + /** + * Sets the function which should called by the implementation on any changes in this structure or + * its descendant structures. + * + *

This is to propagate changes up in the data hierarchy from child mutable structures to + * parent mutable structures. + * + * @param listener listener to be called with this instance as a parameter + */ + void setInvalidator(Consumer listener); + + /** + * Sets the child at the index. + * + *

If {@code index == size()} and the structure is extendable (e.g. List) then this is treated + * as `append()` operation and the size is expanded. In this case `size` should be less than + * `maxSize` + * + * @throws IndexOutOfBoundsException if index > size() or if index == size() but size() == maxSize + */ + void set(int index, SszChildT value); + + /** + * Similar to {@link #set(int, SszData)} but using modifier function which may consider old value + * to calculate new value. The implementation may potentially optimize this case. + */ + default void update(int index, Function mutator) { + set(index, mutator.apply(get(index))); + } + + default void setAll(Iterable newChildren) { + clear(); + int idx = 0; + for (SszChildT newChild : newChildren) { + set(idx++, newChild); + } + } + + @Override + SszComposite commitChanges(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableContainer.java b/src/org/minima/system/network/base/ssz/SszMutableContainer.java new file mode 100644 index 000000000..be265fc16 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableContainer.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** Base class for mutable containers. */ +public interface SszMutableContainer extends SszMutableComposite, SszContainer { + + @Override + SszContainer commitChanges(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableContainerImpl.java b/src/org/minima/system/network/base/ssz/SszMutableContainerImpl.java new file mode 100644 index 000000000..ca821ab7d --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableContainerImpl.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.List; +import java.util.Map; +// import tech.pegasys.teku.ssz.SszContainer; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableContainer; +// import tech.pegasys.teku.ssz.SszMutableData; +// import tech.pegasys.teku.ssz.SszMutableRefContainer; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszContainerSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.tree.TreeUpdates; + +public class SszMutableContainerImpl extends AbstractSszMutableComposite + implements SszMutableRefContainer { + + public SszMutableContainerImpl(SszContainerImpl backingImmutableView) { + super(backingImmutableView); + } + + @Override + protected SszContainerImpl createImmutableSszComposite( + TreeNode backingNode, IntCache viewCache) { + return new SszContainerImpl(getSchema(), backingNode, viewCache); + } + + @Override + public AbstractSszContainerSchema getSchema() { + return (AbstractSszContainerSchema) super.getSchema(); + } + + @Override + public SszContainer commitChanges() { + return (SszContainer) super.commitChanges(); + } + + @Override + public SszMutableContainer createWritableCopy() { + throw new UnsupportedOperationException( + "createWritableCopy() is now implemented for immutable SszData only"); + } + + @Override + protected void checkIndex(int index, boolean set) { + if (index >= size()) { + throw new IndexOutOfBoundsException( + "Invalid index " + index + " for container with size " + size()); + } + } + + @Override + protected TreeUpdates packChanges( + List> newChildValues, TreeNode original) { + throw new UnsupportedOperationException("Packed values are not supported"); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableData.java b/src/org/minima/system/network/base/ssz/SszMutableData.java new file mode 100644 index 000000000..1488f33aa --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableData.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.tree.TreeNode; + +/** + * Base class of mutable structures over Binary Backing Tree ({@link TreeNode}) Each {@link + * SszMutableData} subclass class normally inherits from the corresponding immutable class to have + * both get/set methods however {@link SszMutableData} instance shouldn't be leaked as {@link + * SszData} instance, the {@link #commitChanges()} should be used instead to get immutable structure + */ +public interface SszMutableData extends SszData { + + /** Resets this ssz structure to its default value */ + void clear(); + + /** Creates the corresponding immutable structure with all the changes */ + SszData commitChanges(); + + /** + * Returns the backing tree of this modified structure. + * + *

Note that calling this method on {@link SszMutableData} could be suboptimal from performance + * perspective as it internally needs to create an immutable {@link SszData} via {@link + * #commitChanges()} which is then discarded. It's normally better to make all modifications on + * {@link SszMutableData}, commit the changes and then call either {@link + * SszData#getBackingNode()} or {@link SszData#hashTreeRoot()} on the resulting immutable instance + */ + @Override + default TreeNode getBackingNode() { + return commitChanges().getBackingNode(); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutablePrimitiveCollection.java b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveCollection.java new file mode 100644 index 000000000..cee33b7b1 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveCollection.java @@ -0,0 +1,44 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszMutableComposite; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; + +public interface SszMutablePrimitiveCollection< + ElementT, SszElementT extends SszPrimitive> + extends SszPrimitiveCollection, SszMutableComposite { + + @SuppressWarnings("unchecked") + default SszPrimitiveSchema getPrimitiveElementSchema() { + return (SszPrimitiveSchema) getSchema().getElementSchema(); + } + + default void setElement(int index, ElementT primitiveValue) { + SszElementT sszData = getPrimitiveElementSchema().boxed(primitiveValue); + set(index, sszData); + } + + default void setAllElements(Iterable newChildren) { + clear(); + int idx = 0; + for (ElementT newChild : newChildren) { + setElement(idx++, newChild); + } + } + + @Override + SszPrimitiveCollection commitChanges(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVector.java b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVector.java new file mode 100644 index 000000000..64adf2a46 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVector.java @@ -0,0 +1,30 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszMutableVector; +// import tech.pegasys.teku.ssz.SszPrimitive; + +public interface SszMutablePrimitiveVector< + ElementT, SszElementT extends SszPrimitive> + extends SszMutablePrimitiveCollection, + SszMutableVector, + SszPrimitiveVector { + + @Override + SszPrimitiveVector commitChanges(); + + @Override + SszMutablePrimitiveVector createWritableCopy(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVectorImpl.java b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVectorImpl.java new file mode 100644 index 000000000..d66465ce5 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutablePrimitiveVectorImpl.java @@ -0,0 +1,50 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.collections.SszMutablePrimitiveVector; +// import tech.pegasys.teku.ssz.collections.SszPrimitiveVector; +// import tech.pegasys.teku.ssz.impl.AbstractSszComposite; +// import tech.pegasys.teku.ssz.impl.SszMutableVectorImpl; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszMutablePrimitiveVectorImpl< + ElementT, SszElementT extends SszPrimitive> + extends SszMutableVectorImpl + implements SszMutablePrimitiveVector { + + public SszMutablePrimitiveVectorImpl(AbstractSszComposite backingImmutableData) { + super(backingImmutableData); + } + + @Override + @SuppressWarnings("unchecked") + public SszPrimitiveVector commitChanges() { + return (SszPrimitiveVector) super.commitChanges(); + } + + @Override + protected AbstractSszComposite createImmutableSszComposite( + TreeNode backingNode, IntCache childrenCache) { + return new SszPrimitiveVectorImpl<>(getSchema(), backingNode); + } + + @Override + public SszMutablePrimitiveVector createWritableCopy() { + throw new UnsupportedOperationException( + "Creating a writable copy from writable instance is not supported"); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableRefComposite.java b/src/org/minima/system/network/base/ssz/SszMutableRefComposite.java new file mode 100644 index 000000000..4dd1c98fc --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableRefComposite.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** + * Represents a mutable {@link SszComposite} which is able to return a mutable child 'by reference'. + * Any modifications made to such child are reflected in this structure and its backing tree + */ +public interface SszMutableRefComposite< + ChildReadType extends SszData, ChildWriteType extends ChildReadType> + extends SszMutableComposite { + + /** + * Returns a mutable child at index 'by reference' Any modifications made to such child are + * reflected in this structure and its backing tree + * + * @throws IndexOutOfBoundsException if index >= size() + */ + ChildWriteType getByRef(int index); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableRefContainer.java b/src/org/minima/system/network/base/ssz/SszMutableRefContainer.java new file mode 100644 index 000000000..30a6195ca --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableRefContainer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** + * Base class for mutable {@link SszContainer} which is able to return mutable children by reference + */ +public interface SszMutableRefContainer + extends SszMutableRefComposite, SszMutableContainer { + + @SuppressWarnings({"unchecked", "TypeParameterUnusedInFormals"}) + // container is heterogeneous by its nature so making unsafe cast here + // is more convenient and is not less safe + default W getAnyByRef(int index) { + return (W) getByRef(index); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableRefVector.java b/src/org/minima/system/network/base/ssz/SszMutableRefVector.java new file mode 100644 index 000000000..02cd89e47 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableRefVector.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** + * Represents a mutable {@link SszVector} which is able to return a mutable child 'by reference' Any + * modifications made to such child are reflected in this vector and its backing tree + * + * @param Class of immutable child views + * @param Class of the corresponding mutable child views + */ +public interface SszMutableRefVector< + SszElementT extends SszData, SszMutableElementT extends SszElementT> + extends SszMutableRefComposite, SszMutableVector { + + /** + * Returns a mutable child at index 'by reference' Any modifications made to such child are + * reflected in this structure and its backing tree + * + * @throws IndexOutOfBoundsException if index >= size() + */ + @Override + SszMutableElementT getByRef(int index); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableVector.java b/src/org/minima/system/network/base/ssz/SszMutableVector.java new file mode 100644 index 000000000..a584f3b53 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableVector.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** + * Mutable {@link SszVector} with immutable elements. This type of vector can be modified by setting + * immutable elements + * + * @param Type of elements + */ +public interface SszMutableVector + extends SszMutableComposite, SszVector { + + @Override + SszVector commitChanges(); +} diff --git a/src/org/minima/system/network/base/ssz/SszMutableVectorImpl.java b/src/org/minima/system/network/base/ssz/SszMutableVectorImpl.java new file mode 100644 index 000000000..2f952ec6b --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszMutableVectorImpl.java @@ -0,0 +1,63 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableRefVector; +// import tech.pegasys.teku.ssz.SszMutableVector; +// import tech.pegasys.teku.ssz.SszVector; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszVectorSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszMutableVectorImpl< + SszElementT extends SszData, SszMutableElementT extends SszElementT> + extends AbstractSszMutableCollection + implements SszMutableRefVector { + + public SszMutableVectorImpl(AbstractSszComposite backingImmutableData) { + super(backingImmutableData); + } + + @Override + protected AbstractSszComposite createImmutableSszComposite( + TreeNode backingNode, IntCache childrenCache) { + return new SszVectorImpl<>(getSchema(), backingNode, childrenCache); + } + + @Override + @SuppressWarnings("unchecked") + public SszVectorSchema getSchema() { + return (SszVectorSchema) super.getSchema(); + } + + @Override + @SuppressWarnings("unchecked") + public SszVector commitChanges() { + return (SszVector) super.commitChanges(); + } + + @Override + protected void checkIndex(int index, boolean set) { + if (index < 0 || index >= size()) { + throw new IndexOutOfBoundsException( + "Invalid index " + index + " for vector with size " + size()); + } + } + + @Override + public SszMutableVector createWritableCopy() { + throw new UnsupportedOperationException(); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszNodeTemplate.java b/src/org/minima/system/network/base/ssz/SszNodeTemplate.java new file mode 100644 index 000000000..e570f222e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszNodeTemplate.java @@ -0,0 +1,208 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.LEFTMOST_G_INDEX; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.RIGHTMOST_G_INDEX; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.SELF_G_INDEX; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.gIdxIsSelf; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.gIdxLeftGIndex; +// import static tech.pegasys.teku.ssz.tree.GIndexUtil.gIdxRightGIndex; + +import static org.minima.system.network.base.ssz.GIndexUtil.LEFTMOST_G_INDEX; +import static org.minima.system.network.base.ssz.GIndexUtil.RIGHTMOST_G_INDEX; +import static org.minima.system.network.base.ssz.GIndexUtil.SELF_G_INDEX; +import static org.minima.system.network.base.ssz.GIndexUtil.gIdxIsSelf; +import static org.minima.system.network.base.ssz.GIndexUtil.gIdxLeftGIndex; +import static org.minima.system.network.base.ssz.GIndexUtil.gIdxRightGIndex; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; +//import org.apache.tuweni.crypto.Hash; +//import tech.pegasys.teku.ssz.schema.SszSchema; +import org.minima.utils.Crypto; + +/** + * Represents the tree structure for a fixed size SSZ type See {@link SszSuperNode} docs for more + * details + */ +public class SszNodeTemplate { + + static final class Location { + private final int offset; + private final int length; + private final boolean leaf; + + public Location(int offset, int length, boolean leaf) { + this.offset = offset; + this.length = length; + this.leaf = leaf; + } + + public Location withAddedOffset(int addOffset) { + return new Location(getOffset() + addOffset, getLength(), isLeaf()); + } + + public int getOffset() { + return offset; + } + + public int getLength() { + return length; + } + + public boolean isLeaf() { + return leaf; + } + } + + public static SszNodeTemplate createFromType(SszSchema sszSchema) { + checkArgument(sszSchema.isFixedSize(), "Only fixed size types supported"); + + return createFromTree(sszSchema.getDefaultTree()); + } + + // This should be CANONICAL binary tree + private static SszNodeTemplate createFromTree(TreeNode defaultTree) { + Map gIdxToLoc = + binaryTraverse( + GIndexUtil.SELF_G_INDEX, + defaultTree, + new BinaryVisitor<>() { + @Override + public Map visitLeaf(long gIndex, LeafNode node) { + Map ret = new HashMap<>(); + ret.put(gIndex, new Location(0, node.getData().size(), true)); + return ret; + } + + @Override + public Map visitBranch( + long gIndex, + TreeNode node, + Map leftVisitResult, + Map rightVisitResult) { + Location leftChildLoc = leftVisitResult.get(gIdxLeftGIndex(gIndex)); + Location rightChildLoc = rightVisitResult.get(gIdxRightGIndex(gIndex)); + rightVisitResult.replaceAll( + (idx, loc) -> loc.withAddedOffset(leftChildLoc.getLength())); + leftVisitResult.putAll(rightVisitResult); + leftVisitResult.put( + gIndex, + new Location(0, leftChildLoc.getLength() + rightChildLoc.getLength(), false)); + return leftVisitResult; + } + }); + return new SszNodeTemplate(gIdxToLoc, defaultTree); + } + + private static List nodeSsz(TreeNode node) { + List sszBytes = new ArrayList<>(); + TreeUtil.iterateLeavesData(node, LEFTMOST_G_INDEX, RIGHTMOST_G_INDEX, sszBytes::add); + return sszBytes; + } + + private final Map gIdxToLoc; + private final TreeNode defaultTree; + private final Map subTemplatesCache = new ConcurrentHashMap<>(); + + public SszNodeTemplate(Map gIdxToLoc, TreeNode defaultTree) { + this.gIdxToLoc = gIdxToLoc; + this.defaultTree = defaultTree; + } + + public Location getNodeSszLocation(long generalizedIndex) { + return gIdxToLoc.get(generalizedIndex); + } + + public int getSszLength() { + return gIdxToLoc.get(SELF_G_INDEX).getLength(); + } + + public SszNodeTemplate getSubTemplate(long generalizedIndex) { + return subTemplatesCache.computeIfAbsent(generalizedIndex, this::calcSubTemplate); + } + + private SszNodeTemplate calcSubTemplate(long generalizedIndex) { + if (gIdxIsSelf(generalizedIndex)) { + return this; + } + TreeNode subTree = defaultTree.get(generalizedIndex); + return createFromTree(subTree); + } + + public void update(long generalizedIndex, TreeNode newNode, MutableBytes dest) { + update(generalizedIndex, nodeSsz(newNode), dest); + } + + private void update(long generalizedIndex, List nodeSsz, MutableBytes dest) { + Location leafPos = getNodeSszLocation(generalizedIndex); + int off = 0; + for (int i = 0; i < nodeSsz.size(); i++) { + Bytes newSszChunk = nodeSsz.get(i); + newSszChunk.copyTo(dest, leafPos.getOffset() + off); + off += newSszChunk.size(); + } + checkArgument(off == leafPos.getLength()); + } + + public Bytes32 calculateHashTreeRoot(Bytes ssz, int offset) { + return binaryTraverse( + SELF_G_INDEX, + defaultTree, + new BinaryVisitor<>() { + @Override + public Bytes32 visitLeaf(long gIndex, LeafNode node) { + Location location = gIdxToLoc.get(gIndex); + return Bytes32.rightPad(ssz.slice(offset + location.getOffset(), location.getLength())); + } + + @Override + public Bytes32 visitBranch( + long gIndex, TreeNode node, Bytes32 leftVisitResult, Bytes32 rightVisitResult) { + return Bytes32.wrap((new Crypto()).hashSHA2(Bytes.wrap(leftVisitResult, rightVisitResult).toArray())); +// return Hash.sha2_256(Bytes.wrap(leftVisitResult, rightVisitResult)); + } + }); + } + + private static T binaryTraverse(long gIndex, TreeNode node, BinaryVisitor visitor) { + if (node instanceof LeafNode) { + return visitor.visitLeaf(gIndex, (LeafNode) node); + } else if (node instanceof BranchNode) { + BranchNode branchNode = (BranchNode) node; + return visitor.visitBranch( + gIndex, + branchNode, + binaryTraverse(gIdxLeftGIndex(gIndex), branchNode.left(), visitor), + binaryTraverse(gIdxRightGIndex(gIndex), branchNode.right(), visitor)); + } else { + throw new IllegalArgumentException("Unexpected node type: " + node.getClass()); + } + } + + private interface BinaryVisitor { + + T visitLeaf(long gIndex, LeafNode node); + + T visitBranch(long gIndex, TreeNode node, T leftVisitResult, T rightVisitResult); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitive.java b/src/org/minima/system/network/base/ssz/SszPrimitive.java new file mode 100644 index 000000000..4111ac6e0 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitive.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; + +/** + * A wrapper class for SSZ primitive values. {@link SszPrimitive} classes has no mutable versions + */ +public interface SszPrimitive> + extends SszData { + + /** Returns wrapped primitive value */ + ValueType get(); + + /** {@link SszPrimitive} classes has no mutable versions */ + @Override + default SszMutableData createWritableCopy() { + throw new UnsupportedOperationException("Basic view instances are immutable"); + } + + @Override + default boolean isWritableSupported() { + return false; + } + + @Override + SszPrimitiveSchema getSchema(); +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveCollection.java b/src/org/minima/system/network/base/ssz/SszPrimitiveCollection.java new file mode 100644 index 000000000..b51d16616 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveCollection.java @@ -0,0 +1,54 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.AbstractList; +import java.util.List; +import java.util.stream.Stream; +// import tech.pegasys.teku.ssz.SszCollection; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszCollectionSchema; + +public interface SszPrimitiveCollection< + ElementT, SszElementT extends SszPrimitive> + extends SszCollection { + + default ElementT getElement(int index) { + return get(index).get(); + } + + @Override + SszCollectionSchema getSchema(); + + @Override + SszMutablePrimitiveCollection createWritableCopy(); + + default List asListUnboxed() { + return new AbstractList<>() { + @Override + public ElementT get(int index) { + return SszPrimitiveCollection.this.getElement(index); + } + + @Override + public int size() { + return SszPrimitiveCollection.this.size(); + } + }; + } + + default Stream streamUnboxed() { + return asListUnboxed().stream(); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveCollectionSchema.java b/src/org/minima/system/network/base/ssz/SszPrimitiveCollectionSchema.java new file mode 100644 index 000000000..75eb88d21 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveCollectionSchema.java @@ -0,0 +1,50 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.collections.SszPrimitiveCollection; +// import tech.pegasys.teku.ssz.schema.SszCollectionSchema; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; + +public interface SszPrimitiveCollectionSchema< + ElementT, + SszElementT extends SszPrimitive, + SszCollectionT extends SszPrimitiveCollection> + extends SszCollectionSchema { + + @SuppressWarnings("unchecked") + default SszCollectionT of(ElementT... rawElements) { + return of(Arrays.asList(rawElements)); + } + + default SszCollectionT of(List rawElements) { + SszPrimitiveSchema elementSchema = getPrimitiveElementSchema(); + return createFromElements( + rawElements.stream().map(elementSchema::boxed).collect(Collectors.toList())); + } + + @SuppressWarnings("unchecked") + default SszPrimitiveSchema getPrimitiveElementSchema() { + return (SszPrimitiveSchema) getElementSchema(); + } + + default Collector collectorUnboxed() { + return Collectors.collectingAndThen(Collectors.toList(), this::of); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveSchema.java b/src/org/minima/system/network/base/ssz/SszPrimitiveSchema.java new file mode 100644 index 000000000..b6ef1bfee --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveSchema.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.SszPrimitive; + +public interface SszPrimitiveSchema> + extends SszSchema { + + int getBitsSize(); + + SszDataT boxed(DataT rawValue); + + @Override + default boolean isPrimitive() { + return true; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveSchemas.java b/src/org/minima/system/network/base/ssz/SszPrimitiveSchemas.java new file mode 100644 index 000000000..c12bedf1c --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveSchemas.java @@ -0,0 +1,237 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.nio.ByteOrder; +import java.util.Arrays; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; +// import tech.pegasys.teku.infrastructure.unsigned.UInt64; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.primitive.SszBit; +// import tech.pegasys.teku.ssz.primitive.SszByte; +// import tech.pegasys.teku.ssz.primitive.SszBytes32; +// import tech.pegasys.teku.ssz.primitive.SszBytes4; +// import tech.pegasys.teku.ssz.primitive.SszUInt64; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszPrimitiveSchema; +// import tech.pegasys.teku.ssz.tree.LeafDataNode; +// import tech.pegasys.teku.ssz.tree.LeafNode; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.type.Bytes4; + +/** The collection of commonly used basic types */ +public interface SszPrimitiveSchemas { + AbstractSszPrimitiveSchema BIT_SCHEMA = + new AbstractSszPrimitiveSchema<>(1) { + @Override + public SszBit createFromLeafBackingNode(LeafDataNode node, int idx) { + return SszBit.of((node.getData().get(idx / 8) & (1 << (idx % 8))) != 0); + } + + @Override + public TreeNode updateBackingNode(TreeNode srcNode, int idx, SszData newValue) { + int byteIndex = idx / 8; + int bitIndex = idx % 8; + Bytes originalBytes = ((LeafNode) srcNode).getData(); + byte b = byteIndex < originalBytes.size() ? originalBytes.get(byteIndex) : 0; + boolean bit = ((SszBit) newValue).get(); + if (bit) { + b = (byte) (b | (1 << bitIndex)); + } else { + b = (byte) (b & ~(1 << bitIndex)); + } + Bytes newBytes = updateExtending(originalBytes, byteIndex, Bytes.of(b)); + return LeafNode.create(newBytes); + } + + @Override + public SszBit boxed(Boolean rawValue) { + return SszBit.of(rawValue); + } + + @Override + public TreeNode getDefaultTree() { + return LeafNode.ZERO_LEAVES[1]; + } + + @Override + public String toString() { + return "Bit"; + } + }; + + AbstractSszPrimitiveSchema BYTE_SCHEMA = + new AbstractSszPrimitiveSchema<>(8) { + @Override + public SszByte createFromLeafBackingNode(LeafDataNode node, int internalIndex) { + return SszByte.of(node.getData().get(internalIndex)); + } + + @Override + public TreeNode updateBackingNode(TreeNode srcNode, int index, SszData newValue) { + byte aByte = ((SszByte) newValue).get(); + Bytes curVal = ((LeafNode) srcNode).getData(); + Bytes newBytes = updateExtending(curVal, index, Bytes.of(aByte)); + return LeafNode.create(newBytes); + } + + @Override + public SszByte boxed(Byte rawValue) { + return SszByte.of(rawValue); + } + + @Override + public TreeNode getDefaultTree() { + return LeafNode.ZERO_LEAVES[1]; + } + + @Override + public String toString() { + return "Byte"; + } + }; + + AbstractSszPrimitiveSchema UINT64_SCHEMA = + new AbstractSszPrimitiveSchema<>(64) { + @Override + public SszUInt64 createFromLeafBackingNode(LeafDataNode node, int internalIndex) { + Bytes leafNodeBytes = node.getData(); + try { + Bytes elementBytes = leafNodeBytes.slice(internalIndex * 8, 8); + return SszUInt64.of(UInt64.fromLongBits(elementBytes.toLong(ByteOrder.LITTLE_ENDIAN))); + } catch (Exception e) { + // additional info to track down the bug https://github.com/PegaSysEng/teku/issues/2579 + String info = + "Refer to https://github.com/PegaSysEng/teku/issues/2579 if see this exception. "; + info += "internalIndex = " + internalIndex; + info += ", leafNodeBytes: " + leafNodeBytes.getClass().getSimpleName(); + try { + info += ", leafNodeBytes = " + leafNodeBytes.copy(); + } catch (Exception ex) { + info += "(" + ex + ")"; + } + try { + info += ", leafNodeBytes[] = " + Arrays.toString(leafNodeBytes.toArray()); + } catch (Exception ex) { + info += "(" + ex + ")"; + } + throw new RuntimeException(info, e); + } + } + + @Override + public TreeNode updateBackingNode(TreeNode srcNode, int index, SszData newValue) { + Bytes uintBytes = + Bytes.ofUnsignedLong(((SszUInt64) newValue).longValue(), ByteOrder.LITTLE_ENDIAN); + Bytes curVal = ((LeafNode) srcNode).getData(); + Bytes newBytes = updateExtending(curVal, index * 8, uintBytes); + return LeafNode.create(newBytes); + } + + @Override + public SszUInt64 boxed(UInt64 rawValue) { + return SszUInt64.of(rawValue); + } + + @Override + public TreeNode getDefaultTree() { + return LeafNode.ZERO_LEAVES[8]; + } + + @Override + public String toString() { + return "UInt64"; + } + }; + + AbstractSszPrimitiveSchema BYTES4_SCHEMA = + new AbstractSszPrimitiveSchema<>(32) { + @Override + public SszBytes4 createFromLeafBackingNode(LeafDataNode node, int internalIndex) { + return SszBytes4.of(new Bytes4(node.getData().slice(internalIndex * 4, 4))); + } + + @Override + public TreeNode updateBackingNode(TreeNode srcNode, int internalIndex, SszData newValue) { + checkArgument( + internalIndex >= 0 && internalIndex < 8, "Invalid internal index: %s", internalIndex); + Bytes bytes = ((SszBytes4) newValue).get().getWrappedBytes(); + Bytes curVal = ((LeafNode) srcNode).getData(); + Bytes newBytes = updateExtending(curVal, internalIndex * 4, bytes); + return LeafNode.create(newBytes); + } + + @Override + public SszBytes4 boxed(Bytes4 rawValue) { + return SszBytes4.of(rawValue); + } + + @Override + public TreeNode getDefaultTree() { + return LeafNode.ZERO_LEAVES[4]; + } + + @Override + public String toString() { + return "Bytes4"; + } + }; + + AbstractSszPrimitiveSchema BYTES32_SCHEMA = + new AbstractSszPrimitiveSchema<>(256) { + @Override + public SszBytes32 createFromLeafBackingNode(LeafDataNode node, int internalIndex) { + return SszBytes32.of(node.hashTreeRoot()); + } + + @Override + public TreeNode updateBackingNode(TreeNode srcNode, int internalIndex, SszData newValue) { + return LeafNode.create(((SszBytes32) newValue).get()); + } + + @Override + public SszBytes32 boxed(Bytes32 rawValue) { + return SszBytes32.of(rawValue); + } + + @Override + public TreeNode getDefaultTree() { + return LeafNode.ZERO_LEAVES[32]; + } + + @Override + public String toString() { + return "Bytes32"; + } + }; + + private static Bytes updateExtending(Bytes origBytes, int origOff, Bytes newBytes) { + if (origOff == origBytes.size()) { + return Bytes.wrap(origBytes, newBytes); + } else { + final MutableBytes dest; + if (origOff + newBytes.size() > origBytes.size()) { + dest = MutableBytes.create(origOff + newBytes.size()); + origBytes.copyTo(dest, 0); + } else { + dest = origBytes.mutableCopy(); + } + newBytes.copyTo(dest, origOff); + return dest; + } + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveVector.java b/src/org/minima/system/network/base/ssz/SszPrimitiveVector.java new file mode 100644 index 000000000..b8ce45fcd --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveVector.java @@ -0,0 +1,25 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.SszVector; + +public interface SszPrimitiveVector< + ElementT, SszElementT extends SszPrimitive> + extends SszPrimitiveCollection, SszVector { + + @Override + SszMutablePrimitiveVector createWritableCopy(); +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveVectorImpl.java b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorImpl.java new file mode 100644 index 000000000..7662b3500 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Supplier; +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.collections.SszMutablePrimitiveVector; +// import tech.pegasys.teku.ssz.collections.SszPrimitiveVector; +// import tech.pegasys.teku.ssz.impl.SszVectorImpl; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszPrimitiveVectorImpl< + ElementT, SszElementT extends SszPrimitive> + extends SszVectorImpl implements SszPrimitiveVector { + + public SszPrimitiveVectorImpl(SszCompositeSchema schema, Supplier lazyBackingNode) { + super(schema, lazyBackingNode); + } + + public SszPrimitiveVectorImpl(SszCompositeSchema schema, TreeNode backingNode) { + super(schema, backingNode); + } + + @Override + public SszMutablePrimitiveVector createWritableCopy() { + return new SszMutablePrimitiveVectorImpl<>(this); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchema.java b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchema.java new file mode 100644 index 000000000..b651177ef --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchema.java @@ -0,0 +1,53 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.collections.SszPrimitiveVector; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints; +// import tech.pegasys.teku.ssz.schema.SszVectorSchema; +// import tech.pegasys.teku.ssz.schema.collections.impl.SszPrimitiveVectorSchemaImpl; + +public interface SszPrimitiveVectorSchema< + ElementT, + SszElementT extends SszPrimitive, + SszVectorT extends SszPrimitiveVector> + extends SszPrimitiveCollectionSchema, + SszVectorSchema { + + static > + SszPrimitiveVectorSchema create( + SszPrimitiveSchema elementSchema, int length) { + return create(elementSchema, length, SszSchemaHints.none()); + } + + @SuppressWarnings("unchecked") + static > + SszPrimitiveVectorSchema create( + SszPrimitiveSchema elementSchema, long length, SszSchemaHints hints) { + if (elementSchema == SszPrimitiveSchemas.BIT_SCHEMA) { + return (SszPrimitiveVectorSchema) SszBitvectorSchema.create(length); + } else if (elementSchema == SszPrimitiveSchemas.BYTE_SCHEMA) { + return (SszPrimitiveVectorSchema) + SszByteVectorSchema.create((int) length); + } else if (elementSchema == SszPrimitiveSchemas.BYTES32_SCHEMA) { + return (SszPrimitiveVectorSchema) + SszBytes32VectorSchema.create((int) length); + } else { + return new SszPrimitiveVectorSchemaImpl<>(elementSchema, length); + } + } +} diff --git a/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchemaImpl.java b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchemaImpl.java new file mode 100644 index 000000000..4876ba8bb --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszPrimitiveVectorSchemaImpl.java @@ -0,0 +1,41 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszPrimitive; +// import tech.pegasys.teku.ssz.collections.SszPrimitiveVector; +// import tech.pegasys.teku.ssz.collections.impl.SszPrimitiveVectorImpl; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchema; +// import tech.pegasys.teku.ssz.schema.collections.SszPrimitiveVectorSchema; +// import tech.pegasys.teku.ssz.schema.impl.AbstractSszVectorSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszPrimitiveVectorSchemaImpl< + ElementT, + SszElementT extends SszPrimitive, + SszVectorT extends SszPrimitiveVector> + extends AbstractSszVectorSchema + implements SszPrimitiveVectorSchema { + + public SszPrimitiveVectorSchemaImpl( + SszPrimitiveSchema elementSchema, long vectorLength) { + super(elementSchema, vectorLength); + } + + @Override + @SuppressWarnings("unchecked") + public SszVectorT createFromBackingNode(TreeNode node) { + return (SszVectorT) new SszPrimitiveVectorImpl(this, node); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszReader.java b/src/org/minima/system/network/base/ssz/SszReader.java new file mode 100644 index 000000000..4ba5037c4 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszReader.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.io.Closeable; +import org.apache.tuweni.bytes.Bytes; + +/** Simple reader interface for SSZ stream */ +public interface SszReader extends Closeable { + + /** Creates an instance from {@link Bytes} */ + static SszReader fromBytes(Bytes bytes) { + return new SimpleSszReader(bytes); + } + + /** Number of bytes available for reading */ + int getAvailableBytes(); + + /** + * Returns {@link SszReader} instance limited with {@code size} bytes Advances this reader current + * read position to {@code size} bytes + * + * @throws SszDeserializeException If not enough bytes available for slice + */ + SszReader slice(int size) throws SszDeserializeException; + + /** + * Returns {@code length} bytes and advances this reader current read position to {@code length} + * bytes + * + * @throws SszDeserializeException If not enough bytes available + */ + Bytes read(int length) throws SszDeserializeException; + + /** + * Closes this reader. Doesn't affect the 'parent' reader (if any) this instance was 'sliced' from + * + * @throws SszDeserializeException If unread bytes remain + */ + @Override + void close() throws SszDeserializeException; +} diff --git a/src/org/minima/system/network/base/ssz/SszSchema.java b/src/org/minima/system/network/base/ssz/SszSchema.java new file mode 100644 index 000000000..744e7fd57 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszSchema.java @@ -0,0 +1,92 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** + * Base class for any SSZ structure schema like Vector, List, Container, primitive types + * (https://github.com/ethereum/eth2.0-specs/blob/dev/ssz/simple-serialize.md#typing) + */ +public interface SszSchema extends SszType { + + @SuppressWarnings("unchecked") + static SszSchema as(final Class clazz, final SszSchema schema) { + return (SszSchema) schema; + } + + /** + * Creates a default backing binary tree for this schema + * + *

E.g. if the schema is primitive then normally just a single leaf node is created + * + *

E.g. if the schema is a complex structure with multi-level nested vectors and containers + * then the complete tree including all descendant members subtrees is created + */ + TreeNode getDefaultTree(); + + /** + * Creates immutable ssz structure over the tree which should correspond to this schema. + * + *

Note: if the tree structure doesn't correspond this schema that fact could only be detected + * later during access to structure members + */ + SszDataT createFromBackingNode(TreeNode node); + + /** Returns the default immutable structure of this scheme */ + default SszDataT getDefault() { + return createFromBackingNode(getDefaultTree()); + } + + boolean isPrimitive(); + + /** + * For packed primitive values. Extracts a packed value from the tree node by its 'internal + * index'. For example in `Bitvector(512)` the bit value at index `300` is stored at the second + * leaf node and it's 'internal index' in this node would be `45` + */ + default SszDataT createFromBackingNode(TreeNode node, int internalIndex) { + return createFromBackingNode(node); + } + + /** + * For packed primitive values. Packs the value to the existing node at 'internal index' For + * example in `Bitvector(512)` the bit value at index `300` is stored at the second leaf node and + * it's 'internal index' in this node would be `45` + */ + default TreeNode updateBackingNode(TreeNode srcNode, int internalIndex, SszData newValue) { + return newValue.getBackingNode(); + } + + default Bytes sszSerialize(SszDataT view) { + return sszSerializeTree(view.getBackingNode()); + } + + default int sszSerialize(SszDataT view, SszWriter writer) { + return sszSerializeTree(view.getBackingNode(), writer); + } + + default SszDataT sszDeserialize(SszReader reader) throws SszDeserializeException { + return createFromBackingNode(sszDeserializeTree(reader)); + } + + default SszDataT sszDeserialize(Bytes ssz) throws SszDeserializeException { + return sszDeserialize(SszReader.fromBytes(ssz)); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszSchemaHints.java b/src/org/minima/system/network/base/ssz/SszSchemaHints.java new file mode 100644 index 000000000..f1cc2a7e4 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszSchemaHints.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +//import tech.pegasys.teku.ssz.tree.SszSuperNode; + +/** + * A set of hints for {@link SszSchema} classes on strategies to use for optimizing memory or/and + * performance. + */ +public class SszSchemaHints { + + private static class SszSchemaHint {} + + /** + * Hint to use {@link SszSuperNode} for lists/vectors to save the memory when the list content is + * expected to be rarely updated + * + *

The depth parameter specifies the maximum number (2 ^ depth) of + * list/vector elements a single node can contain. Increasing this parameter saves memory but + * makes list/vector update and hashTreeRoot recalculation more CPU expensive + */ + public static final class SszSuperNodeHint extends SszSchemaHint { + private final int depth; + + public SszSuperNodeHint(int depth) { + this.depth = depth; + } + + public int getDepth() { + return depth; + } + } + + public static SszSchemaHints of(SszSchemaHint... hints) { + return new SszSchemaHints(Arrays.asList(hints)); + } + + public static SszSchemaHints none() { + return of(); + } + + public static SszSchemaHints sszSuperNode(int superNodeDepth) { + return of(new SszSuperNodeHint(superNodeDepth)); + } + + private final List hints; + + private SszSchemaHints(List hints) { + this.hints = hints; + } + + @SuppressWarnings("unchecked") + public Optional getHint(Class hintClass) { + return (Optional) hints.stream().filter(h -> h.getClass() == hintClass).findFirst(); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszSuperNode.java b/src/org/minima/system/network/base/ssz/SszSuperNode.java new file mode 100644 index 000000000..994f76e5e --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszSuperNode.java @@ -0,0 +1,176 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.apache.tuweni.bytes.MutableBytes; +//import org.apache.tuweni.crypto.Hash; +import org.jetbrains.annotations.NotNull; +//import tech.pegasys.teku.ssz.tree.GIndexUtil.NodeRelation; +import org.minima.system.network.base.ssz.SszNodeTemplate.Location; +import org.minima.system.network.base.ssz.GIndexUtil.NodeRelation; +import org.minima.utils.Crypto; + +/** + * Stores consecutive elements of the same fixed size type as a single packed bytes of their leaves + * (this representation exactly matches SSZ representation of elements sequence). + * + *

This node represents a subtree of binary merkle tree for sequence (list or vector) of elements + * with maximum length of 2 ^ depth. If the sequence has less than maximum elements + * then ssz bytes store only existing elements (what again matches SSZ representation + * of a list) + * + *

To address individual nodes inside elements and resolve their internal generalized indexes the + * node uses {@link SszNodeTemplate} which represents element type tree structure + * + *

This node favors memory efficiency over update performance and thus is the best choice for + * rarely updated and space consuming structures (e.g. Eth2 BeaconState.validators + * list) + */ +public class SszSuperNode implements TreeNode, LeafDataNode { + private static final TreeNode DEFAULT_NODE = LeafNode.EMPTY_LEAF; + + private final int depth; + private final SszNodeTemplate elementTemplate; + private final Bytes ssz; + private final Supplier hashTreeRoot = Suppliers.memoize(this::calcHashTreeRoot); + + public SszSuperNode(int depth, SszNodeTemplate elementTemplate, Bytes ssz) { + this.depth = depth; + this.elementTemplate = elementTemplate; + this.ssz = ssz; + checkArgument(ssz.size() % elementTemplate.getSszLength() == 0); + checkArgument(getElementsCount() <= getMaxElements()); + } + + private int getMaxElements() { + return 1 << depth; + } + + private int getElementsCount() { + return ssz.size() / elementTemplate.getSszLength(); + } + + @Override + public Bytes32 hashTreeRoot() { + return hashTreeRoot.get(); + } + + private Bytes32 calcHashTreeRoot() { + return hashTreeRoot(0, 0); + } + + private Bytes32 hashTreeRoot(int curDepth, int offset) { + if (curDepth == depth) { + if (offset < ssz.size()) { + return elementTemplate.calculateHashTreeRoot(ssz, offset); + } else { + assert offset <= elementTemplate.getSszLength() * (getMaxElements() - 1); + return DEFAULT_NODE.hashTreeRoot(); + } + } else { + return Bytes32.wrap((new Crypto()).hashSHA2(Bytes.wrap( + hashTreeRoot(curDepth + 1, offset), + hashTreeRoot( + curDepth + 1, + offset + elementTemplate.getSszLength() * (1 << ((depth - curDepth) - 1)))).toArray())); + // return Hash.sha2_256( + // Bytes.wrap( + // hashTreeRoot(curDepth + 1, offset), + // hashTreeRoot( + // curDepth + 1, + // offset + elementTemplate.getSszLength() * (1 << ((depth - curDepth) - 1))))); + } + } + + @NotNull + @Override + public TreeNode get(long generalizedIndex) { + if (GIndexUtil.gIdxIsSelf(generalizedIndex)) { + return this; + } + int childIndex = GIndexUtil.gIdxGetChildIndex(generalizedIndex, depth); + int childOffset = childIndex * elementTemplate.getSszLength(); + checkArgument(childOffset < ssz.size(), "Invalid index"); + long relativeGIndex = GIndexUtil.gIdxGetRelativeGIndex(generalizedIndex, depth); + Location nodeLoc = elementTemplate.getNodeSszLocation(relativeGIndex); + if (nodeLoc.isLeaf()) { + return LeafNode.create(ssz.slice(childOffset + nodeLoc.getOffset(), nodeLoc.getLength())); + } else if (GIndexUtil.gIdxIsSelf(relativeGIndex)) { + return new SszSuperNode( + 0, elementTemplate, ssz.slice(childOffset, elementTemplate.getSszLength())); + } else { + SszNodeTemplate subTemplate = elementTemplate.getSubTemplate(relativeGIndex); + return new SszSuperNode( + 0, subTemplate, ssz.slice(childOffset + nodeLoc.getOffset(), nodeLoc.getLength())); + } + } + + @Override + public boolean iterate( + long thisGeneralizedIndex, long startGeneralizedIndex, TreeVisitor visitor) { + if (GIndexUtil.gIdxCompare(thisGeneralizedIndex, startGeneralizedIndex) == NodeRelation.Left) { + return true; + } else { + return visitor.visit(this, thisGeneralizedIndex); + } + } + + @Override + public TreeNode updated(TreeUpdates newNodes) { + if (newNodes.isEmpty()) { + return this; + } + long leftmostUpdateIndex = newNodes.getRelativeGIndex(newNodes.size() - 1); + int leftmostChildIndex = GIndexUtil.gIdxGetChildIndex(leftmostUpdateIndex, depth); + int newSszSize = (leftmostChildIndex + 1) * elementTemplate.getSszLength(); + Bytes updatedSizeSsz = + newSszSize <= ssz.size() + ? ssz + : Bytes.wrap(ssz, Bytes.wrap(new byte[newSszSize - ssz.size()])); + MutableBytes mutableCopy = updatedSizeSsz.mutableCopy(); + for (int i = 0; i < newNodes.size(); i++) { + long updateGIndex = newNodes.getRelativeGIndex(i); + int childIndex = GIndexUtil.gIdxGetChildIndex(updateGIndex, depth); + long childGIndex = GIndexUtil.gIdxGetRelativeGIndex(updateGIndex, depth); + int childOffset = childIndex * elementTemplate.getSszLength(); + MutableBytes childMutableSlice = + mutableCopy.mutableSlice(childOffset, elementTemplate.getSszLength()); + elementTemplate.update(childGIndex, newNodes.getNode(i), childMutableSlice); + } + return new SszSuperNode(depth, elementTemplate, mutableCopy); + } + + @Override + public Bytes getData() { + return ssz; + } + + @Override + public String toString() { + int sszLength = elementTemplate.getSszLength(); + return "SszSuperNode{" + + IntStream.range(0, getElementsCount()) + .mapToObj(i -> ssz.slice(i * sszLength, sszLength).toString()) + .collect(Collectors.joining(", ")) + + "}"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszType.java b/src/org/minima/system/network/base/ssz/SszType.java new file mode 100644 index 000000000..1810db0e8 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszType.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.nio.ByteOrder; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.ssz.sos.SszByteArrayWriter; +// import tech.pegasys.teku.ssz.sos.SszDeserializeException; +// import tech.pegasys.teku.ssz.sos.SszLengthBounds; +// import tech.pegasys.teku.ssz.sos.SszReader; +// import tech.pegasys.teku.ssz.sos.SszWriter; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +/** Base class of {@link SszSchema} with SSZ serialization related methods */ +public interface SszType { + + // the size of SSZ UIn32 lengths and offsets + int SSZ_LENGTH_SIZE = 4; + + // serializes int length to SSZ 4 bytes + static Bytes sszLengthToBytes(int length) { + return Bytes.ofUnsignedInt(length, ByteOrder.LITTLE_ENDIAN); + } + + // deserializes int length from SSZ 4 bytes + static int sszBytesToLength(Bytes bytes) { + if (!bytes.slice(SSZ_LENGTH_SIZE).isZero()) { + throw new SszDeserializeException("Invalid length bytes: " + bytes); + } + int ret = bytes.slice(0, SSZ_LENGTH_SIZE).toInt(ByteOrder.LITTLE_ENDIAN); + if (ret < 0) { + throw new SszDeserializeException("Invalid length: " + ret); + } + return ret; + } + + /** Indicates whether the type is fixed or variable size */ + boolean isFixedSize(); + + /** Returns the size of the fixed SSZ part for this type */ + int getSszFixedPartSize(); + + /** Returns the size of the variable SSZ part for this type and specified backing subtree */ + int getSszVariablePartSize(TreeNode node); + + /** Calculates the full SSZ size in bytes for this type and specified backing subtree */ + default int getSszSize(TreeNode node) { + return getSszFixedPartSize() + getSszVariablePartSize(node); + } + + /** SSZ serializes the backing tree instance of this type */ + default Bytes sszSerializeTree(TreeNode node) { + SszByteArrayWriter writer = new SszByteArrayWriter(getSszSize(node)); + sszSerializeTree(node, writer); + return writer.toBytes(); + } + + /** + * SSZ serializes the backing tree of this type and returns the data as bytes 'stream' via passed + * {@code writer} + */ + int sszSerializeTree(TreeNode node, SszWriter writer); + + TreeNode sszDeserializeTree(SszReader reader) throws SszDeserializeException; + + SszLengthBounds getSszLengthBounds(); +} diff --git a/src/org/minima/system/network/base/ssz/SszUInt64.java b/src/org/minima/system/network/base/ssz/SszUInt64.java new file mode 100644 index 000000000..c93cfa994 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszUInt64.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.infrastructure.unsigned.UInt64; +// import tech.pegasys.teku.ssz.impl.AbstractSszPrimitive; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; + +public class SszUInt64 extends AbstractSszPrimitive { + + public static SszUInt64 of(UInt64 val) { + return new SszUInt64(val); + } + + private SszUInt64(UInt64 val) { + super(val, SszPrimitiveSchemas.UINT64_SCHEMA); + } + + public long longValue() { + return get().longValue(); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszVector.java b/src/org/minima/system/network/base/ssz/SszVector.java new file mode 100644 index 000000000..3733b47b6 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszVector.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.schema.SszVectorSchema; + +/** + * Immutable SSZ Vector + * + * @param Type of vector elements + */ +public interface SszVector extends SszCollection { + + @Override + SszMutableVector createWritableCopy(); + + @Override + SszVectorSchema getSchema(); +} diff --git a/src/org/minima/system/network/base/ssz/SszVectorImpl.java b/src/org/minima/system/network/base/ssz/SszVectorImpl.java new file mode 100644 index 000000000..9af24f897 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszVectorImpl.java @@ -0,0 +1,76 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import java.util.function.Supplier; +import java.util.stream.Collectors; +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszMutableVector; +// import tech.pegasys.teku.ssz.SszVector; +// import tech.pegasys.teku.ssz.cache.ArrayIntCache; +// import tech.pegasys.teku.ssz.cache.IntCache; +// import tech.pegasys.teku.ssz.schema.SszCompositeSchema; +// import tech.pegasys.teku.ssz.schema.SszVectorSchema; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszVectorImpl extends AbstractSszCollection + implements SszVector { + + public SszVectorImpl(SszCompositeSchema schema, Supplier lazyBackingNode) { + super(schema, lazyBackingNode); + } + + public SszVectorImpl(SszCompositeSchema schema, TreeNode backingNode) { + super(schema, backingNode); + } + + public SszVectorImpl( + SszCompositeSchema schema, TreeNode backingNode, IntCache cache) { + super(schema, backingNode, cache); + } + + @Override + protected int sizeImpl() { + return (int) Long.min(Integer.MAX_VALUE, this.getSchema().getMaxLength()); + } + + @Override + public SszMutableVector createWritableCopy() { + return new SszMutableVectorImpl<>(this); + } + + @SuppressWarnings("unchecked") + @Override + public SszVectorSchema getSchema() { + return (SszVectorSchema) super.getSchema(); + } + + @Override + protected IntCache createCache() { + return size() > 16384 ? new ArrayIntCache<>() : new ArrayIntCache<>(size()); + } + + @Override + protected void checkIndex(int index) { + if (index < 0 || index >= size()) { + throw new IndexOutOfBoundsException( + "Invalid index " + index + " for vector with size " + size()); + } + } + + @Override + public String toString() { + return "SszVector{" + stream().map(Object::toString).collect(Collectors.joining(", ")) + "}"; + } +} diff --git a/src/org/minima/system/network/base/ssz/SszVectorSchema.java b/src/org/minima/system/network/base/ssz/SszVectorSchema.java new file mode 100644 index 000000000..792dba861 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszVectorSchema.java @@ -0,0 +1,54 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszVector; +// import tech.pegasys.teku.ssz.schema.collections.SszPrimitiveVectorSchema; +// import tech.pegasys.teku.ssz.schema.impl.SszVectorSchemaImpl; +// import tech.pegasys.teku.ssz.tree.GIndexUtil; + +public interface SszVectorSchema< + ElementDataT extends SszData, SszVectorT extends SszVector> + extends SszCollectionSchema { + + long MAX_VECTOR_LENGTH = 1L << (GIndexUtil.MAX_DEPTH - 1); + + default int getLength() { + long maxLength = getMaxLength(); + if (maxLength > Integer.MAX_VALUE) { + throw new IllegalArgumentException("Vector size too large: " + maxLength); + } + return (int) maxLength; + } + + static SszVectorSchema create( + SszSchema elementSchema, long length) { + return create(elementSchema, length, SszSchemaHints.none()); + } + + @SuppressWarnings("unchecked") + static SszVectorSchema create( + SszSchema elementSchema, long length, SszSchemaHints hints) { + checkArgument(length >= 0 && length <= MAX_VECTOR_LENGTH); + if (elementSchema instanceof SszPrimitiveSchema) { + return (SszVectorSchema) + SszPrimitiveVectorSchema.create((SszPrimitiveSchema) elementSchema, length, hints); + } else { + return new SszVectorSchemaImpl<>(elementSchema, length, false, hints); + } + } +} diff --git a/src/org/minima/system/network/base/ssz/SszVectorSchemaImpl.java b/src/org/minima/system/network/base/ssz/SszVectorSchemaImpl.java new file mode 100644 index 000000000..e42d53c94 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszVectorSchemaImpl.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +// import tech.pegasys.teku.ssz.SszData; +// import tech.pegasys.teku.ssz.SszVector; +// import tech.pegasys.teku.ssz.impl.SszVectorImpl; +// import tech.pegasys.teku.ssz.schema.SszSchema; +// import tech.pegasys.teku.ssz.schema.SszSchemaHints; +// import tech.pegasys.teku.ssz.tree.TreeNode; + +public class SszVectorSchemaImpl + extends AbstractSszVectorSchema> { + + SszVectorSchemaImpl(SszSchema elementType, long vectorLength) { + this(elementType, vectorLength, false, SszSchemaHints.none()); + } + + public SszVectorSchemaImpl( + SszSchema elementSchema, + long vectorLength, + boolean isListBacking, + SszSchemaHints hints) { + super(elementSchema, vectorLength, isListBacking, hints); + } + + @Override + public SszVector createFromBackingNode(TreeNode node) { + return new SszVectorImpl<>(this, node); + } +} diff --git a/src/org/minima/system/network/base/ssz/SszWriter.java b/src/org/minima/system/network/base/ssz/SszWriter.java new file mode 100644 index 000000000..bae2a610a --- /dev/null +++ b/src/org/minima/system/network/base/ssz/SszWriter.java @@ -0,0 +1,29 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import org.apache.tuweni.bytes.Bytes; + +public interface SszWriter { + + default void write(Bytes bytes) { + write(bytes.toArrayUnsafe()); + } + + default void write(byte[] bytes) { + write(bytes, 0, bytes.length); + } + + void write(byte[] bytes, int offset, int length); +} diff --git a/src/org/minima/system/network/base/ssz/TillIndexVisitor.java b/src/org/minima/system/network/base/ssz/TillIndexVisitor.java new file mode 100644 index 000000000..6ccce3f33 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TillIndexVisitor.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +//import tech.pegasys.teku.ssz.tree.GIndexUtil.NodeRelation; +import org.minima.system.network.base.ssz.GIndexUtil.NodeRelation; + +class TillIndexVisitor implements TreeVisitor { + + static TreeVisitor create(TreeVisitor delegate, long tillGeneralizedIndex) { + return new TillIndexVisitor(delegate, tillGeneralizedIndex, true); + } + + private final TreeVisitor delegate; + private final long tillGIndex; + private final boolean inclusive; + + public TillIndexVisitor(TreeVisitor delegate, long tillGIndex, boolean inclusive) { + this.delegate = delegate; + this.tillGIndex = tillGIndex; + this.inclusive = inclusive; + } + + @Override + public boolean visit(TreeNode node, long generalizedIndex) { + NodeRelation compareRes = GIndexUtil.gIdxCompare(generalizedIndex, tillGIndex); + if (inclusive && compareRes == NodeRelation.Right) { + return false; + } else if (!inclusive && (compareRes == NodeRelation.Same)) { + return false; + } else { + return delegate.visit(node, generalizedIndex); + } + } +} diff --git a/src/org/minima/system/network/base/ssz/TreeNode.java b/src/org/minima/system/network/base/ssz/TreeNode.java new file mode 100644 index 000000000..cf70290c8 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TreeNode.java @@ -0,0 +1,157 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static java.util.Collections.singletonList; + +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.tuweni.bytes.Bytes32; +import org.jetbrains.annotations.NotNull; + +/** + * Basic interface for Backing Tree node Backing Binary Tree concept for SSZ structures is described + * here: https://github.com/protolambda/eth-merkle-trees/blob/master/typing_partials.md#tree + * + *

Tree node is immutable by design. Any update on a tree creates new nodes which refer both new + * data nodes and old unmodified nodes + */ +public interface TreeNode { + + /** + * Calculates (if necessary) and returns `hash_tree_root` of this tree node. Worth to mention that + * `hash_tree_root` of a {@link LeafNode} is the node {@link Bytes32} content + */ + Bytes32 hashTreeRoot(); + + /** + * Gets this node descendant by its 'generalized index' + * + * @param generalizedIndex generalized index of a tree is specified here: + * https://github.com/ethereum/eth2.0-specs/blob/2787fea5feb8d5977ebee7c578c5d835cff6dc21/specs/light_client/merkle_proofs.md#generalized-merkle-tree-index + * @return node descendant + * @throws IllegalArgumentException if no node exists for the passed generalized index + */ + @NotNull + TreeNode get(long generalizedIndex); + + /** + * Iterates recursively this node children (including the node itself) in the order Self -> Left + * subtree -> Right subtree + * + *

This method can be considered low-level and mostly intended as a single implementation point + * for subclasses. Consider using higher-level methods {@link #iterateRange(long, long, + * TreeVisitor)}, {@link #iterateAll(TreeVisitor)} and {@link #iterateAll(Consumer)} + * + * @param thisGeneralizedIndex the generalized index of this node or {@link + * GIndexUtil#SELF_G_INDEX} if this node is considered the root. {@link + * TreeVisitor#visit(TreeNode, long)} index will be calculated with respect to this parameter + * @param startGeneralizedIndex The generalized index to start iteration from. All tree + * predecessor and successor nodes of a node at this index will be visited. All nodes 'to the + * left' of start node are to be skipped. The index may point to a non-existing node, in this + * case the nearest existing predecessor node would be the starting node To start iteration + * from the leftmost node use {@link GIndexUtil#LEFTMOST_G_INDEX} + * @param visitor Callback for nodes. When visitor returns false, iteration breaks + * @return true if the iteration should proceed or false to break iteration + */ + boolean iterate(long thisGeneralizedIndex, long startGeneralizedIndex, TreeVisitor visitor); + + /** + * Iterates all nodes between and including startGeneralizedIndex and endGeneralizedIndexInclusive + * in order Self -> Left subtree -> Right subtree + * + *

All tree predecessor and successor nodes of startGeneralizedIndex and + * endGeneralizedIndexInclusive nodes will be visited. All nodes 'to the left' of the start node + * and 'to the right' of the end node are to be skipped. An index may point to a non-existing + * node, in this case the nearest existing predecessor node would be considered the starting node. + * + *

To start iteration from the leftmost node specify startGeneralizedIndex equal to {@link + * GIndexUtil#LEFTMOST_G_INDEX} To iteration till the rightmost node specify + * endGeneralizedIndexInclusive equal to {@link GIndexUtil#RIGHTMOST_G_INDEX} + */ + default void iterateRange( + long startGeneralizedIndex, long endGeneralizedIndexInclusive, TreeVisitor visitor) { + iterate( + GIndexUtil.SELF_G_INDEX, + startGeneralizedIndex, + TillIndexVisitor.create(visitor, endGeneralizedIndexInclusive)); + } + + /** Iterates all tree nodes in the order Self -> Left subtree -> Right subtree */ + default void iterateAll(TreeVisitor visitor) { + iterate(GIndexUtil.SELF_G_INDEX, GIndexUtil.LEFTMOST_G_INDEX, visitor); + } + + /** Iterates all tree nodes in the order Self -> Left subtree -> Right subtree */ + default void iterateAll(Consumer simpleVisitor) { + iterateAll( + (node, __) -> { + simpleVisitor.accept(node); + return true; + }); + } + + /** + * The same as {@link #updated(long, TreeNode)} except that existing node can be used to calculate + * a new node + * + *

Three method overloads call each other in a cycle. The implementation class should override + * one of them and may override more for efficiency + * + * @see #updated(TreeUpdates) + * @see #updated(long, TreeNode) + * @see #updated(long, Function) + */ + default TreeNode updated(long generalizedIndex, Function nodeUpdater) { + TreeNode newNode = nodeUpdater.apply(get(generalizedIndex)); + return updated( + new TreeUpdates(singletonList(new TreeUpdates.Update(generalizedIndex, newNode)))); + } + + /** + * Updates the tree in a batch. + * + *

Three method overloads call each other in a cycle. The implementation class should override + * one of them and may override more for efficiency + * + * @see #updated(TreeUpdates) + * @see #updated(long, TreeNode) + * @see #updated(long, Function) + */ + default TreeNode updated(TreeUpdates newNodes) { + TreeNode ret = this; + for (int i = 0; i < newNodes.size(); i++) { + ret = ret.updated(newNodes.getRelativeGIndex(i), newNodes.getNode(i)); + } + return ret; + } + + /** + * 'Sets' a new node on place of the node at generalized index. This node and all its descendants + * are left immutable. The updated subtree node is returned. + * + *

Three method overloads call each other in a cycle. The implementation class should override + * one of them and may override more for efficiency + * + * @param generalizedIndex index of tree node to be replaced + * @param node new node either leaf of subtree root node + * @return the updated subtree root node + * @see #updated(TreeUpdates) + * @see #updated(long, TreeNode) + * @see #updated(long, Function) + */ + default TreeNode updated(long generalizedIndex, TreeNode node) { + return updated(generalizedIndex, oldNode -> node); + } +} diff --git a/src/org/minima/system/network/base/ssz/TreeNodeImpl.java b/src/org/minima/system/network/base/ssz/TreeNodeImpl.java new file mode 100644 index 000000000..785586e08 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TreeNodeImpl.java @@ -0,0 +1,135 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Arrays; +import java.util.Objects; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +import org.jetbrains.annotations.NotNull; + +abstract class TreeNodeImpl implements TreeNode { + + static class LeafNodeImpl extends TreeNodeImpl implements LeafNode { + private final byte[] data; + + public LeafNodeImpl(Bytes data) { + checkArgument(data.size() <= MAX_BYTE_SIZE); + this.data = data.toArrayUnsafe(); + } + + @Override + public Bytes getData() { + return Bytes.wrap(data); + } + + @Override + public Bytes32 hashTreeRoot() { + if (data.length == MAX_BYTE_SIZE) { + return Bytes32.wrap(data); + } else { + return Bytes32.wrap(Arrays.copyOf(data, MAX_BYTE_SIZE)); + } + } + + @Override + public TreeNode updated(TreeUpdates newNodes) { + if (newNodes.isEmpty()) { + return this; + } else { + newNodes.checkLeaf(); + return newNodes.getNode(0); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof LeafNode)) { + return false; + } + LeafNode otherLeaf = (LeafNode) o; + return Objects.equals(getData(), otherLeaf.getData()); + } + + @Override + public int hashCode() { + return getData().hashCode(); + } + + @Override + public String toString() { + return "[" + getData() + "]"; + } + } + + static class BranchNodeImpl extends TreeNodeImpl implements BranchNode { + private final TreeNode left; + private final TreeNode right; + private volatile Bytes32 cachedHash = null; + + public BranchNodeImpl(TreeNode left, TreeNode right) { + this.left = left; + this.right = right; + } + + @NotNull + @Override + public TreeNode left() { + return left; + } + + @NotNull + @Override + public TreeNode right() { + return right; + } + + @Override + public BranchNode rebind(boolean left, TreeNode newNode) { + return left ? new BranchNodeImpl(newNode, right()) : new BranchNodeImpl(left(), newNode); + } + + @Override + public TreeNode updated(TreeUpdates newNodes) { + if (newNodes.isEmpty()) { + return this; + } else if (newNodes.isFinal()) { + return newNodes.getNode(0); + } else { + Pair children = newNodes.splitAtPivot(); + return new BranchNodeImpl( + left().updated(children.getLeft()), right().updated(children.getRight())); + } + } + + @Override + public Bytes32 hashTreeRoot() { + if (cachedHash == null) { + cachedHash = BranchNode.super.hashTreeRoot(); + } + return cachedHash; + } + + @Override + public String toString() { + return left == right ? ("(2x " + left + ")") : ("(" + left + ", " + right + ')'); + } + } +} diff --git a/src/org/minima/system/network/base/ssz/TreeUpdates.java b/src/org/minima/system/network/base/ssz/TreeUpdates.java new file mode 100644 index 000000000..8f9d0b6c1 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TreeUpdates.java @@ -0,0 +1,199 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import com.google.common.annotations.VisibleForTesting; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import org.apache.commons.lang3.tuple.Pair; + +/** + * The collection of nodes and their target generalized indexes to be updated The class also + * contains the target generalized index this set of changes is applicable to. + * + * @see TreeNode#updated(TreeUpdates) + */ +public class TreeUpdates { + + /** A single tree update with target generalized index and the new target {@link TreeNode} */ + public static class Update { + private final long generalizedIndex; + private final TreeNode newNode; + + public Update(long generalizedIndex, TreeNode newNode) { + this.generalizedIndex = generalizedIndex; + this.newNode = newNode; + } + + public long getGeneralizedIndex() { + return generalizedIndex; + } + + public TreeNode getNewNode() { + return newNode; + } + } + + /** Convenient collector for the stream with {@link Update} elements */ + public static Collector collector() { + return Collectors.collectingAndThen(Collectors.toList(), TreeUpdates::new); + } + + private final List gIndexes; + private final List nodes; + + private final long prefix; + private final int heightFromLeaf; + + /** + * Creates a new instance of TreeNodes + * + * @param updates the list of {@link Update}s + *

NOTE: the list should conform to the following prerequisites: + *

    + *
  • all generalized indexes are unique + *
  • the list should be sorted by the target generalized index + *
  • the generalized indexes should be on the same tree level. I.e. the highest order bit + * should be the same for all indexes + *
+ * + * @throws IllegalArgumentException if the list doesn't conform to above restrictions + */ + public TreeUpdates(List updates) { + this( + updates.stream().map(Update::getGeneralizedIndex).collect(Collectors.toList()), + updates.stream().map(Update::getNewNode).collect(Collectors.toList())); + } + + private TreeUpdates(List gIndexes, List nodes) { + this(gIndexes, nodes, 1, getDepthAndValidate(gIndexes)); + } + + private static TreeUpdates create( + List gIndexes, List nodes, long prefix, int heightFromLeaf) { + return new TreeUpdates(gIndexes, nodes, prefix, heightFromLeaf); + } + + private TreeUpdates(List gIndexes, List nodes, long prefix, int heightFromLeaf) { + assert gIndexes.size() == nodes.size(); + + this.gIndexes = gIndexes; + this.nodes = nodes; + this.prefix = prefix; + this.heightFromLeaf = heightFromLeaf; + } + + /** + * Split the nodes to left and right subtree subsets according the target generalized index + * + * @return the pair of node updates for left and right subtrees with accordingly adjusted target + * generalized indexes + */ + public Pair splitAtPivot() { + if (heightFromLeaf <= 0) { + throw new IllegalStateException("Can't split leaf update"); + } + long lPrefix = prefix << 1; + long rPrefix = lPrefix | 1; + long pivotGIndex = rPrefix << (heightFromLeaf - 1); + + int idx = Collections.binarySearch(gIndexes, pivotGIndex); + int insIdx = idx < 0 ? -idx - 1 : idx; + return Pair.of( + TreeUpdates.create( + gIndexes.subList(0, insIdx), nodes.subList(0, insIdx), lPrefix, heightFromLeaf - 1), + TreeUpdates.create( + gIndexes.subList(insIdx, gIndexes.size()), + nodes.subList(insIdx, nodes.size()), + rPrefix, + heightFromLeaf - 1)); + } + + /** Number of updated nodes in this set */ + public int size() { + return gIndexes.size(); + } + + public boolean isEmpty() { + return size() == 0; + } + + /** Gets generalized index for update at position [index] */ + @VisibleForTesting + long getGIndex(int index) { + return gIndexes.get(index); + } + + /** Calculates and returns relative generalized index */ + public long getRelativeGIndex(int index) { + return GIndexUtil.gIdxGetRelativeGIndex(gIndexes.get(index), GIndexUtil.gIdxGetDepth(prefix)); + } + + /** Gets new tree node for update at position [index] */ + public TreeNode getNode(int index) { + return nodes.get(index); + } + + private static int getDepthAndValidate(List gIndexes) { + if (gIndexes.isEmpty()) { + return 0; + } + long highestBit = Long.highestOneBit(gIndexes.get(0)); + long mask = highestBit - 1; + long checkMask = ~mask; + + long lastGIdx = -1; + for (int i = 0; i < gIndexes.size(); i++) { + long gIdx = gIndexes.get(i); + if (gIdx < 1) { + throw new IllegalArgumentException("Invalid gIndex: " + gIdx); + } + if (gIdx <= lastGIdx) { + throw new IllegalArgumentException("Invalid gIndex ordering: " + gIndexes); + } + if ((gIdx & checkMask) != highestBit) { + throw new IllegalArgumentException("Indexes are of different depth: [0] and [" + i + "]"); + } + lastGIdx = gIdx; + } + return Long.bitCount(mask); + } + + /** + * Checks if this instance is correct for the leaf node + * + * @throws IllegalArgumentException if not correct + */ + public void checkLeaf() { + if (heightFromLeaf != 0) { + throw new IllegalArgumentException( + "Non-zero heightFromLeaf for the leaf node: " + heightFromLeaf); + } + if (gIndexes.size() != 1) { + throw new IllegalArgumentException( + "Number of nodes should be 1 for a leaf node: " + gIndexes.size()); + } + if (gIndexes.get(0) != prefix) { + throw new IllegalArgumentException( + "Leaf gIndex != prefix: " + gIndexes.get(0) + " != " + prefix); + } + } + + /** Indicates that this update should be applied to the node target generalized index */ + public boolean isFinal() { + return (gIndexes.size() == 1 && gIndexes.get(0) == prefix); + } +} diff --git a/src/org/minima/system/network/base/ssz/TreeUtil.java b/src/org/minima/system/network/base/ssz/TreeUtil.java new file mode 100644 index 000000000..e32af6a43 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TreeUtil.java @@ -0,0 +1,200 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import org.apache.tuweni.bytes.Bytes; +import org.minima.system.network.base.ssz.TreeNodeImpl.BranchNodeImpl; +//import tech.pegasys.teku.ssz.tree.TreeNodeImpl.BranchNodeImpl; +//import tech.pegasys.teku.ssz.tree.TreeNodeImpl.LeafNodeImpl; +import org.minima.system.network.base.ssz.TreeNodeImpl.LeafNodeImpl; + +/** Misc Backing binary tree utils */ +public class TreeUtil { + + static class ZeroLeafNode extends LeafNodeImpl { + public ZeroLeafNode(int size) { + super(Bytes.wrap(new byte[size])); + } + + @Override + public String toString() { + return "(" + getData() + ")"; + } + } + + private static class ZeroBranchNode extends BranchNodeImpl { + private final int height; + + public ZeroBranchNode(TreeNode left, TreeNode right, int height) { + super(left, right); + this.height = height; + } + + @Override + public String toString() { + return "(ZeroBranch-" + height + ")"; + } + } + + @VisibleForTesting public static final TreeNode[] ZERO_TREES; + + static { + ZERO_TREES = new TreeNode[64]; + ZERO_TREES[0] = LeafNode.EMPTY_LEAF; + for (int i = 1; i < ZERO_TREES.length; i++) { + ZERO_TREES[i] = new ZeroBranchNode(ZERO_TREES[i - 1], ZERO_TREES[i - 1], i); + ZERO_TREES[i].hashTreeRoot(); // pre-cache + } + } + + public static int bitsCeilToBytes(int bits) { + return (bits + 7) / 8; + } + + public static long bitsCeilToBytes(long bits) { + return (bits + 7) / 8; + } + + /** + * Creates a binary tree with `nextPowerOf2(maxLength)` width and following leaf nodes + * [zeroElement] * maxLength + [ZERO_LEAF] * (nextPowerOf2(maxLength) - maxLength) + * + * + * @param maxLength max number of leaf nodes + * @param defaultNode default leaf element. For complex vectors it could be default vector element + * struct subtree + */ + public static TreeNode createDefaultTree(long maxLength, TreeNode defaultNode) { + return createTree( + defaultNode, LeafNode.EMPTY_LEAF.equals(defaultNode) ? 0 : maxLength, treeDepth(maxLength)); + } + + /** Creates a binary tree of width `nextPowerOf2(leafNodes.size())` with specific leaf nodes */ + public static TreeNode createTree(List children) { + return createTree(children, treeDepth(children.size())); + } + + private static TreeNode createTree(TreeNode defaultNode, long defaultNodesCount, int depth) { + if (defaultNodesCount == 0) { + return ZERO_TREES[depth]; + } else if (depth == 0) { + checkArgument(defaultNodesCount == 1); + return defaultNode; + } else { + long leftNodesCount = Math.min(defaultNodesCount, 1 << (depth - 1)); + long rightNodesCount = defaultNodesCount - leftNodesCount; + TreeNode lTree = createTree(defaultNode, leftNodesCount, depth - 1); + TreeNode rTree = + leftNodesCount == rightNodesCount + ? lTree + : createTree(defaultNode, rightNodesCount, depth - 1); + return new BranchNodeImpl(lTree, rTree); + } + } + + public static TreeNode createTree(List leafNodes, int depth) { + if (leafNodes.isEmpty()) { + return ZERO_TREES[depth]; + } else if (depth == 0) { + checkArgument(leafNodes.size() == 1); + return leafNodes.get(0); + } else { + long index = 1L << (depth - 1); + int iIndex = index > leafNodes.size() ? leafNodes.size() : (int) index; + + List leftSublist = leafNodes.subList(0, iIndex); + List rightSublist = leafNodes.subList(iIndex, leafNodes.size()); + return BranchNode.create( + createTree(leftSublist, depth - 1), createTree(rightSublist, depth - 1)); + } + } + + public static TreeNode createTree( + List leafNodes, TreeNode defaultNode, int depth) { + if (leafNodes.isEmpty()) { + if (depth > 0) { + TreeNode defaultChild = createTree(leafNodes, defaultNode, depth - 1); + return BranchNode.create(defaultChild, defaultChild); + } else { + return defaultNode; + } + } else if (depth == 0) { + checkArgument(leafNodes.size() == 1); + return leafNodes.get(0); + } else { + long index = 1L << (depth - 1); + int iIndex = index > leafNodes.size() ? leafNodes.size() : (int) index; + + List leftSublist = leafNodes.subList(0, iIndex); + List rightSublist = leafNodes.subList(iIndex, leafNodes.size()); + return BranchNode.create( + createTree(leftSublist, defaultNode, depth - 1), + createTree(rightSublist, defaultNode, depth - 1)); + } + } + + public static long nextPowerOf2(long x) { + return x <= 1 ? 1 : Long.highestOneBit(x - 1) << 1; + } + + public static int treeDepth(long maxChunks) { + return Long.bitCount(nextPowerOf2(maxChunks) - 1); + } + + /** + * Iterate all leaf tree nodes starting from the node with general index {@code fromGeneralIndex} + * (including all node descendants if this is a branch node) and ending with the node with general + * index {@code toGeneralIndex} inclusive (including all node descendants if this is a branch + * node). On every {@link LeafNode} the supplied {@code visitor} is invoked. + */ + @VisibleForTesting + static void iterateLeaves( + TreeNode node, long fromGeneralIndex, long toGeneralIndex, Consumer visitor) { + node.iterateRange( + fromGeneralIndex, + toGeneralIndex, + (n, idx) -> { + if (n instanceof LeafNode) { + visitor.accept((LeafNode) n); + } + return true; + }); + } + + public static void iterateLeavesData( + TreeNode node, long fromGeneralIndex, long toGeneralIndex, Consumer visitor) { + node.iterateRange( + fromGeneralIndex, + toGeneralIndex, + (n, idx) -> { + if (n instanceof LeafDataNode) { + visitor.accept(((LeafDataNode) n).getData()); + } + return true; + }); + } + + public static Bytes concatenateLeavesData(TreeNode tree) { + List leavesData = new ArrayList<>(); + iterateLeavesData( + tree, GIndexUtil.LEFTMOST_G_INDEX, GIndexUtil.RIGHTMOST_G_INDEX, leavesData::add); + return Bytes.wrap(leavesData.toArray(new Bytes[0])); + } +} diff --git a/src/org/minima/system/network/base/ssz/TreeVisitor.java b/src/org/minima/system/network/base/ssz/TreeVisitor.java new file mode 100644 index 000000000..4a26f63d1 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/TreeVisitor.java @@ -0,0 +1,25 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +/** Visitor callback interface for traversing binary tree {@link TreeNode}s */ +public interface TreeVisitor { + + /** + * 'Visits' a tree node with specifying its generalized index + * + * @return true if the visitor wishes to continue or false if further iteration should break + */ + boolean visit(TreeNode node, long generalizedIndex); +} diff --git a/src/org/minima/system/network/base/ssz/UInt64.java b/src/org/minima/system/network/base/ssz/UInt64.java new file mode 100644 index 000000000..dcbd3d326 --- /dev/null +++ b/src/org/minima/system/network/base/ssz/UInt64.java @@ -0,0 +1,530 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.ssz; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.stream.Stream; + +/** An unsigned 64-bit integer. All instances are immutable. */ +public final class UInt64 implements Comparable { + + private static final long UNSIGNED_MASK = 0x7fffffffffffffffL; + private static final long HIGH_MASK = 0xffffffff00000000L; + private static final long LOW_MASK = 0x00000000ffffffffL; + + public static final int BYTES = 8; + + public static final UInt64 ZERO = new UInt64(0); + public static final UInt64 ONE = new UInt64(1); + public static final UInt64 MAX_VALUE = new UInt64(-1L); + + private final long value; + + private UInt64(final long value) { + this.value = value; + } + + /** + * Create a UInt64 from a long value. The value is treated as signed and must be >= 0. + * + * @param value the signed value to convert to UInt64 + * @return UInt64 value equal to the supplied signed value + * @throws IllegalArgumentException if the value is negative + */ + public static UInt64 valueOf(final long value) { + checkPositive(value); + return fromLongBits(value); + } + + /** + * Parse the string value into a UInt64. + * + * @param value the value to parse + * @return UInt64 parsed from value + * @throws NumberFormatException if value is not a valid unsigned long. + */ + public static UInt64 valueOf(final String value) { + return fromLongBits(Long.parseUnsignedLong(value)); + } + + /** + * Create a UInt64 from a {@link BigInteger} value. The value is treated as signed and must be >= + * 0 and less than {@link #MAX_VALUE}. + * + * @param value the signed value to convert to UInt64 + * @return UInt64 value equal to the supplied signed value + * @throws IllegalArgumentException if the value is negative or too large + */ + public static UInt64 valueOf(final BigInteger value) { + checkArgument( + value.signum() >= 0 && value.bitLength() <= Long.SIZE, + "value (%s) is outside the range for uint64", + value); + return fromLongBits(value.longValue()); + } + + /** + * Create a UInt64 from an unsigned long. The value is treated as unsigned. + * + * @param value the unsigned value to create as a UInt64. + * @return the created UInt64. + */ + public static UInt64 fromLongBits(final long value) { + return value == 0 ? ZERO : new UInt64(value); + } + + /** + * Return the result of adding this value and the specified one. + * + * @param other the value to add. Treated as signed. + * @return a new UInt64 equal to this value + the specified value. + * @throws ArithmeticException if the result exceeds {@link #MAX_VALUE} + * @throws IllegalArgumentException if the specified value is negative + */ + public UInt64 plus(final long other) { + checkPositive(other); + return plus(this.value, other); + } + + /** + * Increment this value by one and return the result. + * + * @return The result of incrementing this value by 1. + */ + public UInt64 increment() { + return plus(1); + } + + /** + * Decrement this value by one and return the result. + * + * @return The result of decrementing this value by 1. + */ + public UInt64 decrement() { + return minus(1); + } + + /** + * Return the result of adding this value and the specified one. + * + * @param other the unsigned value to add. + * @return a new UInt64 equal to this value + the specified value. + * @throws ArithmeticException if the result exceeds {@link #MAX_VALUE} + */ + public UInt64 plus(final UInt64 other) { + return plus(value, other.value); + } + + private UInt64 plus(final long longBits1, final long longBits2) { + if (longBits1 != 0 && Long.compareUnsigned(longBits2, MAX_VALUE.longValue() - longBits1) > 0) { + throw new ArithmeticException("uint64 overflow"); + } + return fromLongBits(longBits1 + longBits2); + } + + /** + * Return the result of subtracting the specified value from this one. + * + * @param other the value to subtract, treated as signed. + * @return a new UInt64 equal to this value minus the specified value. + * @throws ArithmeticException if the result is less than zero. + * @throws IllegalArgumentException if the specified value is negative + */ + public UInt64 minus(final long other) { + checkPositive(other); + return minus(value, other); + } + + /** + * Return the result of subtracting the specified value from this one. + * + * @param other the value to subtract. + * @return a new UInt64 equal to this value minus the specified value. + * @throws ArithmeticException if the result is less than zero. + */ + public UInt64 minus(final UInt64 other) { + return minus(value, other.value); + } + + /** + * Return the result of subtracting the specified value from this one. If the operation would + * cause an underflow, an empty result is returned. + * + * @param other the value to subtract. + * @return a new UInt64 equal to this value minus the specified value. + */ + public Optional safeMinus(final long other) { + checkPositive(other); + if (Long.compareUnsigned(value, other) < 0) { + return Optional.empty(); + } + + return Optional.of(fromLongBits(value - other)); + } + + /** + * Return the result of subtracting the specified value from this one. If the operation would + * cause an underflow, an empty result is returned. + * + * @param other the value to subtract. + * @return a new UInt64 equal to this value minus the specified value. + */ + public Optional safeMinus(final UInt64 other) { + if (isLessThan(other)) { + return Optional.empty(); + } + return Optional.of(fromLongBits(value - other.value)); + } + + private UInt64 minus(final long longBits1, final long longBits2) { + if (Long.compareUnsigned(longBits1, longBits2) < 0) { + throw new ArithmeticException("uint64 underflow"); + } + return fromLongBits(longBits1 - longBits2); + } + + public UInt64 minusMinZero(final long other) { + return minusMinZero(valueOf(other)); + } + + public UInt64 minusMinZero(final UInt64 other) { + return isGreaterThan(other) ? minus(other) : ZERO; + } + + /** + * Return the result of multiplying the specified value with this one. + * + * @param other the value to multiply, treated as signed. + * @return a new UInt64 equal to this value times the specified value. + * @throws ArithmeticException if the result is exceeds {@link #MAX_VALUE} + * @throws IllegalArgumentException if the specified value is negative + */ + public UInt64 times(final long other) { + checkPositive(other); + return times(value, other); + } + + /** + * Return the result of multiplying the specified value with this one. + * + * @param other the value to multiply. + * @return a new UInt64 equal to this value times the specified value. + * @throws ArithmeticException if the result is exceeds {@link #MAX_VALUE} + */ + public UInt64 times(final UInt64 other) { + return times(value, other.value); + } + + /** Naive long-multiplication is quite efficient */ + private UInt64 times(final long longBits1, final long longBits2) { + if (Long.numberOfLeadingZeros(longBits1) + Long.numberOfLeadingZeros(longBits2) >= 64) { + return UInt64.fromLongBits(longBits1 * longBits2); + } + final long longBits1Hi = longBits1 >>> 32; + final long longBits1Lo = longBits1 & LOW_MASK; + final long longBits2Hi = longBits2 >>> 32; + final long longBits2Lo = longBits2 & LOW_MASK; + if (longBits1Hi * longBits2Hi != 0) { + throw new ArithmeticException("uint64 overflow"); + } + // One or the other of longBits1Hi and longBits2Hi is zero + final long crossProduct = + (longBits1Hi == 0) ? longBits1Lo * longBits2Hi : longBits1Hi * longBits2Lo; + if ((crossProduct & HIGH_MASK) != 0) { + throw new ArithmeticException("uint64 overflow"); + } + return plus(crossProduct << 32, longBits1Lo * longBits2Lo); + } + + /** + * Return the result of dividing this value by the specified value. + * + * @param divisor the value to divide by, treated as signed. + * @return a new UInt64 equal to this value divided by the specified value. + * @throws ArithmeticException if the specified divisor is 0. + * @throws IllegalArgumentException if the specified value is negative + */ + public UInt64 dividedBy(final long divisor) { + checkPositive(divisor); + return dividedBy(value, divisor); + } + + /** + * Return the result of dividing this value by the specified value. + * + * @param divisor the value to divide by. + * @return a new UInt64 equal to this value divided by the specified value. + * @throws ArithmeticException if the specified divisor is 0. + */ + public UInt64 dividedBy(final UInt64 divisor) { + return dividedBy(value, divisor.value); + } + + private UInt64 dividedBy(final long unsignedDividend, final long unsignedDivisor) { + return fromLongBits(Long.divideUnsigned(unsignedDividend, unsignedDivisor)); + } + + /** + * Returns this value modulo the specified value. + * + * @param divisor the divisor, treated as signed. + * @return a new UInt64 equal to this value modulo the specified value. + * @throws ArithmeticException if the specified divisor is 0. + * @throws IllegalArgumentException if the specified value is negative + */ + public UInt64 mod(final long divisor) { + checkPositive(divisor); + return mod(value, divisor); + } + + /** + * Returns this value modulo the specified value. + * + * @param divisor the divisor. + * @return a new UInt64 equal to this value modulo the specified value. + * @throws ArithmeticException if the specified divisor is 0. + */ + public UInt64 mod(final UInt64 divisor) { + return mod(value, divisor.value); + } + + private UInt64 mod(final long dividendBits, final long divisorBits) { + return fromLongBits(Long.remainderUnsigned(dividendBits, divisorBits)); + } + + /** + * Return the larger of this value or the specified value. + * + * @param other the value to compare with + * @return the larger value + */ + public UInt64 max(final UInt64 other) { + return compareTo(other) >= 0 ? this : other; + } + + /** + * Return the larger of this value or the specified value. + * + * @param other the value to compare with, treated as signed + * @return the larger value + */ + public UInt64 max(final long other) { + checkPositive(other); + return Long.compareUnsigned(value, other) >= 0 ? this : UInt64.valueOf(other); + } + + /** + * Return the smaller of this value or the specified value. + * + * @param other the value to compare with + * @return the larger value + */ + public UInt64 min(final UInt64 other) { + return compareTo(other) >= 0 ? other : this; + } + + /** + * Return the smaller of this value or the specified value. + * + * @param other the value to compare with, treated as signed + * @return the larger value + */ + public UInt64 min(final long other) { + checkPositive(other); + return Long.compareUnsigned(value, other) <= 0 ? this : UInt64.valueOf(other); + } + + @Override + public int compareTo(final UInt64 o) { + return Long.compareUnsigned(value, o.value); + } + + public int compareTo(final long other) { + checkPositive(other); + return Long.compareUnsigned(value, other); + } + + /** + * Returns true if this value is zero. + * + * @return true if this value is zero. + */ + public boolean isZero() { + return value == 0; + } + + /** + * Returns true if this value is strictly greater than the specified value. + * + * @param other the value to compare to + * @return true if this value is strictly greater than the specified value + */ + public boolean isGreaterThan(final UInt64 other) { + return compareTo(other) > 0; + } + + /** + * Returns true if this value is strictly greater than the specified value. + * + * @param other the value to compare to + * @return true if this value is strictly greater than the specified value + */ + public boolean isGreaterThan(final long other) { + return compareTo(other) > 0; + } + + /** + * Returns true if this value is greater than or equal to the specified value. + * + * @param other the value to compare to + * @return true if this value is greater or equal than the specified value + */ + public boolean isGreaterThanOrEqualTo(final UInt64 other) { + return compareTo(other) >= 0; + } + + /** + * Returns true if this value is greater than or equal to the specified value. + * + * @param other the value to compare to + * @return true if this value is greater or equal than the specified value + */ + public boolean isGreaterThanOrEqualTo(final long other) { + return compareTo(other) >= 0; + } + + /** + * Returns true if this value is strictly less than the specified value. + * + * @param other the value to compare to + * @return true if this value is strictly less than the specified value + */ + public boolean isLessThan(final UInt64 other) { + return compareTo(other) < 0; + } + + /** + * Returns true if this value is strictly less than the specified value. + * + * @param other the value to compare to + * @return true if this value is strictly less than the specified value + */ + public boolean isLessThan(final long other) { + return compareTo(other) < 0; + } + + /** + * Returns true if this value is less than or equal to the specified value. + * + * @param other the value to compare to + * @return true if this value is less than or equal to the specified value + */ + public boolean isLessThanOrEqualTo(final UInt64 other) { + return compareTo(other) <= 0; + } + + /** + * Returns true if this value is less than or equal to the specified value. + * + * @param other the value to compare to + * @return true if this value is less than or equal to the specified value + */ + public boolean isLessThanOrEqualTo(final long other) { + return compareTo(other) <= 0; + } + + /** + * Returns the value as a long. If this value exceeds {@link Long#MAX_VALUE} the result will be + * negative. + * + * @return this value as an unsigned long + */ + public long longValue() { + return value; + } + + /** + * This value as an int. + * + * @return this value as a signed int. + * @throws ArithmeticException if the value is greater than {@link Integer#MAX_VALUE} + */ + public int intValue() { + final int intValue = Math.toIntExact(value); + if (intValue < 0) { + throw new ArithmeticException("integer overflow"); + } + return intValue; + } + + /** + * This value as a double. + * + * @return this value as a double. + */ + public double doubleValue() { + return value; + } + + /** + * Returns this value as a {@link BigInteger} + * + * @return this value as a BigInteger + */ + public BigInteger bigIntegerValue() { + return toUnsignedBigInteger(value); + } + // From Guava UnsignedLong.bigIntegerValue(). Apache 2 license. + + private static BigInteger toUnsignedBigInteger(final long value) { + BigInteger bigInt = BigInteger.valueOf(value & UNSIGNED_MASK); + if (value < 0) { + bigInt = bigInt.setBit(Long.SIZE - 1); + } + return bigInt; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final UInt64 uInt64 = (UInt64) o; + return value == uInt64.value; + } + + @Override + public int hashCode() { + return Long.hashCode(value); + } + + @Override + public String toString() { + return Long.toUnsignedString(value); + } + + private static void checkPositive(final long other) { + checkArgument(other >= 0, "value (%s) must be >= 0", other); + } + + public static Stream range(final UInt64 fromInclusive, final UInt64 toExclusive) { + return Stream.iterate(fromInclusive, value -> value.isLessThan(toExclusive), UInt64::increment); + } +} From 1dd12cde8b3f5ec3d72e0327fb0a3507b42245dd Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 8 Apr 2021 17:06:04 +0100 Subject: [PATCH 06/55] p2p: added teku peer classes to represent peer hosts. --- .../network/base/peer/DisconnectReason.java | 61 +++++++++++++++++ .../base/peer/DisconnectRequestHandler.java | 23 +++++++ .../minima/system/network/base/peer/Peer.java | 66 +++++++++++++++++++ .../system/network/base/peer/PeerAddress.java | 65 ++++++++++++++++++ .../base/peer/PeerConnectedSubscriber.java | 20 ++++++ .../base/peer/PeerDisconnectedSubscriber.java | 22 +++++++ .../system/network/base/peer/PeerPools.java | 53 +++++++++++++++ .../base/peer/PeerSelectionStrategy.java | 31 +++++++++ .../base/peer/ReputationAdjustment.java | 35 ++++++++++ 9 files changed, 376 insertions(+) create mode 100644 src/org/minima/system/network/base/peer/DisconnectReason.java create mode 100644 src/org/minima/system/network/base/peer/DisconnectRequestHandler.java create mode 100644 src/org/minima/system/network/base/peer/Peer.java create mode 100644 src/org/minima/system/network/base/peer/PeerAddress.java create mode 100644 src/org/minima/system/network/base/peer/PeerConnectedSubscriber.java create mode 100644 src/org/minima/system/network/base/peer/PeerDisconnectedSubscriber.java create mode 100644 src/org/minima/system/network/base/peer/PeerPools.java create mode 100644 src/org/minima/system/network/base/peer/PeerSelectionStrategy.java create mode 100644 src/org/minima/system/network/base/peer/ReputationAdjustment.java diff --git a/src/org/minima/system/network/base/peer/DisconnectReason.java b/src/org/minima/system/network/base/peer/DisconnectReason.java new file mode 100644 index 000000000..a5a46d32d --- /dev/null +++ b/src/org/minima/system/network/base/peer/DisconnectReason.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Optional; +import java.util.stream.Stream; +//import tech.pegasys.teku.infrastructure.unsigned.UInt64; +//import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.GoodbyeMessage; + +// from GoodbyeMessage.java +// public static final UInt64 REASON_CLIENT_SHUT_DOWN = UInt64.valueOf(1); +// public static final UInt64 REASON_IRRELEVANT_NETWORK = UInt64.valueOf(2); +// public static final UInt64 REASON_FAULT_ERROR = UInt64.valueOf(3); +// public static final UInt64 MIN_CUSTOM_REASON_CODE = UInt64.valueOf(128); + +// // Custom reasons +// public static final UInt64 REASON_UNABLE_TO_VERIFY_NETWORK = UInt64.valueOf(128); +// public static final UInt64 REASON_TOO_MANY_PEERS = UInt64.valueOf(129); +// public static final UInt64 REASON_RATE_LIMITING = UInt64.valueOf(130) +public enum DisconnectReason { + IRRELEVANT_NETWORK(2, true), + UNABLE_TO_VERIFY_NETWORK(128, true), + TOO_MANY_PEERS(129, false), + REMOTE_FAULT(3, false), + UNRESPONSIVE(3, false), + SHUTTING_DOWN(1, false), + RATE_LIMITING(130, false); + + private final long reasonCode; + private final boolean isPermanent; + + DisconnectReason(final long reasonCode, final boolean isPermanent) { + this.reasonCode = reasonCode; + this.isPermanent = isPermanent; + } + + public static Optional fromReasonCode(final long reasonCode) { + return Stream.of(values()) + .filter(reason -> reason.getReasonCode() == reasonCode) + .findAny(); + } + + public long getReasonCode() { + return reasonCode; + } + + public boolean isPermanent() { + return isPermanent; + } +} diff --git a/src/org/minima/system/network/base/peer/DisconnectRequestHandler.java b/src/org/minima/system/network/base/peer/DisconnectRequestHandler.java new file mode 100644 index 000000000..40581f073 --- /dev/null +++ b/src/org/minima/system/network/base/peer/DisconnectRequestHandler.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import org.minima.system.network.base.SafeFuture; + +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +public interface DisconnectRequestHandler { + + SafeFuture requestDisconnect(DisconnectReason reason); +} diff --git a/src/org/minima/system/network/base/peer/Peer.java b/src/org/minima/system/network/base/peer/Peer.java new file mode 100644 index 000000000..749ed451f --- /dev/null +++ b/src/org/minima/system/network/base/peer/Peer.java @@ -0,0 +1,66 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Objects; +import java.util.Optional; + +//import org.minima.system.network.base.DisconnectReason; +//import org.minima.system.network.base.peer.PeerDisconnectedSubscriber; +import org.minima.system.network.base.RpcMethod; +import org.minima.system.network.base.RpcRequestHandler; +import org.minima.system.network.base.RpcStream; +import org.minima.system.network.base.SafeFuture; + +//import tech.pegasys.teku.networking.p2p.peer.DisconnectRequestHandler; + +//import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; +//import tech.pegasys.teku.networking.p2p.network.PeerAddress; +//import tech.pegasys.teku.networking.p2p.reputation.ReputationAdjustment; +//import tech.pegasys.teku.networking.p2p.rpc.RpcMethod; +//import tech.pegasys.teku.networking.p2p.rpc.RpcRequestHandler; +//import tech.pegasys.teku.networking.p2p.rpc.RpcStream; + +public interface Peer { + + default NodeId getId() { + return getAddress().getId(); + } + + PeerAddress getAddress(); + + boolean isConnected(); + + void disconnectImmediately(Optional reason, boolean locallyInitiated); + + SafeFuture disconnectCleanly(DisconnectReason reason); + + void setDisconnectRequestHandler(DisconnectRequestHandler handler); + + void subscribeDisconnect(PeerDisconnectedSubscriber subscriber); + + SafeFuture sendRequest( + RpcMethod rpcMethod, byte[] initialPayload, RpcRequestHandler handler); + + boolean connectionInitiatedLocally(); + + boolean connectionInitiatedRemotely(); + + default boolean idMatches(final Peer other) { + return other != null && Objects.equals(getId(), other.getId()); + } + + void adjustReputation(final ReputationAdjustment adjustment); +} diff --git a/src/org/minima/system/network/base/peer/PeerAddress.java b/src/org/minima/system/network/base/peer/PeerAddress.java new file mode 100644 index 000000000..43786e6af --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerAddress.java @@ -0,0 +1,65 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.networking.p2p.network; + +import java.util.Objects; +import java.util.Optional; +import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public class PeerAddress { + private final NodeId id; + + public PeerAddress(final NodeId id) { + this.id = id; + } + + public String toExternalForm() { + return toString(); + } + + public NodeId getId() { + return id; + } + + @SuppressWarnings("unchecked") + public Optional as(final Class clazz) { + if (clazz.isInstance(this)) { + return Optional.of((T) this); + } else { + return Optional.empty(); + } + } + + @Override + public String toString() { + return id.toString(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final PeerAddress that = (PeerAddress) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/src/org/minima/system/network/base/peer/PeerConnectedSubscriber.java b/src/org/minima/system/network/base/peer/PeerConnectedSubscriber.java new file mode 100644 index 000000000..a5c2c5b07 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerConnectedSubscriber.java @@ -0,0 +1,20 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +@FunctionalInterface +public interface PeerConnectedSubscriber { + + void onConnected(T peer); +} diff --git a/src/org/minima/system/network/base/peer/PeerDisconnectedSubscriber.java b/src/org/minima/system/network/base/peer/PeerDisconnectedSubscriber.java new file mode 100644 index 000000000..31ca1ca49 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerDisconnectedSubscriber.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Optional; + +@FunctionalInterface +public interface PeerDisconnectedSubscriber { + + void onDisconnected(Optional reason, boolean locallyInitiated); +} diff --git a/src/org/minima/system/network/base/peer/PeerPools.java b/src/org/minima/system/network/base/peer/PeerPools.java new file mode 100644 index 000000000..5bf8f62b5 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerPools.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public class PeerPools { + + private static final PeerPool DEFAULT_POOL = PeerPool.SCORE_BASED; + private final Map knownSources = new ConcurrentHashMap<>(); + + public void addPeerToPool(final NodeId nodeId, final PeerPool peerPool) { + if (peerPool == DEFAULT_POOL) { + // No need to record the peer if it's in the default pool anyway. + forgetPeer(nodeId); + } else { + knownSources.put(nodeId, peerPool); + } + } + + public void forgetPeer(final NodeId nodeId) { + knownSources.remove(nodeId); + } + + public PeerPool getPool(final NodeId nodeId) { + return knownSources.getOrDefault(nodeId, DEFAULT_POOL); + } + + public enum PeerPool { + /** Default pool where peers are ranked based on their usefulness */ + SCORE_BASED, + /** + * Pool of peers we randomly selected which are kept connected to provide Sybil resistance + * regardless of their usefulness + */ + RANDOMLY_SELECTED, + /** Static peers which we maintain persistent connections to */ + STATIC + } +} diff --git a/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java b/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java new file mode 100644 index 000000000..9768b0341 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.List; +import java.util.function.Supplier; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.network.P2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.Peer; + +import org.minima.system.network.base.DiscoveryPeer; +import org.minima.system.network.base.P2PNetwork; + +public interface PeerSelectionStrategy { + List selectPeersToConnect( + P2PNetwork network, PeerPools peerPools, Supplier> candidates); + + List selectPeersToDisconnect(P2PNetwork network, PeerPools peerPools); +} diff --git a/src/org/minima/system/network/base/peer/ReputationAdjustment.java b/src/org/minima/system/network/base/peer/ReputationAdjustment.java new file mode 100644 index 000000000..ef2e85b7c --- /dev/null +++ b/src/org/minima/system/network/base/peer/ReputationAdjustment.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public enum ReputationAdjustment { + + // from ReputationManager.java + //static final int LARGE_CHANGE = 10; + //static final int SMALL_CHANGE = 3; + LARGE_PENALTY(-10), + SMALL_PENALTY(-3), + SMALL_REWARD(3), + LARGE_REWARD(10); + + private final int scoreChange; + + ReputationAdjustment(final int scoreChange) { + this.scoreChange = scoreChange; + } + + int getScoreChange() { + return scoreChange; + } +} From b871b600c96988794cc192a4eb934c78030d81e5 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 8 Apr 2021 17:06:27 +0100 Subject: [PATCH 07/55] p2p: added teku/besu metrics classes needed for p2p classes. --- .../network/base/metrics/NoOpCounter.java | 24 ++++ .../base/metrics/NoOpMetricsSystem.java | 126 ++++++++++++++++++ .../base/metrics/ObservableMetricsSystem.java | 44 ++++++ .../network/base/metrics/Observation.java | 84 ++++++++++++ 4 files changed, 278 insertions(+) create mode 100644 src/org/minima/system/network/base/metrics/NoOpCounter.java create mode 100644 src/org/minima/system/network/base/metrics/NoOpMetricsSystem.java create mode 100644 src/org/minima/system/network/base/metrics/ObservableMetricsSystem.java create mode 100644 src/org/minima/system/network/base/metrics/Observation.java diff --git a/src/org/minima/system/network/base/metrics/NoOpCounter.java b/src/org/minima/system/network/base/metrics/NoOpCounter.java new file mode 100644 index 000000000..cc7f34fc7 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/NoOpCounter.java @@ -0,0 +1,24 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +class NoOpCounter implements Counter { + + @Override + public void inc() {} + + @Override + public void inc(final long amount) {} +} \ No newline at end of file diff --git a/src/org/minima/system/network/base/metrics/NoOpMetricsSystem.java b/src/org/minima/system/network/base/metrics/NoOpMetricsSystem.java new file mode 100644 index 000000000..091532710 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/NoOpMetricsSystem.java @@ -0,0 +1,126 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +//import org.hyperledger.besu.metrics.ObservableMetricsSystem; +//import org.hyperledger.besu.metrics.Observation; +//import org.hyperledger.besu.plugin.services.metrics.Counter; +//import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +//import org.hyperledger.besu.plugin.services.metrics.MetricCategory; +//import org.hyperledger.besu.plugin.services.metrics.OperationTimer; + +import java.util.Collections; +import java.util.Set; +import java.util.function.DoubleSupplier; +import java.util.stream.Stream; + +import com.google.common.base.Preconditions; + +public class NoOpMetricsSystem implements ObservableMetricsSystem { + + public static final Counter NO_OP_COUNTER = new NoOpCounter(); + private static final OperationTimer.TimingContext NO_OP_TIMING_CONTEXT = () -> 0; + public static final OperationTimer NO_OP_OPERATION_TIMER = () -> NO_OP_TIMING_CONTEXT; + + public static final LabelledMetric NO_OP_LABELLED_1_COUNTER = + new LabelCountingNoOpMetric<>(1, NO_OP_COUNTER); + public static final LabelledMetric NO_OP_LABELLED_2_COUNTER = + new LabelCountingNoOpMetric<>(2, NO_OP_COUNTER); + public static final LabelledMetric NO_OP_LABELLED_3_COUNTER = + new LabelCountingNoOpMetric<>(3, NO_OP_COUNTER); + public static final LabelledMetric NO_OP_LABELLED_1_OPERATION_TIMER = + new LabelCountingNoOpMetric<>(1, NO_OP_OPERATION_TIMER); + + @Override + public LabelledMetric createLabelledCounter( + final MetricCategory category, + final String name, + final String help, + final String... labelNames) { + return getCounterLabelledMetric(labelNames.length); + } + + public static LabelledMetric getCounterLabelledMetric(final int labelCount) { + switch (labelCount) { + case 1: + return NO_OP_LABELLED_1_COUNTER; + case 2: + return NO_OP_LABELLED_2_COUNTER; + case 3: + return NO_OP_LABELLED_3_COUNTER; + default: + return new LabelCountingNoOpMetric<>(labelCount, NO_OP_COUNTER); + } + } + + @Override + public LabelledMetric createLabelledTimer( + final MetricCategory category, + final String name, + final String help, + final String... labelNames) { + return getOperationTimerLabelledMetric(labelNames.length); + } + + public static LabelledMetric getOperationTimerLabelledMetric( + final int labelCount) { + if (labelCount == 1) { + return NO_OP_LABELLED_1_OPERATION_TIMER; + } else { + return new LabelCountingNoOpMetric<>(labelCount, NO_OP_OPERATION_TIMER); + } + } + + @Override + public void createGauge( + final MetricCategory category, + final String name, + final String help, + final DoubleSupplier valueSupplier) {} + + @Override + public Stream streamObservations(final MetricCategory category) { + return Stream.empty(); + } + + @Override + public Stream streamObservations() { + return Stream.empty(); + } + + @Override + public Set getEnabledCategories() { + return Collections.emptySet(); + } + + public static class LabelCountingNoOpMetric implements LabelledMetric { + + final int labelCount; + final T fakeMetric; + + LabelCountingNoOpMetric(final int labelCount, final T fakeMetric) { + this.labelCount = labelCount; + this.fakeMetric = fakeMetric; + } + + @Override + public T labels(final String... labels) { + Preconditions.checkArgument( + labels.length == labelCount, + "The count of labels used must match the count of labels expected."); + return fakeMetric; + } + } +} \ No newline at end of file diff --git a/src/org/minima/system/network/base/metrics/ObservableMetricsSystem.java b/src/org/minima/system/network/base/metrics/ObservableMetricsSystem.java new file mode 100644 index 000000000..8f0a4366a --- /dev/null +++ b/src/org/minima/system/network/base/metrics/ObservableMetricsSystem.java @@ -0,0 +1,44 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.minima.system.network.base.metrics; + +import java.util.Set; +import java.util.stream.Stream; + +public interface ObservableMetricsSystem extends MetricsSystem { + + Stream streamObservations(MetricCategory category); + + Stream streamObservations(); + + /** + * Provides an immutable view into the metric categories enabled for metric collection. + * + * @return the set of enabled metric categories. + */ + Set getEnabledCategories(); + + /** + * Checks if a particular category of metrics is enabled. + * + * @param category the category to check + * @return true if the category is enabled, false otherwise + */ + default boolean isCategoryEnabled(final MetricCategory category) { + return getEnabledCategories().stream() + .anyMatch(metricCategory -> metricCategory.getName().equals(category.getName())); + } +} diff --git a/src/org/minima/system/network/base/metrics/Observation.java b/src/org/minima/system/network/base/metrics/Observation.java new file mode 100644 index 000000000..1d4d43279 --- /dev/null +++ b/src/org/minima/system/network/base/metrics/Observation.java @@ -0,0 +1,84 @@ +/* + * Copyright ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.minima.system.network.base.metrics; + +import java.util.List; +import java.util.Objects; + +import com.google.common.base.MoreObjects; + +public class Observation { + private final MetricCategory category; + private final String metricName; + private final List labels; + private final Object value; + + public Observation( + final MetricCategory category, + final String metricName, + final Object value, + final List labels) { + this.category = category; + this.metricName = metricName; + this.value = value; + this.labels = labels; + } + + public MetricCategory getCategory() { + return category; + } + + public String getMetricName() { + return metricName; + } + + public List getLabels() { + return labels; + } + + public Object getValue() { + return value; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final Observation that = (Observation) o; + return category == that.category + && Objects.equals(metricName, that.metricName) + && Objects.equals(labels, that.labels) + && Objects.equals(value, that.value); + } + + @Override + public int hashCode() { + return Objects.hash(category, metricName, labels, value); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("category", category) + .add("metricName", metricName) + .add("labels", labels) + .add("value", value) + .toString(); + } +} \ No newline at end of file From f9c5798b3a423b065c784149b084dfb2238c2982 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 8 Apr 2021 17:07:40 +0100 Subject: [PATCH 08/55] p2p: higher level classes to enable libp2p peer discovery. --- .../network/base/ConnectionManager.java | 235 ++++++++++++++++++ .../network/base/DelegatingP2PNetwork.java | 131 ++++++++++ .../system/network/base/DiscoveryConfig.java | 123 +++++++++ .../system/network/base/DiscoveryNetwork.java | 232 +++++++++++++++++ .../network/base/DiscoveryNetworkFactory.java | 143 +++++++++++ .../system/network/base/DiscoveryPeer.java | 86 +++++++ .../system/network/base/DiscoveryService.java | 36 +++ .../minima/system/network/base/EnrForkId.java | 78 ++++++ .../base/InvalidConfigurationException.java | 28 +++ .../system/network/base/KeyValueStore.java | 98 ++++++++ .../system/network/base/NetworkConfig.java | 209 ++++++++++++++++ .../network/base/NoOpDiscoveryService.java | 57 +++++ .../system/network/base/P2PNetwork.java | 115 +++++++++ .../minima/system/network/base/P2PStart.java | 50 +--- .../system/network/base/PortAvailability.java | 61 +++++ .../minima/system/network/base/RpcMethod.java | 21 ++ .../network/base/RpcRequestHandler.java | 30 +++ .../minima/system/network/base/RpcStream.java | 37 +++ .../minima/system/network/base/Service.java | 52 ++++ .../network/base/StreamClosedException.java | 16 ++ .../minima/system/network/base/Waiter.java | 84 +++++++ .../system/network/base/WireLogsConfig.java | 95 +++++++ 22 files changed, 1970 insertions(+), 47 deletions(-) create mode 100644 src/org/minima/system/network/base/ConnectionManager.java create mode 100644 src/org/minima/system/network/base/DelegatingP2PNetwork.java create mode 100644 src/org/minima/system/network/base/DiscoveryConfig.java create mode 100644 src/org/minima/system/network/base/DiscoveryNetwork.java create mode 100644 src/org/minima/system/network/base/DiscoveryNetworkFactory.java create mode 100644 src/org/minima/system/network/base/DiscoveryPeer.java create mode 100644 src/org/minima/system/network/base/DiscoveryService.java create mode 100644 src/org/minima/system/network/base/EnrForkId.java create mode 100644 src/org/minima/system/network/base/InvalidConfigurationException.java create mode 100644 src/org/minima/system/network/base/KeyValueStore.java create mode 100644 src/org/minima/system/network/base/NetworkConfig.java create mode 100644 src/org/minima/system/network/base/NoOpDiscoveryService.java create mode 100644 src/org/minima/system/network/base/P2PNetwork.java create mode 100644 src/org/minima/system/network/base/PortAvailability.java create mode 100644 src/org/minima/system/network/base/RpcMethod.java create mode 100644 src/org/minima/system/network/base/RpcRequestHandler.java create mode 100644 src/org/minima/system/network/base/RpcStream.java create mode 100644 src/org/minima/system/network/base/Service.java create mode 100644 src/org/minima/system/network/base/StreamClosedException.java create mode 100644 src/org/minima/system/network/base/Waiter.java create mode 100644 src/org/minima/system/network/base/WireLogsConfig.java diff --git a/src/org/minima/system/network/base/ConnectionManager.java b/src/org/minima/system/network/base/ConnectionManager.java new file mode 100644 index 000000000..1400a40b9 --- /dev/null +++ b/src/org/minima/system/network/base/ConnectionManager.java @@ -0,0 +1,235 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static java.util.stream.Collectors.toList; + +import java.time.Duration; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.minima.system.network.base.metrics.Counter; +import org.minima.system.network.base.metrics.LabelledMetric; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.metrics.TekuMetricCategory; +import org.minima.system.network.base.peer.DisconnectReason; +import org.minima.system.network.base.peer.Peer; +// // import org.hyperledger.besu.plugin.services.MetricsSystem; +// // import org.hyperledger.besu.plugin.services.metrics.Counter; +// // import org.hyperledger.besu.plugin.services.metrics.LabelledMetric; +// import tech.pegasys.teku.infrastructure.async.AsyncRunner; +// import tech.pegasys.teku.infrastructure.async.Cancellable; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; +// import tech.pegasys.teku.networking.p2p.connection.PeerPools.PeerPool; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService; +// import tech.pegasys.teku.networking.p2p.network.P2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.DisconnectReason; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.service.serviceutils.Service; +import org.minima.system.network.base.peer.PeerAddress; +import org.minima.system.network.base.peer.PeerPools; +import org.minima.system.network.base.peer.PeerSelectionStrategy; +import org.minima.system.network.base.peer.PeerPools.PeerPool; + +public class ConnectionManager extends Service { + private static final Logger LOG = LogManager.getLogger(); + private static final Duration RECONNECT_TIMEOUT = Duration.ofSeconds(20); + private static final Duration DISCOVERY_INTERVAL = Duration.ofSeconds(30); + private final AsyncRunner asyncRunner; + private final P2PNetwork network; + private final Set staticPeers; + private final DiscoveryService discoveryService; + private final PeerSelectionStrategy peerSelectionStrategy; + private final Counter attemptedConnectionCounter; + private final Counter successfulConnectionCounter; + private final Counter failedConnectionCounter; + private final PeerPools peerPools = new PeerPools(); + private final Collection> peerPredicates = new CopyOnWriteArrayList<>(); + + private volatile long peerConnectedSubscriptionId; + private volatile Cancellable periodicPeerSearch; + + public ConnectionManager( + final MetricsSystem metricsSystem, + final DiscoveryService discoveryService, + final AsyncRunner asyncRunner, + final P2PNetwork network, + final PeerSelectionStrategy peerSelectionStrategy, + final List peerAddresses) { + this.asyncRunner = asyncRunner; + this.network = network; + this.staticPeers = new HashSet<>(peerAddresses); + this.discoveryService = discoveryService; + this.peerSelectionStrategy = peerSelectionStrategy; + + final LabelledMetric connectionAttemptCounter = + metricsSystem.createLabelledCounter( + TekuMetricCategory.NETWORK, + "peer_connection_attempt_count", + "Total number of outbound connection attempts made", + "status"); + attemptedConnectionCounter = connectionAttemptCounter.labels("attempted"); + successfulConnectionCounter = connectionAttemptCounter.labels("successful"); + failedConnectionCounter = connectionAttemptCounter.labels("failed"); + } + + @Override + protected SafeFuture doStart() { + LOG.trace("Starting discovery manager"); + synchronized (this) { + staticPeers.forEach(this::createPersistentConnection); + } + periodicPeerSearch = + asyncRunner.runWithFixedDelay( + this::searchForPeers, + DISCOVERY_INTERVAL, + error -> LOG.error("Error while searching for peers", error)); + connectToKnownPeers(); + searchForPeers(); + peerConnectedSubscriptionId = network.subscribeConnect(this::onPeerConnected); + return SafeFuture.COMPLETE; + } + + private void connectToKnownPeers() { + peerSelectionStrategy + .selectPeersToConnect( + network, + peerPools, + () -> discoveryService.streamKnownPeers().filter(this::isPeerValid).collect(toList())) + .forEach(this::attemptConnection); + } + + private void searchForPeers() { + if (!isRunning()) { + LOG.trace("Not running so not searching for peers"); + return; + } + LOG.trace("Searching for peers"); + discoveryService + .searchForPeers() + .orTimeout(10, TimeUnit.SECONDS) + .finish( + this::connectToKnownPeers, + error -> { + LOG.debug("Discovery failed", error); + connectToKnownPeers(); + }); + } + + private void attemptConnection(final PeerAddress peerAddress) { + LOG.trace("Attempting to connect to {}", peerAddress.getId()); + attemptedConnectionCounter.inc(); + network + .connect(peerAddress) + .finish( + peer -> { + LOG.trace("Successfully connected to peer {}", peer.getId()); + successfulConnectionCounter.inc(); + peer.subscribeDisconnect( + (reason, locallyInitiated) -> peerPools.forgetPeer(peer.getId())); + }, + error -> { + LOG.trace(() -> "Failed to connect to peer: " + peerAddress.getId(), error); + failedConnectionCounter.inc(); + peerPools.forgetPeer(peerAddress.getId()); + }); + } + + private void onPeerConnected(final Peer peer) { + peerSelectionStrategy + .selectPeersToDisconnect(network, peerPools) + .forEach( + peerToDrop -> + peerToDrop.disconnectCleanly(DisconnectReason.TOO_MANY_PEERS).reportExceptions()); + } + + @Override + protected SafeFuture doStop() { + network.unsubscribeConnect(peerConnectedSubscriptionId); + final Cancellable peerSearchTask = this.periodicPeerSearch; + if (peerSearchTask != null) { + peerSearchTask.cancel(); + } + return SafeFuture.COMPLETE; + } + + public synchronized void addStaticPeer(final PeerAddress peerAddress) { + if (!staticPeers.contains(peerAddress)) { + staticPeers.add(peerAddress); + createPersistentConnection(peerAddress); + } + } + + private void createPersistentConnection(final PeerAddress peerAddress) { + maintainPersistentConnection(peerAddress).reportExceptions(); + } + + private SafeFuture maintainPersistentConnection(final PeerAddress peerAddress) { + if (!isRunning()) { + // We've been stopped so halt the process. + return new SafeFuture<>(); + } + LOG.debug("Connecting to peer {}", peerAddress); + peerPools.addPeerToPool(peerAddress.getId(), PeerPool.STATIC); + attemptedConnectionCounter.inc(); + return network + .connect(peerAddress) + .thenApply( + peer -> { + LOG.debug("Connection to peer {} was successful", peer.getId()); + successfulConnectionCounter.inc(); + peer.subscribeDisconnect( + (reason, locallyInitiated) -> { + LOG.debug( + "Peer {} disconnected. Will try to reconnect in {} sec", + peerAddress, + RECONNECT_TIMEOUT.toSeconds()); + asyncRunner + .runAfterDelay( + () -> maintainPersistentConnection(peerAddress), RECONNECT_TIMEOUT) + .reportExceptions(); + }); + return peer; + }) + .exceptionallyCompose( + error -> { + LOG.debug( + "Connection to {} failed: {}. Will retry in {} sec", + peerAddress, + error, + RECONNECT_TIMEOUT.toSeconds()); + failedConnectionCounter.inc(); + return asyncRunner.runAfterDelay( + () -> maintainPersistentConnection(peerAddress), RECONNECT_TIMEOUT); + }); + } + + public void addPeerPredicate(final Predicate predicate) { + peerPredicates.add(predicate); + } + + private boolean isPeerValid(DiscoveryPeer peer) { + return !peer.getNodeAddress().getAddress().isAnyLocalAddress() + && peerPredicates.stream().allMatch(predicate -> predicate.test(peer)); + } +} diff --git a/src/org/minima/system/network/base/DelegatingP2PNetwork.java b/src/org/minima/system/network/base/DelegatingP2PNetwork.java new file mode 100644 index 000000000..a1419e585 --- /dev/null +++ b/src/org/minima/system/network/base/DelegatingP2PNetwork.java @@ -0,0 +1,131 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +// import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.gossip.TopicChannel; +// import tech.pegasys.teku.networking.p2p.gossip.TopicHandler; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipTopicsScoringConfig; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; + +import org.minima.system.network.base.gossip.TopicChannel; +import org.minima.system.network.base.gossip.TopicHandler; +import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; +import org.minima.system.network.base.peer.NodeId; +import org.minima.system.network.base.peer.Peer; +import org.minima.system.network.base.peer.PeerAddress; + +public abstract class DelegatingP2PNetwork implements P2PNetwork { + private final P2PNetwork network; + + protected DelegatingP2PNetwork(final P2PNetwork network) { + this.network = network; + } + + @Override + public SafeFuture connect(final PeerAddress peer) { + return network.connect(peer); + } + + @Override + public PeerAddress createPeerAddress(final DiscoveryPeer discoveryPeer) { + return network.createPeerAddress(discoveryPeer); + } + + @Override + public NodeId parseNodeId(final String nodeId) { + return network.parseNodeId(nodeId); + } + + @Override + public boolean isConnected(final PeerAddress peerAddress) { + return network.isConnected(peerAddress); + } + + @Override + public byte[] getPrivateKey() { + return network.getPrivateKey(); + } + + @Override + public PeerAddress createPeerAddress(final String peerAddress) { + return network.createPeerAddress(peerAddress); + } + + @Override + public int getPeerCount() { + return network.getPeerCount(); + } + + @Override + public String getNodeAddress() { + return network.getNodeAddress(); + } + + @Override + public NodeId getNodeId() { + return network.getNodeId(); + } + + @Override + public Optional getEnr() { + return network.getEnr(); + } + + @Override + public Optional getDiscoveryAddress() { + return network.getDiscoveryAddress(); + } + + @Override + public int getListenPort() { + return network.getListenPort(); + } + + @Override + public SafeFuture start() { + return network.start(); + } + + @Override + public SafeFuture stop() { + return network.stop(); + } + + @Override + public SafeFuture gossip(final String topic, final byte[] data) { + return network.gossip(topic, data); + } + + @Override + public TopicChannel subscribe(final String topic, final TopicHandler topicHandler) { + return network.subscribe(topic, topicHandler); + } + + @Override + public Map> getSubscribersByTopic() { + return network.getSubscribersByTopic(); + } + + @Override + public void updateGossipTopicScoring(final GossipTopicsScoringConfig config) { + network.updateGossipTopicScoring(config); + } +} diff --git a/src/org/minima/system/network/base/DiscoveryConfig.java b/src/org/minima/system/network/base/DiscoveryConfig.java new file mode 100644 index 000000000..0acf98428 --- /dev/null +++ b/src/org/minima/system/network/base/DiscoveryConfig.java @@ -0,0 +1,123 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.Collections; +import java.util.List; + +public class DiscoveryConfig { + private final boolean isDiscoveryEnabled; + private final List staticPeers; + private final List bootnodes; + private final int minPeers; + private final int maxPeers; + private final int minRandomlySelectedPeers; + + private DiscoveryConfig( + final boolean isDiscoveryEnabled, + final List staticPeers, + final List bootnodes, + final int minPeers, + final int maxPeers, + final int minRandomlySelectedPeers) { + this.isDiscoveryEnabled = isDiscoveryEnabled; + this.staticPeers = staticPeers; + this.bootnodes = bootnodes; + this.minPeers = minPeers; + this.maxPeers = maxPeers; + this.minRandomlySelectedPeers = minRandomlySelectedPeers; + } + + public static Builder builder() { + return new Builder(); + } + + public boolean isDiscoveryEnabled() { + return isDiscoveryEnabled; + } + + public List getStaticPeers() { + return staticPeers; + } + + public List getBootnodes() { + return bootnodes; + } + + public int getMinPeers() { + return minPeers; + } + + public int getMaxPeers() { + return maxPeers; + } + + public int getMinRandomlySelectedPeers() { + return minRandomlySelectedPeers; + } + + public static class Builder { + private Boolean isDiscoveryEnabled = true; + private List staticPeers = Collections.emptyList(); + private List bootnodes = Collections.emptyList(); + private int minPeers = 64; + private int maxPeers = 74; + private int minRandomlySelectedPeers = 2; + + private Builder() {} + + public Builder isDiscoveryEnabled(final Boolean discoveryEnabled) { + checkNotNull(discoveryEnabled); + isDiscoveryEnabled = discoveryEnabled; + return this; + } + + public DiscoveryConfig build() { + return new DiscoveryConfig( + isDiscoveryEnabled, staticPeers, bootnodes, minPeers, maxPeers, minRandomlySelectedPeers); + } + + public Builder staticPeers(final List staticPeers) { + checkNotNull(staticPeers); + this.staticPeers = staticPeers; + return this; + } + + public Builder bootnodes(final List bootnodes) { + checkNotNull(bootnodes); + this.bootnodes = bootnodes; + return this; + } + + public Builder minPeers(final Integer minPeers) { + checkNotNull(minPeers); + this.minPeers = minPeers; + return this; + } + + public Builder maxPeers(final Integer maxPeers) { + checkNotNull(maxPeers); + this.maxPeers = maxPeers; + return this; + } + + public Builder minRandomlySelectedPeers(final Integer minRandomlySelectedPeers) { + checkNotNull(minRandomlySelectedPeers); + this.minRandomlySelectedPeers = minRandomlySelectedPeers; + return this; + } + } +} diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java new file mode 100644 index 000000000..935eb33aa --- /dev/null +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -0,0 +1,232 @@ + +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static java.util.stream.Collectors.toList; +//import static tech.pegasys.teku.util.config.Constants.ATTESTATION_SUBNET_COUNT; + +import java.util.Optional; +import java.util.stream.Stream; + +import com.google.common.io.ByteSink; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.NodeId; +//import org.apache.logging.log4j.status.StatusLogger; +import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes32; +// import org.hyperledger.besu.plugin.services.MetricsSystem; +// import tech.pegasys.teku.infrastructure.async.AsyncRunner; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.logging.StatusLogger; +// import tech.pegasys.teku.infrastructure.unsigned.UInt64; +// import tech.pegasys.teku.networking.p2p.connection.ConnectionManager; +// import tech.pegasys.teku.networking.p2p.connection.PeerSelectionStrategy; +// import tech.pegasys.teku.networking.p2p.discovery.discv5.DiscV5Service; +// import tech.pegasys.teku.networking.p2p.discovery.noop.NoOpDiscoveryService; +// import tech.pegasys.teku.networking.p2p.network.DelegatingP2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.P2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.config.NetworkConfig; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; +// import tech.pegasys.teku.ssz.schema.collections.SszBitvectorSchema; +// import tech.pegasys.teku.ssz.type.Bytes4; +import org.minima.system.network.base.peer.Peer; +import org.minima.system.network.base.peer.PeerConnectedSubscriber; +import org.minima.system.network.base.peer.PeerSelectionStrategy; +import org.minima.system.network.base.ssz.SszBitvectorSchema; + +// import tech.pegasys.teku.storage.store.KeyValueStore; + +public class DiscoveryNetwork

extends DelegatingP2PNetwork

{ + private static final Logger LOG = LogManager.getLogger(); + + public static final String ATTESTATION_SUBNET_ENR_FIELD = "attnets"; + public static final String ETH2_ENR_FIELD = "eth2"; + + // from tech.pegasys.teku.util.config.Constants.ATTESTATION_SUBNET_COUNT + public static final int ATTESTATION_SUBNET_COUNT = 64; + + private final P2PNetwork

p2pNetwork; + private final DiscoveryService discoveryService; + private final ConnectionManager connectionManager; + + private volatile Optional enrForkId = Optional.empty(); + + DiscoveryNetwork( + final P2PNetwork

p2pNetwork, + final DiscoveryService discoveryService, + final ConnectionManager connectionManager) { + super(p2pNetwork); + this.p2pNetwork = p2pNetwork; + this.discoveryService = discoveryService; + this.connectionManager = connectionManager; + initialize(); + } + + public void initialize() { + //TODO: set ENR data set by setpregenesisforkinfo + //setPreGenesisForkInfo(); + //TODO: log this in another way + // getEnr().ifPresent(StatusLogger.STATUS_LOG::listeningForDiscv5PreGenesis); + + // Set connection manager peer predicate so that we don't attempt to connect peers with + // different fork digests +// connectionManager.addPeerPredicate(this::dontConnectPeersWithDifferentForkDigests); + } + + public static

DiscoveryNetwork

create( + final MetricsSystem metricsSystem, + final AsyncRunner asyncRunner, + final KeyValueStore kvStore, + final P2PNetwork

p2pNetwork, + final PeerSelectionStrategy peerSelectionStrategy, + final DiscoveryConfig discoveryConfig, + final NetworkConfig p2pConfig + ) { + final DiscoveryService discoveryService = + createDiscoveryService(discoveryConfig, p2pConfig, kvStore, Bytes.wrap(p2pNetwork.getPrivateKey())); + final ConnectionManager connectionManager = + new ConnectionManager( + metricsSystem, + discoveryService, + asyncRunner, + p2pNetwork, + peerSelectionStrategy, + discoveryConfig.getStaticPeers().stream() + .map(p2pNetwork::createPeerAddress) + .collect(toList())); + return new DiscoveryNetwork<>(p2pNetwork, discoveryService, connectionManager); + } + + private static DiscoveryService createDiscoveryService( + final DiscoveryConfig discoConfig, + final NetworkConfig p2pConfig, + final KeyValueStore kvStore, + final Bytes privateKey) { + final DiscoveryService discoveryService; + if (discoConfig.isDiscoveryEnabled()) { + discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); + } else { + discoveryService = new NoOpDiscoveryService(); + } + return discoveryService; + } + + + @Override + public SafeFuture start() { + return SafeFuture.allOfFailFast(p2pNetwork.start(), discoveryService.start()) + .thenCompose(__ -> connectionManager.start()) + .thenRun(() -> getEnr()); // TODO: log ENR info and discovery start + } // ifPresent(StatusLogger.STATUS_LOG::listeningForDiscv5) + + @Override + public SafeFuture stop() { + return connectionManager + .stop() + .handleComposed( + (__, err) -> { + if (err != null) { + LOG.warn("Error shutting down connection manager", err); + } + return SafeFuture.allOf(p2pNetwork.stop(), discoveryService.stop()); + }); + } + + public void addStaticPeer(final String peerAddress) { + connectionManager.addStaticPeer(p2pNetwork.createPeerAddress(peerAddress)); + } + + @Override + public Optional getEnr() { + return discoveryService.getEnr(); + } + + @Override + public Optional getDiscoveryAddress() { + return discoveryService.getDiscoveryAddress(); + } + + public void setLongTermAttestationSubnetSubscriptions(Iterable subnetIds) { + discoveryService.updateCustomENRField( + ATTESTATION_SUBNET_ENR_FIELD, + SszBitvectorSchema.create(ATTESTATION_SUBNET_COUNT).ofBits(subnetIds).sszSerialize()); + } + + // public void setPreGenesisForkInfo() { + // final Bytes4 genesisForkVersion = spec.getGenesisSpecConfig().getGenesisForkVersion(); + // final EnrForkId enrForkId = + // new EnrForkId( + // spec.getGenesisBeaconStateUtil().computeForkDigest(genesisForkVersion, Bytes32.ZERO), + // genesisForkVersion, + // SpecConfig.FAR_FUTURE_EPOCH); + // discoveryService.updateCustomENRField(ETH2_ENR_FIELD, enrForkId.sszSerialize()); + // this.enrForkId = Optional.of(enrForkId); + // } + + // public void setForkInfo(final ForkInfo currentForkInfo, final Optional nextForkInfo) { + // // If no future fork is planned, set next_fork_version = current_fork_version to signal this + // final Bytes4 nextVersion = + // nextForkInfo + // .map(Fork::getCurrent_version) + // .orElse(currentForkInfo.getFork().getCurrent_version()); + // // If no future fork is planned, set next_fork_epoch = FAR_FUTURE_EPOCH to signal this + // final UInt64 nextForkEpoch = + // nextForkInfo.map(Fork::getEpoch).orElse(SpecConfig.FAR_FUTURE_EPOCH); + + // final Bytes4 forkDigest = currentForkInfo.getForkDigest(); + // final EnrForkId enrForkId = new EnrForkId(forkDigest, nextVersion, nextForkEpoch); + // final Bytes encodedEnrForkId = enrForkId.sszSerialize(); + + // discoveryService.updateCustomENRField(ETH2_ENR_FIELD, encodedEnrForkId); + // this.enrForkId = Optional.of(enrForkId); + // } + + // private boolean dontConnectPeersWithDifferentForkDigests(DiscoveryPeer peer) { + // return enrForkId + // .map(EnrForkId::getForkDigest) + // .flatMap( + // localForkDigest -> + // peer.getEnrForkId() + // .map(EnrForkId::getForkDigest) + // .map(peerForkDigest -> peerForkDigest.equals(localForkDigest))) + // .orElse(false); + // } + + @Override + public long subscribeConnect(final PeerConnectedSubscriber

subscriber) { + return p2pNetwork.subscribeConnect(subscriber); + } + + @Override + public void unsubscribeConnect(final long subscriptionId) { + p2pNetwork.unsubscribeConnect(subscriptionId); + } + + @Override + public Optional

getPeer(final NodeId id) { + return p2pNetwork.getPeer(id); + } + + @Override + public Stream

streamPeers() { + return p2pNetwork.streamPeers(); + } +} + diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java new file mode 100644 index 000000000..ceabc4d7d --- /dev/null +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -0,0 +1,143 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.net.BindException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +// import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; +// import tech.pegasys.teku.infrastructure.async.DelayedExecutorAsyncRunner; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.async.Waiter; +// import tech.pegasys.teku.infrastructure.time.StubTimeProvider; +// import tech.pegasys.teku.network.p2p.jvmlibp2p.PrivateKeyGenerator; +// import tech.pegasys.teku.network.p2p.peer.SimplePeerSelectionStrategy; +// import tech.pegasys.teku.networking.p2p.connection.PeerSelectionStrategy; +// import tech.pegasys.teku.networking.p2p.connection.TargetPeerRange; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryConfig; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryNetwork; +// import tech.pegasys.teku.networking.p2p.libp2p.LibP2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.config.NetworkConfig; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.reputation.ReputationManager; +// import tech.pegasys.teku.spec.Spec; +// import tech.pegasys.teku.spec.SpecFactory; +// import tech.pegasys.teku.storage.store.MemKeyValueStore; +// import tech.pegasys.teku.util.config.Constants; +import org.minima.system.network.base.metrics.NoOpMetricsSystem; +import org.minima.system.network.base.peer.Peer; +import org.minima.system.network.base.peer.PeerSelectionStrategy; + + +public class DiscoveryNetworkFactory { + + protected static final Logger LOG = LogManager.getLogger(); + protected static final NoOpMetricsSystem METRICS_SYSTEM = new NoOpMetricsSystem(); + private static final int MIN_PORT = 9000; + private static final int MAX_PORT = 12000; + + private final List> networks = new ArrayList<>(); + + public DiscoveryNetworkBuilder builder() { + return new DiscoveryNetworkBuilder(); + } + + public void stopAll() throws InterruptedException, ExecutionException, TimeoutException { + Waiter.waitFor( + SafeFuture.allOf(networks.stream().map(DiscoveryNetwork::stop).toArray(SafeFuture[]::new))); + } + + public class DiscoveryNetworkBuilder { + private final List staticPeers = new ArrayList<>(); + private final List bootnodes = new ArrayList<>(); + + private DiscoveryNetworkBuilder() {} + + public DiscoveryNetworkBuilder staticPeer(final String staticPeer) { + this.staticPeers.add(staticPeer); + return this; + } + + public DiscoveryNetworkBuilder bootnode(final String bootnode) { + this.bootnodes.add(bootnode); + return this; + } + + public DiscoveryNetwork buildAndStart() throws Exception { + int attempt = 1; + while (true) { + + final Random random = new Random(); + final int port = MIN_PORT + random.nextInt(MAX_PORT - MIN_PORT); + final DiscoveryConfig discoveryConfig = + DiscoveryConfig.builder().staticPeers(staticPeers).bootnodes(bootnodes).build(); + final NetworkConfig config = + NetworkConfig.builder().listenPort(port).networkInterface("127.0.0.1").build(); + final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); + final ReputationManager reputationManager = + new ReputationManager( + metricsSystem, + StubTimeProvider.withTimeInSeconds(1000), + Constants.REPUTATION_MANAGER_CAPACITY); + final PeerSelectionStrategy peerSelectionStrategy = + new SimplePeerSelectionStrategy(new TargetPeerRange(20, 30, 0)); + final DiscoveryNetwork network = + DiscoveryNetwork.create( + metricsSystem, + DelayedExecutorAsyncRunner.create(), + new MemKeyValueStore<>(), + new LibP2PNetwork( + DelayedExecutorAsyncRunner.create(), + config, + PrivateKeyGenerator::generate, + reputationManager, + METRICS_SYSTEM, + Collections.emptyList(), + Collections.emptyList(), + (__1, __2) -> { + throw new UnsupportedOperationException(); + }, + topic -> true), + peerSelectionStrategy, + discoveryConfig, + config); + try { + network.start().get(30, TimeUnit.SECONDS); + networks.add(network); + return network; + } catch (final ExecutionException e) { + if (e.getCause() instanceof BindException) { + if (attempt > 10) { + throw new RuntimeException("Failed to find a free port after multiple attempts", e); + } + LOG.info( + "Port conflict detected, retrying with a new port. Original message: {}", + e.getMessage()); + attempt++; + Waiter.waitFor(network.stop()); + } else { + throw e; + } + } + } + } + } +} diff --git a/src/org/minima/system/network/base/DiscoveryPeer.java b/src/org/minima/system/network/base/DiscoveryPeer.java new file mode 100644 index 000000000..3a64ae598 --- /dev/null +++ b/src/org/minima/system/network/base/DiscoveryPeer.java @@ -0,0 +1,86 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; + +import org.minima.system.network.base.ssz.SszBitvector; + +import java.net.InetSocketAddress; +import java.util.Optional; +// import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.EnrForkId; +// import tech.pegasys.teku.ssz.collections.SszBitvector; + +public class DiscoveryPeer { + private final byte[] publicKey; + private final InetSocketAddress nodeAddress; + private final Optional enrForkId; + private final SszBitvector persistentSubnets; + + public DiscoveryPeer( + final byte[] publicKey, + final InetSocketAddress nodeAddress, + final Optional enrForkId, + final SszBitvector persistentSubnets) { + this.publicKey = publicKey; + this.nodeAddress = nodeAddress; + this.enrForkId = enrForkId; + this.persistentSubnets = persistentSubnets; + } + + public byte[] getPublicKey() { + return publicKey; + } + + public InetSocketAddress getNodeAddress() { + return nodeAddress; + } + + public Optional getEnrForkId() { + return enrForkId; + } + + public SszBitvector getPersistentSubnets() { + return persistentSubnets; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DiscoveryPeer)) return false; + DiscoveryPeer that = (DiscoveryPeer) o; + return Objects.equal(getPublicKey(), that.getPublicKey()) + && Objects.equal(getNodeAddress(), that.getNodeAddress()) + && Objects.equal(getEnrForkId(), that.getEnrForkId()) + && Objects.equal(getPersistentSubnets(), that.getPersistentSubnets()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getPublicKey(), getNodeAddress(), getEnrForkId(), getPersistentSubnets()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("publicKey", publicKey) + .add("nodeAddress", nodeAddress) + .add("enrForkId", enrForkId) + .add("persistentSubnets", persistentSubnets) + .toString(); + } +} diff --git a/src/org/minima/system/network/base/DiscoveryService.java b/src/org/minima/system/network/base/DiscoveryService.java new file mode 100644 index 000000000..9f87e7f0f --- /dev/null +++ b/src/org/minima/system/network/base/DiscoveryService.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Optional; +import java.util.stream.Stream; +import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +public interface DiscoveryService { + + SafeFuture start(); + + SafeFuture stop(); + + Stream streamKnownPeers(); + + SafeFuture searchForPeers(); + + Optional getEnr(); + + Optional getDiscoveryAddress(); + + void updateCustomENRField(String fieldName, Bytes value); +} diff --git a/src/org/minima/system/network/base/EnrForkId.java b/src/org/minima/system/network/base/EnrForkId.java new file mode 100644 index 000000000..1ee5fd77e --- /dev/null +++ b/src/org/minima/system/network/base/EnrForkId.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import javax.swing.tree.TreeNode; + +import org.minima.system.network.base.ssz.Bytes4; +import org.minima.system.network.base.ssz.SszBytes4; +import org.minima.system.network.base.ssz.SszPrimitiveSchemas; +import org.minima.system.network.base.ssz.SszUInt64; +import org.minima.system.network.base.ssz.UInt64; + +// import tech.pegasys.teku.infrastructure.unsigned.UInt64; +// import tech.pegasys.teku.ssz.containers.Container3; +// import tech.pegasys.teku.ssz.containers.ContainerSchema3; +// import tech.pegasys.teku.ssz.primitive.SszBytes4; +// import tech.pegasys.teku.ssz.primitive.SszUInt64; +// import tech.pegasys.teku.ssz.schema.SszPrimitiveSchemas; +// import tech.pegasys.teku.ssz.tree.TreeNode; +// import tech.pegasys.teku.ssz.type.Bytes4; + +public class EnrForkId extends Container3 { + + public static class EnrForkIdSchema + extends ContainerSchema3 { + + public EnrForkIdSchema() { + super( + "EnrForkId", + namedSchema("forkDigest", SszPrimitiveSchemas.BYTES4_SCHEMA), + namedSchema("nextForkVersion", SszPrimitiveSchemas.BYTES4_SCHEMA), + namedSchema("nextForkEpoch", SszPrimitiveSchemas.UINT64_SCHEMA)); + } + + @Override + public EnrForkId createFromBackingNode(TreeNode node) { + return new EnrForkId(this, node); + } + } + + public static final EnrForkIdSchema SSZ_SCHEMA = new EnrForkIdSchema(); + + private EnrForkId(EnrForkIdSchema type, TreeNode backingNode) { + super(type, backingNode); + } + + public EnrForkId( + final Bytes4 forkDigest, final Bytes4 nextForkVersion, final UInt64 nextForkEpoch) { + super( + SSZ_SCHEMA, + SszBytes4.of(forkDigest), + SszBytes4.of(nextForkVersion), + SszUInt64.of(nextForkEpoch)); + } + + public Bytes4 getForkDigest() { + return getField0().get(); + } + + public Bytes4 getNextForkVersion() { + return getField1().get(); + } + + public UInt64 getNextForkEpoch() { + return getField2().get(); + } +} diff --git a/src/org/minima/system/network/base/InvalidConfigurationException.java b/src/org/minima/system/network/base/InvalidConfigurationException.java new file mode 100644 index 000000000..b239e9db5 --- /dev/null +++ b/src/org/minima/system/network/base/InvalidConfigurationException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public class InvalidConfigurationException extends RuntimeException { + public InvalidConfigurationException(final String message) { + super(message); + } + + public InvalidConfigurationException(final String message, final Throwable cause) { + super(message, cause); + } + + public InvalidConfigurationException(final Throwable cause) { + super(cause.getMessage(), cause); + } +} diff --git a/src/org/minima/system/network/base/KeyValueStore.java b/src/org/minima/system/network/base/KeyValueStore.java new file mode 100644 index 000000000..95d07ca62 --- /dev/null +++ b/src/org/minima/system/network/base/KeyValueStore.java @@ -0,0 +1,98 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Optional; +import org.jetbrains.annotations.NotNull; + +/** + * Generic simple key-value store interface Both key and value are not allowed to be null + * + * @param key type + * @param value type + */ +public interface KeyValueStore { + + /** Puts a new value. If the value is {@code null} then the entry is removed if exist */ + void put(@NotNull TKey key, @NotNull TValue value); + + /** Removes entry with the specified key */ + void remove(@NotNull TKey key); + + /** + * Returns a value corresponding to the key or {{@link Optional#empty()}} if entry doesn't exist + * in the store + */ + Optional get(@NotNull TKey key); + + /** + * Performs batch store update. + * + *

The implementation may override this default method and declare it to be an atomic store + * update. Though this generic interface makes no restrictions on atomicity of this method + */ + default void updateAll(Iterable> data) { + data.forEach( + update -> { + if (update.getType() == UpdateType.UPDATE) { + put(update.getKey(), update.getValue()); + } else if (update.getType() == UpdateType.REMOVE) { + remove(update.getKey()); + } else { + throw new IllegalArgumentException("Unknown type: " + update.getType()); + } + }); + } + + enum UpdateType { + UPDATE, + REMOVE + } + + /** Represents a batched update entry */ + class EntryUpdate { + private final UpdateType type; + private final K key; + private final V value; + + public static EntryUpdate update(K key, V value) { + return new EntryUpdate<>(UpdateType.UPDATE, key, value); + } + + public static EntryUpdate remove(K key) { + return new EntryUpdate<>(UpdateType.REMOVE, key, null); + } + + private EntryUpdate(UpdateType type, K key, V value) { + this.type = type; + this.key = key; + this.value = value; + } + + public UpdateType getType() { + return type; + } + + public K getKey() { + return key; + } + + public V getValue() { + if (getType() != UpdateType.UPDATE) { + throw new IllegalStateException("No value for this update"); + } + return value; + } + } +} diff --git a/src/org/minima/system/network/base/NetworkConfig.java b/src/org/minima/system/network/base/NetworkConfig.java new file mode 100644 index 000000000..45a584636 --- /dev/null +++ b/src/org/minima/system/network/base/NetworkConfig.java @@ -0,0 +1,209 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.net.InetAddresses.isInetAddress; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +// import tech.pegasys.teku.infrastructure.io.PortAvailability; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipConfig; +// import tech.pegasys.teku.util.config.InvalidConfigurationException; +import org.minima.system.network.base.gossip.config.GossipConfig; + +public class NetworkConfig { + private static final Logger LOG = LogManager.getLogger(); + + private final GossipConfig gossipConfig; + private final WireLogsConfig wireLogsConfig; + + private final boolean isEnabled; + private final Optional privateKeyFile; + private final String networkInterface; + private final Optional advertisedIp; + private final int listenPort; + private final OptionalInt advertisedPort; + + private NetworkConfig( + final boolean isEnabled, + final GossipConfig gossipConfig, + final WireLogsConfig wireLogsConfig, + final Optional privateKeyFile, + final String networkInterface, + final Optional advertisedIp, + final int listenPort, + final OptionalInt advertisedPort) { + + this.privateKeyFile = privateKeyFile; + this.networkInterface = networkInterface; + + this.advertisedIp = advertisedIp.filter(ip -> !ip.isBlank()); + this.isEnabled = isEnabled; + if (this.advertisedIp.map(ip -> !isInetAddress(ip)).orElse(false)) { + throw new InvalidConfigurationException( + String.format( + "Advertised ip (%s) is set incorrectly.", this.advertisedIp.orElse("EMPTY"))); + } + + this.listenPort = listenPort; + this.advertisedPort = advertisedPort; + this.gossipConfig = gossipConfig; + this.wireLogsConfig = wireLogsConfig; + } + + public static Builder builder() { + return new Builder(); + } + + public void validateListenPortAvailable() { + if (listenPort != 0 && !PortAvailability.isPortAvailable(listenPort)) { + throw new InvalidConfigurationException( + String.format( + "P2P Port %d (TCP/UDP) is already in use. " + + "Check for other processes using this port.", + listenPort)); + } + } + + public boolean isEnabled() { + return isEnabled; + } + + public Optional getPrivateKeyFile() { + return privateKeyFile; + } + + public String getNetworkInterface() { + return networkInterface; + } + + public String getAdvertisedIp() { + return resolveAnyLocalAddress(advertisedIp.orElse(networkInterface)); + } + + public boolean hasUserExplicitlySetAdvertisedIp() { + return advertisedIp.isPresent(); + } + + public int getListenPort() { + return listenPort; + } + + public int getAdvertisedPort() { + return advertisedPort.orElse(listenPort); + } + + public GossipConfig getGossipConfig() { + return gossipConfig; + } + + public WireLogsConfig getWireLogsConfig() { + return wireLogsConfig; + } + + private String resolveAnyLocalAddress(final String ipAddress) { + try { + final InetAddress advertisedAddress = InetAddress.getByName(ipAddress); + if (advertisedAddress.isAnyLocalAddress()) { + return InetAddress.getLocalHost().getHostAddress(); + } else { + return ipAddress; + } + } catch (UnknownHostException err) { + LOG.error( + "Unable to start LibP2PNetwork due to failed attempt at obtaining host address", err); + return ipAddress; + } + } + + public static class Builder { + public static final int DEFAULT_P2P_PORT = 9000; + + private final GossipConfig.Builder gossipConfigBuilder = GossipConfig.builder(); + private final WireLogsConfig.Builder wireLogsConfig = WireLogsConfig.builder(); + + private Boolean isEnabled = true; + private Optional privateKeyFile = Optional.empty(); + private String networkInterface = "0.0.0.0"; + private Optional advertisedIp = Optional.empty(); + private Integer listenPort = DEFAULT_P2P_PORT; + private OptionalInt advertisedPort = OptionalInt.empty(); + + private Builder() {} + + public NetworkConfig build() { + return new NetworkConfig( + isEnabled, + gossipConfigBuilder.build(), + wireLogsConfig.build(), + privateKeyFile, + networkInterface, + advertisedIp, + listenPort, + advertisedPort); + } + + public Builder isEnabled(final Boolean enabled) { + checkNotNull(enabled); + isEnabled = enabled; + return this; + } + + public Builder gossipConfig(final Consumer consumer) { + consumer.accept(gossipConfigBuilder); + return this; + } + + public Builder wireLogs(final Consumer consumer) { + consumer.accept(wireLogsConfig); + return this; + } + + public Builder privateKeyFile(final String privateKeyFile) { + checkNotNull(privateKeyFile); + this.privateKeyFile = Optional.of(privateKeyFile).filter(f -> !f.isBlank()); + return this; + } + + public Builder networkInterface(final String networkInterface) { + checkNotNull(networkInterface); + this.networkInterface = networkInterface; + return this; + } + + public Builder advertisedIp(final Optional advertisedIp) { + checkNotNull(advertisedIp); + this.advertisedIp = advertisedIp; + return this; + } + + public Builder listenPort(final Integer listenPort) { + checkNotNull(listenPort); + this.listenPort = listenPort; + return this; + } + + public Builder advertisedPort(final OptionalInt advertisedPort) { + checkNotNull(advertisedPort); + this.advertisedPort = advertisedPort; + return this; + } + } +} diff --git a/src/org/minima/system/network/base/NoOpDiscoveryService.java b/src/org/minima/system/network/base/NoOpDiscoveryService.java new file mode 100644 index 000000000..828c91f50 --- /dev/null +++ b/src/org/minima/system/network/base/NoOpDiscoveryService.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Optional; +import java.util.stream.Stream; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService; + +public class NoOpDiscoveryService implements DiscoveryService { + + @Override + public SafeFuture start() { + return SafeFuture.COMPLETE; + } + + @Override + public SafeFuture stop() { + return SafeFuture.COMPLETE; + } + + @Override + public Stream streamKnownPeers() { + return Stream.empty(); + } + + @Override + public SafeFuture searchForPeers() { + return SafeFuture.COMPLETE; + } + + @Override + public Optional getEnr() { + return Optional.empty(); + } + + @Override + public Optional getDiscoveryAddress() { + return Optional.empty(); + } + + @Override + public void updateCustomENRField(String fieldName, Bytes value) {} +} diff --git a/src/org/minima/system/network/base/P2PNetwork.java b/src/org/minima/system/network/base/P2PNetwork.java new file mode 100644 index 000000000..e1640ab97 --- /dev/null +++ b/src/org/minima/system/network/base/P2PNetwork.java @@ -0,0 +1,115 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Optional; +import java.util.stream.Stream; +// import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.gossip.GossipNetwork; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; + +import org.minima.system.network.base.DiscoveryPeer; +import org.minima.system.network.base.SafeFuture; +import org.minima.system.network.base.gossip.GossipNetwork; +import org.minima.system.network.base.peer.NodeId; +import org.minima.system.network.base.peer.Peer; +import org.minima.system.network.base.peer.PeerAddress; +import org.minima.system.network.base.peer.PeerConnectedSubscriber; + +public interface P2PNetwork extends GossipNetwork { + + enum State { + IDLE, + RUNNING, + STOPPED + } + + /** + * Connects to a Peer using a user supplied address. The address format is specific to the network + * implementation. If a connection already exists for this peer, the future completes with the + * existing peer. + * + *

The {@link PeerAddress} must have been created using the {@link #createPeerAddress(String)} + * method of this same implementation. + * + * @param peer Peer to connect to. + * @return A future which completes when the connection is established, containing the newly + * connected peer. + */ + SafeFuture connect(PeerAddress peer); + + /** + * Parses a peer address in any of this network's supported formats. + * + * @param peerAddress the address to parse + * @return a {@link PeerAddress} which is supported by {@link #connect(PeerAddress)} for + * initiating connections + */ + PeerAddress createPeerAddress(String peerAddress); + + /** + * Converts a {@link DiscoveryPeer} to a {@link PeerAddress} which can be used with this network's + * {@link #connect(PeerAddress)} method. + * + * @param discoveryPeer the discovery peer to convert + * @return a {@link PeerAddress} which is supported by {@link #connect(PeerAddress)} for + * initiating connections + */ + PeerAddress createPeerAddress(DiscoveryPeer discoveryPeer); + + long subscribeConnect(PeerConnectedSubscriber subscriber); + + void unsubscribeConnect(long subscriptionId); + + boolean isConnected(PeerAddress peerAddress); + + byte[] getPrivateKey(); + + Optional getPeer(NodeId id); + + Stream streamPeers(); + + NodeId parseNodeId(final String nodeId); + + int getPeerCount(); + + String getNodeAddress(); + + NodeId getNodeId(); + + int getListenPort(); + + /** + * Get the Ethereum Node Record (ENR) for the local node, if one exists. + * + * @return the local ENR. + */ + Optional getEnr(); + + Optional getDiscoveryAddress(); + + /** + * Starts the P2P network layer. + * + * @return + */ + SafeFuture start(); + + /** Stops the P2P network layer. */ + SafeFuture stop(); +} diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 20af5525c..e89f47d74 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -24,6 +24,7 @@ // Import log4j classes. import org.apache.logging.log4j.Logger; +import org.minima.system.network.base.libp2p.gossip.LibP2PGossipNetwork; import org.apache.logging.log4j.LogManager; public class P2PStart { @@ -34,54 +35,9 @@ public class P2PStart { public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); - - - // Create a libp2p node and configure it - // to accept TCP connections on a random port (0) - // Host node = new HostBuilder() - // .protocol(new Ping()) - // .listen("/ip4/127.0.0.1/tcp/0") - // .build(); - - Host node = new HostBuilder() - .protocol(new P2PMinimaDiscovery()) - .listen("/ip4/127.0.0.1/tcp/0") - .build(); - - // start listening - node.start().get(); - - System.out.print("Node started and listening for P2P Minima Discovery Protocol on "); - System.out.println(node.listenAddresses()); - - if (args.length > 0) { - System.out.println("Found args: " + args[0]); - Multiaddr address = Multiaddr.fromString(args[0]); - P2PMinimaDiscovery disc = new P2PMinimaDiscovery(); - System.out.println("Created disc, looking up controller..."); - P2PMinimaDiscoveryProtocolController discoverer = disc.dial( - node, - address - ).getController().get(); - - // P2PMinimaDiscoveryProtocolController discoverer = new P2PMinimaDiscovery().dial( - // node, - // address - // ).getController().get(); - - System.out.println("Sending a message to the world..."); - discoverer.send("Hello world!"); - - node.stop().get(); - // System.out.println("Sending 5 ping messages to " + address.toString()); - // for (int i = 1; i <= 5; ++i) { - // long latency = pinger.ping().get(); - // System.out.println("Ping " + i + ", latency " + latency + "ms"); - // } - - // node.stop().get(); + // LibP2PGossipNetwork gossipNet = new LibP2PGossipNetwork(new MetricsSystem()); + } - } } diff --git a/src/org/minima/system/network/base/PortAvailability.java b/src/org/minima/system/network/base/PortAvailability.java new file mode 100644 index 000000000..e915c5c69 --- /dev/null +++ b/src/org/minima/system/network/base/PortAvailability.java @@ -0,0 +1,61 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.io.IOException; +import java.net.DatagramSocket; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class PortAvailability { + private static final Logger LOG = LogManager.getLogger(); + + public static boolean isPortAvailableForTcp(final int port) { + if (!isPortValid(port)) { + return false; + } + try (final ServerSocket serverSocket = new ServerSocket()) { + serverSocket.setReuseAddress(true); + serverSocket.bind(new InetSocketAddress(port)); + return true; + } catch (IOException ex) { + LOG.trace(String.format("failed to open port %d for TCP", port), ex); + } + return false; + } + + public static boolean isPortAvailableForUdp(final int port) { + if (!isPortValid(port)) { + return false; + } + try (final DatagramSocket datagramSocket = new DatagramSocket(null)) { + datagramSocket.setReuseAddress(true); + datagramSocket.bind(new InetSocketAddress(port)); + return true; + } catch (IOException ex) { + LOG.trace(String.format("failed to open port %d for UDP", port), ex); + } + return false; + } + + public static boolean isPortValid(final int port) { + return (port >= 0 && port <= 65535); + } + + public static boolean isPortAvailable(final int port) { + return isPortAvailableForTcp(port) && isPortAvailableForUdp(port); + } +} diff --git a/src/org/minima/system/network/base/RpcMethod.java b/src/org/minima/system/network/base/RpcMethod.java new file mode 100644 index 000000000..970039958 --- /dev/null +++ b/src/org/minima/system/network/base/RpcMethod.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.networking.p2p.rpc; + +public interface RpcMethod { + + String getId(); + + RpcRequestHandler createIncomingRequestHandler(); +} diff --git a/src/org/minima/system/network/base/RpcRequestHandler.java b/src/org/minima/system/network/base/RpcRequestHandler.java new file mode 100644 index 000000000..925ab7b2a --- /dev/null +++ b/src/org/minima/system/network/base/RpcRequestHandler.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import org.minima.system.network.base.peer.NodeId; + +import io.netty.buffer.ByteBuf; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public interface RpcRequestHandler { + + void active(final NodeId nodeId, final RpcStream rpcStream); + + void processData(final NodeId nodeId, final RpcStream rpcStream, final ByteBuf data); + + void readComplete(final NodeId nodeId, final RpcStream rpcStream); + + void closed(final NodeId nodeId, final RpcStream rpcStream); +} diff --git a/src/org/minima/system/network/base/RpcStream.java b/src/org/minima/system/network/base/RpcStream.java new file mode 100644 index 000000000..2113f9f3b --- /dev/null +++ b/src/org/minima/system/network/base/RpcStream.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +//import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +public interface RpcStream { + + SafeFuture writeBytes(byte[] bytes) throws StreamClosedException; + + /** + * Close the stream altogether, allowing no further reads or writes. + * + * @return A future completing when the stream is closed. + */ + SafeFuture closeAbruptly(); + + /** + * Close the write side of the stream. When both sides of the stream close their write stream, the + * entire stream will be closed. + * + * @return A future completing when the write stream is closed. + */ + SafeFuture closeWriteStream(); +} diff --git a/src/org/minima/system/network/base/Service.java b/src/org/minima/system/network/base/Service.java new file mode 100644 index 000000000..4f3791d97 --- /dev/null +++ b/src/org/minima/system/network/base/Service.java @@ -0,0 +1,52 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.concurrent.atomic.AtomicReference; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +public abstract class Service { + enum State { + IDLE, + RUNNING, + STOPPED + } + + private final AtomicReference state = new AtomicReference<>(State.IDLE); + + public SafeFuture start() { + if (!state.compareAndSet(State.IDLE, State.RUNNING)) { + return SafeFuture.failedFuture( + new IllegalStateException("Attempt to start an already started service.")); + } + return doStart(); + } + + protected abstract SafeFuture doStart(); + + public SafeFuture stop() { + if (state.compareAndSet(State.RUNNING, State.STOPPED)) { + return doStop(); + } else { + // Return a successful future if there's nothing to do at this point + return SafeFuture.COMPLETE; + } + } + + protected abstract SafeFuture doStop(); + + public boolean isRunning() { + return state.get() == State.RUNNING; + } +} diff --git a/src/org/minima/system/network/base/StreamClosedException.java b/src/org/minima/system/network/base/StreamClosedException.java new file mode 100644 index 000000000..1f14ed432 --- /dev/null +++ b/src/org/minima/system/network/base/StreamClosedException.java @@ -0,0 +1,16 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +public class StreamClosedException extends RuntimeException {} diff --git a/src/org/minima/system/network/base/Waiter.java b/src/org/minima/system/network/base/Waiter.java new file mode 100644 index 000000000..38f6a36e1 --- /dev/null +++ b/src/org/minima/system/network/base/Waiter.java @@ -0,0 +1,84 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.time.Duration; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.awaitility.Awaitility; +import org.awaitility.pollinterval.IterativePollInterval; + +/** + * A simpler wrapper around Awaitility that directs people towards best practices for waiting. The + * native Awaitility wrapper has a number of "gotchas" that can lead to intermittency which this + * wrapper aims to prevent. + */ +public class Waiter { + + private static final int DEFAULT_TIMEOUT_SECONDS = 30; + private static final Duration INITIAL_POLL_INTERVAL = Duration.ofMillis(200); + private static final Duration MAX_POLL_INTERVAL = Duration.ofSeconds(5); + + public static void waitFor( + final Condition assertion, final int timeoutValue, final TimeUnit timeUnit) { + Awaitility.waitAtMost(timeoutValue, timeUnit) + .ignoreExceptions() + .pollInterval( + IterativePollInterval.iterative(Waiter::nextPollInterval, INITIAL_POLL_INTERVAL)) + .untilAsserted(assertion::run); + } + + public static void waitFor(final Condition assertion) { + waitFor(assertion, DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + private static Duration nextPollInterval(final Duration duration) { + final Duration nextInterval = duration.multipliedBy(2); + return nextInterval.compareTo(MAX_POLL_INTERVAL) <= 0 ? nextInterval : MAX_POLL_INTERVAL; + } + + public static T waitFor(final Future future) + throws InterruptedException, ExecutionException, TimeoutException { + return future.get(DEFAULT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + public static T waitFor(final Future future, final Duration duration) + throws InterruptedException, ExecutionException, TimeoutException { + return future.get(duration.toSeconds(), TimeUnit.SECONDS); + } + + public interface Condition { + void run() throws Throwable; + } + + public static void ensureConditionRemainsMet( + final Condition condition, int waitTimeInMilliseconds) throws InterruptedException { + final long mustBeTrueUntil = System.currentTimeMillis() + waitTimeInMilliseconds; + while (System.currentTimeMillis() < mustBeTrueUntil) { + try { + condition.run(); + } catch (final Throwable t) { + throw new RuntimeException("Condition did not remain met", t); + } + Thread.sleep(500); + } + } + + public static void ensureConditionRemainsMet(final Condition condition) + throws InterruptedException { + ensureConditionRemainsMet(condition, 2000); + } +} diff --git a/src/org/minima/system/network/base/WireLogsConfig.java b/src/org/minima/system/network/base/WireLogsConfig.java new file mode 100644 index 000000000..dfcc67bf4 --- /dev/null +++ b/src/org/minima/system/network/base/WireLogsConfig.java @@ -0,0 +1,95 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static com.google.common.base.Preconditions.checkNotNull; + +public class WireLogsConfig { + private final boolean logWireCipher; + private final boolean logWirePlain; + private final boolean logWireMuxFrames; + private final boolean logWireGossip; + + private WireLogsConfig( + boolean logWireCipher, + boolean logWirePlain, + boolean logWireMuxFrames, + boolean logWireGossip) { + this.logWireCipher = logWireCipher; + this.logWirePlain = logWirePlain; + this.logWireMuxFrames = logWireMuxFrames; + this.logWireGossip = logWireGossip; + } + + public static Builder builder() { + return new Builder(); + } + + public static WireLogsConfig createDefault() { + return builder().build(); + } + + public boolean isLogWireCipher() { + return logWireCipher; + } + + public boolean isLogWirePlain() { + return logWirePlain; + } + + public boolean isLogWireMuxFrames() { + return logWireMuxFrames; + } + + public boolean isLogWireGossip() { + return logWireGossip; + } + + public static class Builder { + private Boolean logWireCipher = false; + private Boolean logWirePlain = false; + private Boolean logWireMuxFrames = false; + private Boolean logWireGossip = false; + + private Builder() {} + + public WireLogsConfig build() { + return new WireLogsConfig(logWireCipher, logWirePlain, logWireMuxFrames, logWireGossip); + } + + public Builder logWireCipher(final Boolean logWireCipher) { + checkNotNull(logWireCipher); + this.logWireCipher = logWireCipher; + return this; + } + + public Builder logWirePlain(final Boolean logWirePlain) { + checkNotNull(logWirePlain); + this.logWirePlain = logWirePlain; + return this; + } + + public Builder logWireMuxFrames(final Boolean logWireMuxFrames) { + checkNotNull(logWireMuxFrames); + this.logWireMuxFrames = logWireMuxFrames; + return this; + } + + public Builder logWireGossip(final Boolean logWireGossip) { + checkNotNull(logWireGossip); + this.logWireGossip = logWireGossip; + return this; + } + } +} From d585772c6e3a02b05daeabb5ef8a80c73bc7a163 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 8 Apr 2021 17:08:22 +0100 Subject: [PATCH 09/55] p2p: added teku dependencies jar to enable libp2p peer hosts discovery. --- build.gradle | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d1905e999..c3ca8071d 100644 --- a/build.gradle +++ b/build.gradle @@ -54,7 +54,7 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0-M1") - implementation 'io.libp2p:jvm-libp2p-minimal:0.8.0-RELEASE' + implementation 'io.libp2p:jvm-libp2p-minimal:0.8.0-RELEASE' // implementation 'io.libp2p:jvm-libp2p-minimal' // compile files('libs/jvm-libp2p-minimal-0.8.0-RELEASE.jar') @@ -63,6 +63,16 @@ dependencies { implementation("org.apache.logging.log4j:log4j-api:2.11.2") implementation("org.apache.logging.log4j:log4j-core:2.11.2") + // Bytes object needed for collections such as KVS + implementation("org.apache.tuweni:bytes:1.3.0") + // commons-lang3 is needed for Pair<,> in ssz + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' + // not sure we need the below, as we imported teku fork of tuweni ssz... But maybe we could switch to it. + implementation group: 'org.apache.tuweni', name: 'tuweni-ssz', version: '1.1.0', ext: 'pom' + // awaitility is used by Waiter.java + implementation group: 'org.awaitility', name: 'awaitility', version: '4.0.3' + + } sourceSets { From b738490a438a56124b63fe62ca58ba8b8635e444 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Fri, 9 Apr 2021 10:19:29 +0100 Subject: [PATCH 10/55] p2p: added most/all classes needed. --- .../base/DelayedExecutorAsyncRunner.java | 82 +++++ .../network/base/DiscoveryNetworkFactory.java | 10 +- .../minima/system/network/base/EnrForkId.java | 5 +- .../minima/system/network/base/LRUCache.java | 101 ++++++ .../system/network/base/LibP2PNetwork.java | 321 ++++++++++++++++++ .../system/network/base/LibP2PNodeId.java | 6 +- .../system/network/base/MemKeyValueStore.java | 40 +++ .../system/network/base/P2PNetwork.java | 2 - .../minima/system/network/base/RpcMethod.java | 4 +- .../system/network/base/StubTimeProvider.java | 57 ++++ .../system/network/base/TimeProvider.java | 29 ++ .../network/base/peer/DiscoveryPeer.java | 87 +++++ .../network/base/peer/ExceptionUtil.java | 54 +++ .../network/base/peer/LibP2PNodeId.java | 36 ++ .../system/network/base/peer/LibP2PPeer.java | 163 +++++++++ .../network/base/peer/LibP2PRpcStream.java | 87 +++++ .../base/peer/MultiaddrPeerAddress.java | 80 +++++ .../network/base/peer/MultiaddrUtil.java | 69 ++++ .../system/network/base/peer/NodeId.java | 4 +- .../minima/system/network/base/peer/Peer.java | 10 +- .../peer/PeerAlreadyConnectedException.java | 37 ++ .../base/peer/PeerDisconnectedException.java | 23 ++ .../system/network/base/peer/PeerHandler.java | 22 ++ .../system/network/base/peer/PeerManager.java | 164 +++++++++ .../network/base/peer/ReputationManager.java | 166 +++++++++ .../system/network/base/peer/RpcHandler.java | 229 +++++++++++++ .../system/network/base/peer/RpcMethod.java | 21 ++ .../base/{ => peer}/RpcRequestHandler.java | 4 +- .../system/network/base/peer/RpcStream.java | 38 +++ .../peer/SimplePeerSelectionStrategy.java | 68 ++++ .../base/peer/StreamClosedException.java | 16 + .../base/peer/StreamTimeoutException.java | 21 ++ .../system/network/base/peer/Subscribers.java | 128 +++++++ .../network/base/peer/TargetPeerRange.java | 56 +++ 34 files changed, 2222 insertions(+), 18 deletions(-) create mode 100644 src/org/minima/system/network/base/DelayedExecutorAsyncRunner.java create mode 100644 src/org/minima/system/network/base/LRUCache.java create mode 100644 src/org/minima/system/network/base/LibP2PNetwork.java create mode 100644 src/org/minima/system/network/base/MemKeyValueStore.java create mode 100644 src/org/minima/system/network/base/StubTimeProvider.java create mode 100644 src/org/minima/system/network/base/TimeProvider.java create mode 100644 src/org/minima/system/network/base/peer/DiscoveryPeer.java create mode 100644 src/org/minima/system/network/base/peer/ExceptionUtil.java create mode 100644 src/org/minima/system/network/base/peer/LibP2PNodeId.java create mode 100644 src/org/minima/system/network/base/peer/LibP2PPeer.java create mode 100644 src/org/minima/system/network/base/peer/LibP2PRpcStream.java create mode 100644 src/org/minima/system/network/base/peer/MultiaddrPeerAddress.java create mode 100644 src/org/minima/system/network/base/peer/MultiaddrUtil.java create mode 100644 src/org/minima/system/network/base/peer/PeerAlreadyConnectedException.java create mode 100644 src/org/minima/system/network/base/peer/PeerDisconnectedException.java create mode 100644 src/org/minima/system/network/base/peer/PeerHandler.java create mode 100644 src/org/minima/system/network/base/peer/PeerManager.java create mode 100644 src/org/minima/system/network/base/peer/ReputationManager.java create mode 100644 src/org/minima/system/network/base/peer/RpcHandler.java create mode 100644 src/org/minima/system/network/base/peer/RpcMethod.java rename src/org/minima/system/network/base/{ => peer}/RpcRequestHandler.java (91%) create mode 100644 src/org/minima/system/network/base/peer/RpcStream.java create mode 100644 src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java create mode 100644 src/org/minima/system/network/base/peer/StreamClosedException.java create mode 100644 src/org/minima/system/network/base/peer/StreamTimeoutException.java create mode 100644 src/org/minima/system/network/base/peer/Subscribers.java create mode 100644 src/org/minima/system/network/base/peer/TargetPeerRange.java diff --git a/src/org/minima/system/network/base/DelayedExecutorAsyncRunner.java b/src/org/minima/system/network/base/DelayedExecutorAsyncRunner.java new file mode 100644 index 000000000..eff7cf3b3 --- /dev/null +++ b/src/org/minima/system/network/base/DelayedExecutorAsyncRunner.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import com.google.common.annotations.VisibleForTesting; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * An AsyncRunner that uses the common ForkJoinPool so that it is guaranteed not to leak threads + * even if the test doesn't shut it down. + */ +public class DelayedExecutorAsyncRunner implements AsyncRunner { + private static final Logger LOG = LogManager.getLogger(); + private final ExecutorFactory executorFactory; + + private DelayedExecutorAsyncRunner(ExecutorFactory executorFactory) { + this.executorFactory = executorFactory; + } + + public static DelayedExecutorAsyncRunner create() { + return new DelayedExecutorAsyncRunner(CompletableFuture::delayedExecutor); + } + + @Override + public SafeFuture runAsync(final ExceptionThrowingFutureSupplier action) { + final Executor executor = getAsyncExecutor(); + return runAsync(action, executor); + } + + @Override + public SafeFuture runAfterDelay( + ExceptionThrowingFutureSupplier action, Duration delay) { + final Executor executor = getDelayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); + return runAsync(action, executor); + } + + @Override + public void shutdown() {} + + @VisibleForTesting + SafeFuture runAsync( + final ExceptionThrowingFutureSupplier action, final Executor executor) { + final SafeFuture result = new SafeFuture<>(); + try { + executor.execute(() -> SafeFuture.of(action).propagateTo(result)); + } catch (final RejectedExecutionException ex) { + LOG.debug("shutting down ", ex); + } catch (final Throwable t) { + result.completeExceptionally(t); + } + return result; + } + + private Executor getAsyncExecutor() { + return getDelayedExecutor(-1, TimeUnit.SECONDS); + } + + private Executor getDelayedExecutor(long delayAmount, TimeUnit delayUnit) { + return executorFactory.create(delayAmount, delayUnit); + } + + private interface ExecutorFactory { + Executor create(long delayAmount, TimeUnit delayUnit); + } +} diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index ceabc4d7d..14b37b20b 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -45,7 +45,9 @@ import org.minima.system.network.base.metrics.NoOpMetricsSystem; import org.minima.system.network.base.peer.Peer; import org.minima.system.network.base.peer.PeerSelectionStrategy; - +import org.minima.system.network.base.peer.ReputationManager; +import org.minima.system.network.base.peer.SimplePeerSelectionStrategy; +import org.minima.system.network.base.peer.TargetPeerRange; public class DiscoveryNetworkFactory { @@ -56,6 +58,12 @@ public class DiscoveryNetworkFactory { private final List> networks = new ArrayList<>(); + // from tech.pegasys.teku.util.config.Constants + public final class Constants { + public static final int REPUTATION_MANAGER_CAPACITY = 1024; + public static final int ATTESTATION_SUBNET_COUNT = 64; + } + public DiscoveryNetworkBuilder builder() { return new DiscoveryNetworkBuilder(); } diff --git a/src/org/minima/system/network/base/EnrForkId.java b/src/org/minima/system/network/base/EnrForkId.java index 1ee5fd77e..08bf7670f 100644 --- a/src/org/minima/system/network/base/EnrForkId.java +++ b/src/org/minima/system/network/base/EnrForkId.java @@ -13,13 +13,14 @@ package org.minima.system.network.base; -import javax.swing.tree.TreeNode; - import org.minima.system.network.base.ssz.Bytes4; +import org.minima.system.network.base.ssz.Container3; +import org.minima.system.network.base.ssz.ContainerSchema3; import org.minima.system.network.base.ssz.SszBytes4; import org.minima.system.network.base.ssz.SszPrimitiveSchemas; import org.minima.system.network.base.ssz.SszUInt64; import org.minima.system.network.base.ssz.UInt64; +import org.minima.system.network.base.ssz.TreeNode; // import tech.pegasys.teku.infrastructure.unsigned.UInt64; // import tech.pegasys.teku.ssz.containers.Container3; diff --git a/src/org/minima/system/network/base/LRUCache.java b/src/org/minima/system/network/base/LRUCache.java new file mode 100644 index 000000000..34da71967 --- /dev/null +++ b/src/org/minima/system/network/base/LRUCache.java @@ -0,0 +1,101 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +//import tech.pegasys.teku.infrastructure.collections.LimitedMap; + +import org.minima.system.network.base.ssz.Cache; + +/** + * Cache made around LRU-map with fixed size, removing eldest entries (by added) when the space is + * over + * + * @param Keys type + * @param Values type + */ +public class LRUCache implements Cache { + + private final Map cacheData; + private final int maxCapacity; + + /** + * Creates cache + * + * @param capacity Size of the cache + */ + public LRUCache(int capacity) { + this(capacity, Collections.emptyMap()); + } + + private LRUCache(int capacity, Map initialCachedContent) { + this.maxCapacity = capacity; + Map cacheMap = LimitedMap.create(maxCapacity); + // copy safely, initialCachedContent is always a SynchronizedMap instance + synchronized (initialCachedContent) { + cacheMap.putAll(initialCachedContent); + } + this.cacheData = cacheMap; + } + + @Override + public Cache copy() { + return new LRUCache<>(maxCapacity, cacheData); + } + + /** + * Queries value from the cache. If it's not found there, fallback function is used to calculate + * value. After calculation result is put in cache and returned. + * + * @param key Key to query + * @param fallback Fallback function for calculation of the result in case of missed cache entry + * @return expected value result for provided key + */ + @Override + public V get(K key, Function fallback) { + V result = cacheData.get(key); + + if (result == null) { + result = fallback.apply(key); + if (result != null) { + cacheData.put(key, result); + } + } + + return result; + } + + @Override + public Optional getCached(K key) { + return Optional.ofNullable(cacheData.get(key)); + } + + @Override + public void invalidate(K key) { + cacheData.remove(key); + } + + @Override + public void clear() { + cacheData.clear(); + } + + @Override + public int size() { + return cacheData.size(); + } +} diff --git a/src/org/minima/system/network/base/LibP2PNetwork.java b/src/org/minima/system/network/base/LibP2PNetwork.java new file mode 100644 index 000000000..b3912e293 --- /dev/null +++ b/src/org/minima/system/network/base/LibP2PNetwork.java @@ -0,0 +1,321 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static org.minima.system.network.base.SafeFuture.failedFuture; +//import static tech.pegasys.teku.infrastructure.logging.StatusLogger.STATUS_LOG; + +import identify.pb.IdentifyOuterClass; +import io.libp2p.core.Host; +import io.libp2p.core.PeerId; +import io.libp2p.core.crypto.PrivKey; +import io.libp2p.core.dsl.Builder.Defaults; +import io.libp2p.core.dsl.BuilderJKt; +import io.libp2p.core.multiformats.Multiaddr; +import io.libp2p.core.multistream.ProtocolBinding; +import io.libp2p.core.mux.StreamMuxerProtocol; +import io.libp2p.etc.types.ByteArrayExtKt; +import io.libp2p.etc.util.P2PService.PeerHandler; +import io.libp2p.protocol.Identify; +import io.libp2p.protocol.Ping; +import io.libp2p.security.noise.NoiseXXSecureChannel; +import io.libp2p.transport.tcp.TcpTransport; +import io.netty.handler.logging.LogLevel; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.minima.system.network.base.gossip.PreparedGossipMessageFactory; +import org.minima.system.network.base.libp2p.gossip.GossipTopicFilter; +import org.minima.system.network.base.libp2p.gossip.LibP2PGossipNetwork; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.MultiaddrPeerAddress; +import org.minima.system.network.base.peer.MultiaddrUtil; +import org.minima.system.network.base.peer.NodeId; +//import org.hyperledger.besu.plugin.services.MetricsSystem; +// import tech.pegasys.teku.infrastructure.async.AsyncRunner; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.version.VersionProvider; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.gossip.PreparedGossipMessageFactory; +// import tech.pegasys.teku.networking.p2p.gossip.TopicChannel; +// import tech.pegasys.teku.networking.p2p.gossip.TopicHandler; +// import tech.pegasys.teku.networking.p2p.gossip.config.GossipTopicsScoringConfig; +// import tech.pegasys.teku.networking.p2p.libp2p.gossip.GossipTopicFilter; +// import tech.pegasys.teku.networking.p2p.libp2p.gossip.LibP2PGossipNetwork; +// import tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler; +// import tech.pegasys.teku.networking.p2p.network.P2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.network.PeerHandler; +// import tech.pegasys.teku.networking.p2p.network.config.NetworkConfig; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; +// import tech.pegasys.teku.networking.p2p.reputation.ReputationManager; +// import tech.pegasys.teku.networking.p2p.rpc.RpcMethod; +import org.minima.system.network.base.peer.Peer; +import org.minima.system.network.base.peer.PeerAddress; +import org.minima.system.network.base.peer.PeerConnectedSubscriber; +import org.minima.system.network.base.peer.PeerManager; +import org.minima.system.network.base.peer.ReputationManager; +import org.minima.system.network.base.peer.RpcHandler; + +public class LibP2PNetwork implements P2PNetwork { + + private static final Logger LOG = LogManager.getLogger(); + private static final int REMOTE_OPEN_STREAMS_RATE_LIMIT = 256; + private static final int REMOTE_PARALLEL_OPEN_STREAMS_COUNT_LIMIT = 256; + + private final PrivKey privKey; + private final NodeId nodeId; + + private final Host host; + private final PeerManager peerManager; + private final Multiaddr advertisedAddr; + private final LibP2PGossipNetwork gossipNetwork; + + private final AtomicReference state = new AtomicReference<>(State.IDLE); + private final Map rpcHandlers = new ConcurrentHashMap<>(); + private final int listenPort; + + public LibP2PNetwork( + final AsyncRunner asyncRunner, + final NetworkConfig config, + final PrivateKeyProvider privateKeyProvider, + final ReputationManager reputationManager, + final MetricsSystem metricsSystem, + final List rpcMethods, + final List peerHandlers, + final PreparedGossipMessageFactory defaultMessageFactory, + final GossipTopicFilter gossipTopicFilter) { + this.privKey = privateKeyProvider.get(); + this.nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); + + advertisedAddr = + MultiaddrUtil.fromInetSocketAddress( + new InetSocketAddress(config.getAdvertisedIp(), config.getAdvertisedPort()), nodeId); + this.listenPort = config.getListenPort(); + + // Setup gossip + gossipNetwork = + LibP2PGossipNetwork.create( + metricsSystem, + config.getGossipConfig(), + defaultMessageFactory, + gossipTopicFilter, + config.getWireLogsConfig().isLogWireGossip()); + + // Setup rpc methods + rpcMethods.forEach(method -> rpcHandlers.put(method, new RpcHandler(asyncRunner, method))); + + // Setup peers + peerManager = new PeerManager(metricsSystem, reputationManager, peerHandlers, rpcHandlers); + + final Multiaddr listenAddr = + MultiaddrUtil.fromInetSocketAddress( + new InetSocketAddress(config.getNetworkInterface(), config.getListenPort())); + host = + BuilderJKt.hostJ( + Defaults.None, + b -> { + b.getIdentity().setFactory(() -> privKey); + b.getTransports().add(TcpTransport::new); + b.getSecureChannels().add(NoiseXXSecureChannel::new); + b.getMuxers().add(StreamMuxerProtocol.getMplex()); + + b.getNetwork().listen(listenAddr.toString()); + + b.getProtocols().addAll(getDefaultProtocols()); + b.getProtocols().addAll(rpcHandlers.values()); + + if (config.getWireLogsConfig().isLogWireCipher()) { + b.getDebug().getBeforeSecureHandler().addLogger(LogLevel.DEBUG, "wire.ciphered"); + } + Firewall firewall = new Firewall(Duration.ofSeconds(30)); + b.getDebug().getBeforeSecureHandler().addNettyHandler(firewall); + + if (config.getWireLogsConfig().isLogWirePlain()) { + b.getDebug().getAfterSecureHandler().addLogger(LogLevel.DEBUG, "wire.plain"); + } + if (config.getWireLogsConfig().isLogWireMuxFrames()) { + b.getDebug().getMuxFramesHandler().addLogger(LogLevel.DEBUG, "wire.mux"); + } + + b.getConnectionHandlers().add(peerManager); + + MplexFirewall mplexFirewall = + new MplexFirewall( + REMOTE_OPEN_STREAMS_RATE_LIMIT, REMOTE_PARALLEL_OPEN_STREAMS_COUNT_LIMIT); + b.getDebug().getMuxFramesHandler().addHandler(mplexFirewall); + }); + } + + private List> getDefaultProtocols() { + final Ping ping = new Ping(); + IdentifyOuterClass.Identify identifyMsg = + IdentifyOuterClass.Identify.newBuilder() + .setProtocolVersion("ipfs/0.1.0") + .setAgentVersion(VersionProvider.CLIENT_IDENTITY + "/" + VersionProvider.VERSION) + .setPublicKey(ByteArrayExtKt.toProtobuf(privKey.publicKey().bytes())) + .addListenAddrs(ByteArrayExtKt.toProtobuf(advertisedAddr.getBytes())) + .setObservedAddr(ByteArrayExtKt.toProtobuf(advertisedAddr.getBytes())) + .addAllProtocols(ping.getProtocolDescriptor().getAnnounceProtocols()) + .addAllProtocols( + gossipNetwork.getGossip().getProtocolDescriptor().getAnnounceProtocols()) + .build(); + return List.of(ping, new Identify(identifyMsg), gossipNetwork.getGossip()); + } + + @Override + public SafeFuture start() { + if (!state.compareAndSet(State.IDLE, State.RUNNING)) { + return SafeFuture.failedFuture(new IllegalStateException("Network already started")); + } + LOG.info("Starting libp2p network..."); + return SafeFuture.of(host.start()) + .thenApply( + i -> { + STATUS_LOG.listeningForLibP2P(getNodeAddress()); + return null; + }); + } + + @Override + public String getNodeAddress() { + return advertisedAddr.toString(); + } + + @Override + public SafeFuture connect(final PeerAddress peer) { + return peer.as(MultiaddrPeerAddress.class) + .map(staticPeer -> peerManager.connect(staticPeer, host.getNetwork())) + .orElseGet( + () -> + failedFuture( + new IllegalArgumentException( + "Unsupported peer address: " + peer.getClass().getName()))); + } + + @Override + public PeerAddress createPeerAddress(final String peerAddress) { + return MultiaddrPeerAddress.fromAddress(peerAddress); + } + + @Override + public PeerAddress createPeerAddress(final DiscoveryPeer discoveryPeer) { + return MultiaddrPeerAddress.fromDiscoveryPeer(discoveryPeer); + } + + @Override + public long subscribeConnect(final PeerConnectedSubscriber subscriber) { + return peerManager.subscribeConnect(subscriber); + } + + @Override + public void unsubscribeConnect(final long subscriptionId) { + peerManager.unsubscribeConnect(subscriptionId); + } + + @Override + public boolean isConnected(final PeerAddress peerAddress) { + return peerManager.getPeer(peerAddress.getId()).isPresent(); + } + + @Override + public Bytes getPrivateKey() { + return Bytes.wrap(privKey.raw()); + } + + @Override + public Optional getPeer(final NodeId id) { + return peerManager.getPeer(id); + } + + @Override + public Stream streamPeers() { + return peerManager.streamPeers(); + } + + @Override + public NodeId parseNodeId(final String nodeId) { + return new LibP2PNodeId(PeerId.fromBase58(nodeId)); + } + + @Override + public int getPeerCount() { + return peerManager.getPeerCount(); + } + + @Override + public int getListenPort() { + return listenPort; + } + + @Override + public SafeFuture stop() { + if (!state.compareAndSet(State.RUNNING, State.STOPPED)) { + return SafeFuture.COMPLETE; + } + LOG.debug("JvmLibP2PNetwork.stop()"); + return SafeFuture.of(host.stop()); + } + + @Override + public NodeId getNodeId() { + return nodeId; + } + + @Override + public Optional getEnr() { + return Optional.empty(); + } + + @Override + public Optional getDiscoveryAddress() { + return Optional.empty(); + } + + @Override + public SafeFuture gossip(final String topic, final Bytes data) { + return gossipNetwork.gossip(topic, data); + } + + @Override + public TopicChannel subscribe(final String topic, final TopicHandler topicHandler) { + return gossipNetwork.subscribe(topic, topicHandler); + } + + @Override + public Map> getSubscribersByTopic() { + return gossipNetwork.getSubscribersByTopic(); + } + + @Override + public void updateGossipTopicScoring(final GossipTopicsScoringConfig config) { + gossipNetwork.updateGossipTopicScoring(config); + } + + @FunctionalInterface + public interface PrivateKeyProvider { + PrivKey get(); + } +} diff --git a/src/org/minima/system/network/base/LibP2PNodeId.java b/src/org/minima/system/network/base/LibP2PNodeId.java index 5c52446a6..44093cfc7 100644 --- a/src/org/minima/system/network/base/LibP2PNodeId.java +++ b/src/org/minima/system/network/base/LibP2PNodeId.java @@ -14,7 +14,7 @@ package org.minima.system.network.base; import io.libp2p.core.PeerId; -//import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes; //import tech.pegasys.teku.networking.p2p.peer.NodeId; import org.minima.system.network.base.peer.NodeId; @@ -26,8 +26,8 @@ public LibP2PNodeId(final PeerId peerId) { } @Override - public byte[] toBytes() { - return peerId.getBytes(); + public Bytes toBytes() { + return Bytes.wrap(peerId.getBytes()); } @Override diff --git a/src/org/minima/system/network/base/MemKeyValueStore.java b/src/org/minima/system/network/base/MemKeyValueStore.java new file mode 100644 index 000000000..8eb21af27 --- /dev/null +++ b/src/org/minima/system/network/base/MemKeyValueStore.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import org.jetbrains.annotations.NotNull; + +/** Simple {@link java.util.HashMap} backed {@link KeyValueStore} implementation */ +public class MemKeyValueStore implements KeyValueStore { + + private final Map store = new ConcurrentHashMap<>(); + + @Override + public void put(@NotNull K key, @NotNull V value) { + store.put(key, value); + } + + @Override + public void remove(@NotNull K key) { + store.remove(key); + } + + @Override + public Optional get(@NotNull K key) { + return Optional.ofNullable(store.get(key)); + } +} diff --git a/src/org/minima/system/network/base/P2PNetwork.java b/src/org/minima/system/network/base/P2PNetwork.java index e1640ab97..cb02cbe88 100644 --- a/src/org/minima/system/network/base/P2PNetwork.java +++ b/src/org/minima/system/network/base/P2PNetwork.java @@ -23,8 +23,6 @@ // import tech.pegasys.teku.networking.p2p.peer.Peer; // import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; -import org.minima.system.network.base.DiscoveryPeer; -import org.minima.system.network.base.SafeFuture; import org.minima.system.network.base.gossip.GossipNetwork; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; diff --git a/src/org/minima/system/network/base/RpcMethod.java b/src/org/minima/system/network/base/RpcMethod.java index 970039958..3e30f57a4 100644 --- a/src/org/minima/system/network/base/RpcMethod.java +++ b/src/org/minima/system/network/base/RpcMethod.java @@ -11,7 +11,9 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.networking.p2p.rpc; +package org.minima.system.network.base; + +import org.minima.system.network.base.peer.RpcRequestHandler; public interface RpcMethod { diff --git a/src/org/minima/system/network/base/StubTimeProvider.java b/src/org/minima/system/network/base/StubTimeProvider.java new file mode 100644 index 000000000..c40ba1e56 --- /dev/null +++ b/src/org/minima/system/network/base/StubTimeProvider.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import java.util.concurrent.TimeUnit; +//import tech.pegasys.teku.infrastructure.unsigned.UInt64; + +import org.minima.system.network.base.ssz.UInt64; + +public class StubTimeProvider implements TimeProvider { + + private UInt64 timeInMillis; + + private StubTimeProvider(final UInt64 timeInMillis) { + this.timeInMillis = timeInMillis; + } + + public static StubTimeProvider withTimeInSeconds(final long timeInSeconds) { + return withTimeInSeconds(UInt64.valueOf(timeInSeconds)); + } + + public static StubTimeProvider withTimeInSeconds(final UInt64 timeInSeconds) { + return withTimeInMillis(timeInSeconds.times(MILLIS_PER_SECOND)); + } + + public static StubTimeProvider withTimeInMillis(final long timeInMillis) { + return withTimeInMillis(UInt64.valueOf(timeInMillis)); + } + + public static StubTimeProvider withTimeInMillis(final UInt64 timeInMillis) { + return new StubTimeProvider(timeInMillis); + } + + public void advanceTimeBySeconds(final long seconds) { + advanceTimeByMillis(TimeUnit.SECONDS.toMillis(seconds)); + } + + public void advanceTimeByMillis(final long millis) { + this.timeInMillis = timeInMillis.plus(millis); + } + + @Override + public UInt64 getTimeInMillis() { + return timeInMillis; + } +} diff --git a/src/org/minima/system/network/base/TimeProvider.java b/src/org/minima/system/network/base/TimeProvider.java new file mode 100644 index 000000000..78f16456d --- /dev/null +++ b/src/org/minima/system/network/base/TimeProvider.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import org.minima.system.network.base.ssz.UInt64; + +//import tech.pegasys.teku.infrastructure.unsigned.UInt64; + +public interface TimeProvider { + + UInt64 MILLIS_PER_SECOND = UInt64.valueOf(1000); + + UInt64 getTimeInMillis(); + + default UInt64 getTimeInSeconds() { + return getTimeInMillis().dividedBy(MILLIS_PER_SECOND); + } +} diff --git a/src/org/minima/system/network/base/peer/DiscoveryPeer.java b/src/org/minima/system/network/base/peer/DiscoveryPeer.java new file mode 100644 index 000000000..aedfc4322 --- /dev/null +++ b/src/org/minima/system/network/base/peer/DiscoveryPeer.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Objects; + +import org.minima.system.network.base.EnrForkId; +import org.minima.system.network.base.ssz.SszBitvector; + +import java.net.InetSocketAddress; +import java.util.Optional; +// import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.EnrForkId; +// import tech.pegasys.teku.ssz.collections.SszBitvector; + +public class DiscoveryPeer { + private final byte[] publicKey; + private final InetSocketAddress nodeAddress; + private final Optional enrForkId; + private final SszBitvector persistentSubnets; + + public DiscoveryPeer( + final byte[] publicKey, + final InetSocketAddress nodeAddress, + final Optional enrForkId, + final SszBitvector persistentSubnets) { + this.publicKey = publicKey; + this.nodeAddress = nodeAddress; + this.enrForkId = enrForkId; + this.persistentSubnets = persistentSubnets; + } + + public byte[] getPublicKey() { + return publicKey; + } + + public InetSocketAddress getNodeAddress() { + return nodeAddress; + } + + public Optional getEnrForkId() { + return enrForkId; + } + + public SszBitvector getPersistentSubnets() { + return persistentSubnets; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof DiscoveryPeer)) return false; + DiscoveryPeer that = (DiscoveryPeer) o; + return Objects.equal(getPublicKey(), that.getPublicKey()) + && Objects.equal(getNodeAddress(), that.getNodeAddress()) + && Objects.equal(getEnrForkId(), that.getEnrForkId()) + && Objects.equal(getPersistentSubnets(), that.getPersistentSubnets()); + } + + @Override + public int hashCode() { + return Objects.hashCode( + getPublicKey(), getNodeAddress(), getEnrForkId(), getPersistentSubnets()); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("publicKey", publicKey) + .add("nodeAddress", nodeAddress) + .add("enrForkId", enrForkId) + .add("persistentSubnets", persistentSubnets) + .toString(); + } +} diff --git a/src/org/minima/system/network/base/peer/ExceptionUtil.java b/src/org/minima/system/network/base/peer/ExceptionUtil.java new file mode 100644 index 000000000..f0430e6dd --- /dev/null +++ b/src/org/minima/system/network/base/peer/ExceptionUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Optional; +import java.util.function.Consumer; +import org.apache.commons.lang3.exception.ExceptionUtils; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; +import org.minima.system.network.base.SafeFuture; + +public class ExceptionUtil { + + @SuppressWarnings("unchecked") + public static Optional getCause( + final Throwable err, Class targetType) { + return ExceptionUtils.getThrowableList(err).stream() + .filter(targetType::isInstance) + .map(e -> (T) e) + .findFirst(); + } + + public static Runnable exceptionHandlingRunnable( + final Runnable runnable, final SafeFuture target) { + return () -> { + try { + runnable.run(); + } catch (final Throwable t) { + target.completeExceptionally(t); + } + }; + } + + public static Consumer exceptionHandlingConsumer( + final Consumer consumer, final SafeFuture target) { + return value -> { + try { + consumer.accept(value); + } catch (final Throwable t) { + target.completeExceptionally(t); + } + }; + } +} diff --git a/src/org/minima/system/network/base/peer/LibP2PNodeId.java b/src/org/minima/system/network/base/peer/LibP2PNodeId.java new file mode 100644 index 000000000..8caab3c74 --- /dev/null +++ b/src/org/minima/system/network/base/peer/LibP2PNodeId.java @@ -0,0 +1,36 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import io.libp2p.core.PeerId; +import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public class LibP2PNodeId extends NodeId { + private final PeerId peerId; + + public LibP2PNodeId(final PeerId peerId) { + this.peerId = peerId; + } + + @Override + public Bytes toBytes() { + return Bytes.wrap(peerId.getBytes()); + } + + @Override + public String toBase58() { + return peerId.toBase58(); + } +} diff --git a/src/org/minima/system/network/base/peer/LibP2PPeer.java b/src/org/minima/system/network/base/peer/LibP2PPeer.java new file mode 100644 index 000000000..f27b4e7a7 --- /dev/null +++ b/src/org/minima/system/network/base/peer/LibP2PPeer.java @@ -0,0 +1,163 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import io.libp2p.core.Connection; +import io.libp2p.core.PeerId; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; + +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.DisconnectReason; +// import tech.pegasys.teku.networking.p2p.peer.DisconnectRequestHandler; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.peer.PeerDisconnectedSubscriber; +// import tech.pegasys.teku.networking.p2p.reputation.ReputationAdjustment; +// import tech.pegasys.teku.networking.p2p.reputation.ReputationManager; +// import tech.pegasys.teku.networking.p2p.rpc.RpcMethod; +// import tech.pegasys.teku.networking.p2p.rpc.RpcRequestHandler; +// import tech.pegasys.teku.networking.p2p.rpc.RpcStream; +import org.minima.system.network.base.SafeFuture; + +import org.minima.system.network.base.peer.RpcMethod; +import org.minima.system.network.base.peer.RpcHandler; +import org.minima.system.network.base.peer.RpcRequestHandler; +import org.minima.system.network.base.peer.RpcStream; + +public class LibP2PPeer implements Peer { + private static final Logger LOG = LogManager.getLogger(); + + private final Map rpcHandlers; + private final ReputationManager reputationManager; + private final Connection connection; + private final AtomicBoolean connected = new AtomicBoolean(true); + private final MultiaddrPeerAddress peerAddress; + + private volatile Optional disconnectReason = Optional.empty(); + private volatile boolean disconnectLocallyInitiated = false; + private volatile DisconnectRequestHandler disconnectRequestHandler = + reason -> { + disconnectImmediately(Optional.of(reason), true); + return SafeFuture.COMPLETE; + }; + + public LibP2PPeer( + final Connection connection, + final Map rpcHandlers, + final ReputationManager reputationManager) { + this.connection = connection; + this.rpcHandlers = rpcHandlers; + this.reputationManager = reputationManager; + + final PeerId peerId = connection.secureSession().getRemoteId(); + final NodeId nodeId = new LibP2PNodeId(peerId); + peerAddress = new MultiaddrPeerAddress(nodeId, connection.remoteAddress()); + SafeFuture.of(connection.closeFuture()) + .finish( + this::handleConnectionClosed, + error -> + LOG.trace( + "Peer {} connection close future completed exceptionally", peerId, error)); + } + + @Override + public PeerAddress getAddress() { + return peerAddress; + } + + @Override + public boolean isConnected() { + return connected.get(); + } + + @Override + public void disconnectImmediately( + final Optional reason, final boolean locallyInitiated) { + connected.set(false); + disconnectReason = reason; + disconnectLocallyInitiated = locallyInitiated; + SafeFuture.of(connection.close()) + .finish( + () -> LOG.trace("Disconnected from {}", getId()), + error -> LOG.warn("Failed to disconnect from peer {}", getId(), error)); + } + + @Override + public SafeFuture disconnectCleanly(final DisconnectReason reason) { + connected.set(false); + disconnectReason = Optional.of(reason); + disconnectLocallyInitiated = true; + return disconnectRequestHandler + .requestDisconnect(reason) + .handle( + (__, error) -> { + if (error != null) { + LOG.debug("Failed to disconnect from " + getId() + " cleanly.", error); + } + disconnectImmediately(Optional.of(reason), true); + return null; + }); + } + + @Override + public void setDisconnectRequestHandler(final DisconnectRequestHandler handler) { + this.disconnectRequestHandler = handler; + } + + @Override + public void subscribeDisconnect(final PeerDisconnectedSubscriber subscriber) { + SafeFuture.of(connection.closeFuture()) + .always(() -> subscriber.onDisconnected(disconnectReason, disconnectLocallyInitiated)); + } + + @Override + public SafeFuture sendRequest( + RpcMethod rpcMethod, final Bytes initialPayload, final RpcRequestHandler handler) { + RpcHandler rpcHandler = rpcHandlers.get(rpcMethod); + if (rpcHandler == null) { + throw new IllegalArgumentException("Unknown rpc method invoked: " + rpcMethod.getId()); + } + return rpcHandler.sendRequest(connection, initialPayload, handler); + } + + @Override + public boolean connectionInitiatedLocally() { + return connection.isInitiator(); + } + + @Override + public boolean connectionInitiatedRemotely() { + return !connectionInitiatedLocally(); + } + + private void handleConnectionClosed() { + LOG.debug("Disconnected from peer {}", getId()); + connected.set(false); + } + + @Override + public void adjustReputation(final ReputationAdjustment adjustment) { + final boolean shouldDisconnect = reputationManager.adjustReputation(getAddress(), adjustment); + if (shouldDisconnect) { + disconnectCleanly(DisconnectReason.REMOTE_FAULT).reportExceptions(); + } + } +} diff --git a/src/org/minima/system/network/base/peer/LibP2PRpcStream.java b/src/org/minima/system/network/base/peer/LibP2PRpcStream.java new file mode 100644 index 000000000..3e239fd8d --- /dev/null +++ b/src/org/minima/system/network/base/peer/LibP2PRpcStream.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import com.google.common.base.MoreObjects; +import io.libp2p.core.P2PChannel; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelHandlerContext; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.tuweni.bytes.Bytes; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.rpc.RpcStream; +// import tech.pegasys.teku.networking.p2p.rpc.StreamClosedException; +import org.minima.system.network.base.SafeFuture; + +public class LibP2PRpcStream implements RpcStream { + + private final P2PChannel p2pChannel; + private final ChannelHandlerContext ctx; + private final AtomicBoolean writeStreamClosed = new AtomicBoolean(false); + private final NodeId nodeId; + + public LibP2PRpcStream( + final NodeId nodeId, final P2PChannel p2pChannel, final ChannelHandlerContext ctx) { + this.nodeId = nodeId; + this.p2pChannel = p2pChannel; + this.ctx = ctx; + } + + @Override + public SafeFuture writeBytes(final Bytes bytes) throws StreamClosedException { + if (writeStreamClosed.get()) { + throw new StreamClosedException(); + } + final ByteBuf reqByteBuf = ctx.alloc().buffer(); + reqByteBuf.writeBytes(bytes.toArrayUnsafe()); + + return toSafeFuture(ctx.writeAndFlush(reqByteBuf)); + } + + @Override + public SafeFuture closeAbruptly() { + writeStreamClosed.set(true); + return SafeFuture.of(p2pChannel.close()).thenApply((res) -> null); + } + + @Override + public SafeFuture closeWriteStream() { + writeStreamClosed.set(true); + return toSafeFuture(ctx.channel().disconnect()); + } + + private SafeFuture toSafeFuture(ChannelFuture channelFuture) { + final SafeFuture future = new SafeFuture<>(); + channelFuture.addListener( + (f) -> { + if (f.isSuccess()) { + future.complete(null); + } else { + future.completeExceptionally(f.cause()); + } + }); + return future; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("nodeId", nodeId) + .add("channel id", ctx.channel().id()) + .add("writeStreamClosed", writeStreamClosed) + .toString(); + } +} diff --git a/src/org/minima/system/network/base/peer/MultiaddrPeerAddress.java b/src/org/minima/system/network/base/peer/MultiaddrPeerAddress.java new file mode 100644 index 000000000..0f45589cb --- /dev/null +++ b/src/org/minima/system/network/base/peer/MultiaddrPeerAddress.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import io.libp2p.core.PeerId; +import io.libp2p.core.multiformats.Multiaddr; +import io.libp2p.core.multiformats.Protocol; +import java.util.Objects; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public class MultiaddrPeerAddress extends PeerAddress { + + private final Multiaddr multiaddr; + + MultiaddrPeerAddress(final NodeId nodeId, final Multiaddr multiaddr) { + super(nodeId); + this.multiaddr = multiaddr; + } + + @Override + public String toExternalForm() { + return multiaddr.toString(); + } + + public static MultiaddrPeerAddress fromAddress(final String address) { + final Multiaddr multiaddr = Multiaddr.fromString(address); + return fromMultiaddr(multiaddr); + } + + public static MultiaddrPeerAddress fromDiscoveryPeer(final DiscoveryPeer discoveryPeer) { + final Multiaddr multiaddr = MultiaddrUtil.fromDiscoveryPeer(discoveryPeer); + return fromMultiaddr(multiaddr); + } + + private static MultiaddrPeerAddress fromMultiaddr(final Multiaddr multiaddr) { + final String p2pComponent = multiaddr.getStringComponent(Protocol.P2P); + if (p2pComponent == null) { + throw new IllegalArgumentException("No peer ID present in multiaddr: " + multiaddr); + } + final LibP2PNodeId nodeId = new LibP2PNodeId(PeerId.fromBase58(p2pComponent)); + return new MultiaddrPeerAddress(nodeId, multiaddr); + } + + public Multiaddr getMultiaddr() { + return multiaddr; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + final MultiaddrPeerAddress that = (MultiaddrPeerAddress) o; + return Objects.equals(multiaddr, that.multiaddr); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), multiaddr); + } +} diff --git a/src/org/minima/system/network/base/peer/MultiaddrUtil.java b/src/org/minima/system/network/base/peer/MultiaddrUtil.java new file mode 100644 index 000000000..60054d816 --- /dev/null +++ b/src/org/minima/system/network/base/peer/MultiaddrUtil.java @@ -0,0 +1,69 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import static io.libp2p.crypto.keys.Secp256k1Kt.unmarshalSecp256k1PublicKey; + +import io.libp2p.core.PeerId; +import io.libp2p.core.crypto.PubKey; +import io.libp2p.core.multiformats.Multiaddr; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; + +public class MultiaddrUtil { + + public static Multiaddr fromDiscoveryPeer(final DiscoveryPeer peer) { + return fromInetSocketAddress(peer.getNodeAddress(), getNodeId(peer)); + } + + public static Multiaddr fromDiscoveryPeerAsUdp(final DiscoveryPeer peer) { + return addPeerId(fromInetSocketAddress(peer.getNodeAddress(), "udp"), getNodeId(peer)); + } + + static Multiaddr fromInetSocketAddress(final InetSocketAddress address) { + return fromInetSocketAddress(address, "tcp"); + } + + static Multiaddr fromInetSocketAddress(final InetSocketAddress address, final String protocol) { + final String addrString = + String.format( + "/%s/%s/%s/%d", + protocol(address.getAddress()), + address.getAddress().getHostAddress(), + protocol, + address.getPort()); + return Multiaddr.fromString(addrString); + } + + public static Multiaddr fromInetSocketAddress( + final InetSocketAddress address, final NodeId nodeId) { + return addPeerId(fromInetSocketAddress(address, "tcp"), nodeId); + } + + private static Multiaddr addPeerId(final Multiaddr addr, final NodeId nodeId) { + return new Multiaddr(addr, Multiaddr.fromString("/p2p/" + nodeId.toBase58())); + } + + private static LibP2PNodeId getNodeId(final DiscoveryPeer peer) { + final PubKey pubKey = unmarshalSecp256k1PublicKey(peer.getPublicKey()); + return new LibP2PNodeId(PeerId.fromPubKey(pubKey)); + } + + private static String protocol(final InetAddress address) { + return address instanceof Inet6Address ? "ip6" : "ip4"; + } +} diff --git a/src/org/minima/system/network/base/peer/NodeId.java b/src/org/minima/system/network/base/peer/NodeId.java index e9469543b..ed78c6695 100644 --- a/src/org/minima/system/network/base/peer/NodeId.java +++ b/src/org/minima/system/network/base/peer/NodeId.java @@ -13,9 +13,11 @@ package org.minima.system.network.base.peer; +import org.apache.tuweni.bytes.Bytes; + public abstract class NodeId { - public abstract byte[] toBytes(); + public abstract Bytes toBytes(); public abstract String toBase58(); diff --git a/src/org/minima/system/network/base/peer/Peer.java b/src/org/minima/system/network/base/peer/Peer.java index 749ed451f..ef8ce313a 100644 --- a/src/org/minima/system/network/base/peer/Peer.java +++ b/src/org/minima/system/network/base/peer/Peer.java @@ -18,14 +18,14 @@ //import org.minima.system.network.base.DisconnectReason; //import org.minima.system.network.base.peer.PeerDisconnectedSubscriber; -import org.minima.system.network.base.RpcMethod; -import org.minima.system.network.base.RpcRequestHandler; -import org.minima.system.network.base.RpcStream; +import org.minima.system.network.base.peer.RpcMethod; +import org.minima.system.network.base.peer.RpcRequestHandler; +import org.minima.system.network.base.peer.RpcStream; import org.minima.system.network.base.SafeFuture; //import tech.pegasys.teku.networking.p2p.peer.DisconnectRequestHandler; -//import org.apache.tuweni.bytes.Bytes; +import org.apache.tuweni.bytes.Bytes; //import tech.pegasys.teku.infrastructure.async.SafeFuture; //import tech.pegasys.teku.networking.p2p.network.PeerAddress; //import tech.pegasys.teku.networking.p2p.reputation.ReputationAdjustment; @@ -52,7 +52,7 @@ default NodeId getId() { void subscribeDisconnect(PeerDisconnectedSubscriber subscriber); SafeFuture sendRequest( - RpcMethod rpcMethod, byte[] initialPayload, RpcRequestHandler handler); + RpcMethod rpcMethod, Bytes initialPayload, RpcRequestHandler handler); boolean connectionInitiatedLocally(); diff --git a/src/org/minima/system/network/base/peer/PeerAlreadyConnectedException.java b/src/org/minima/system/network/base/peer/PeerAlreadyConnectedException.java new file mode 100644 index 000000000..cd43d6459 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerAlreadyConnectedException.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +//import tech.pegasys.teku.networking.p2p.peer.Peer; + +/** + * Indicates that two connections to the same PeerID were incorrectly established. + * + *

LibP2P usually detects attempts to establish multiple connections at the same time, but if we + * have incoming and outgoing connections simultaneously to the same peer, sometimes it slips + * through. In that case this exception is thrown so that the new connection is terminated before + * handshakes complete and we are able to identify the situation and return the existing peer. + */ +public class PeerAlreadyConnectedException extends RuntimeException { + private final Peer peer; + + public PeerAlreadyConnectedException(final Peer peer) { + super("Already connected to peer " + peer.getId().toBase58()); + this.peer = peer; + } + + public Peer getPeer() { + return peer; + } +} diff --git a/src/org/minima/system/network/base/peer/PeerDisconnectedException.java b/src/org/minima/system/network/base/peer/PeerDisconnectedException.java new file mode 100644 index 000000000..10b518061 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerDisconnectedException.java @@ -0,0 +1,23 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public class PeerDisconnectedException extends RuntimeException { + + public PeerDisconnectedException() {} + + public PeerDisconnectedException(Throwable cause) { + super(cause); + } +} diff --git a/src/org/minima/system/network/base/peer/PeerHandler.java b/src/org/minima/system/network/base/peer/PeerHandler.java new file mode 100644 index 000000000..a54c25a99 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerHandler.java @@ -0,0 +1,22 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +//import tech.pegasys.teku.networking.p2p.peer.Peer; + +public interface PeerHandler { + void onConnect(final Peer peer); + + void onDisconnect(final Peer peer); +} diff --git a/src/org/minima/system/network/base/peer/PeerManager.java b/src/org/minima/system/network/base/peer/PeerManager.java new file mode 100644 index 000000000..3e2b01be0 --- /dev/null +++ b/src/org/minima/system/network/base/peer/PeerManager.java @@ -0,0 +1,164 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +//package tech.pegasys.teku.networking.p2p.libp2p; +package org.minima.system.network.base.peer; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; +import io.libp2p.core.Connection; +import io.libp2p.core.ConnectionHandler; +import io.libp2p.core.Network; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +//import org.hyperledger.besu.plugin.services.MetricsSystem; +import org.jetbrains.annotations.NotNull; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; +// import tech.pegasys.teku.infrastructure.subscribers.Subscribers; +// import tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler; +// import tech.pegasys.teku.networking.p2p.network.PeerHandler; +// import tech.pegasys.teku.networking.p2p.peer.DisconnectReason; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.Peer; +// import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; +// import tech.pegasys.teku.networking.p2p.reputation.ReputationManager; +// import tech.pegasys.teku.networking.p2p.rpc.RpcMethod; +import org.minima.system.network.base.SafeFuture; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.metrics.TekuMetricCategory; + +public class PeerManager implements ConnectionHandler { + + private static final Logger LOG = LogManager.getLogger(); + + private final Map rpcHandlers; + + private final ConcurrentHashMap connectedPeerMap = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> pendingConnections = + new ConcurrentHashMap<>(); + private final ReputationManager reputationManager; + private final List peerHandlers; + + private final Subscribers> connectSubscribers = + Subscribers.create(true); + + public PeerManager( + final MetricsSystem metricsSystem, + final ReputationManager reputationManager, + final List peerHandlers, + final Map rpcHandlers) { + this.reputationManager = reputationManager; + this.peerHandlers = peerHandlers; + this.rpcHandlers = rpcHandlers; + metricsSystem.createGauge( + TekuMetricCategory.LIBP2P, "peers", "Tracks number of libp2p peers", this::getPeerCount); + } + + @Override + public void handleConnection(@NotNull final Connection connection) { + Peer peer = new LibP2PPeer(connection, rpcHandlers, reputationManager); + onConnectedPeer(peer); + } + + public long subscribeConnect(final PeerConnectedSubscriber subscriber) { + return connectSubscribers.subscribe(subscriber); + } + + public void unsubscribeConnect(final long subscriptionId) { + connectSubscribers.unsubscribe(subscriptionId); + } + + public SafeFuture connect(final MultiaddrPeerAddress peer, final Network network) { + return pendingConnections.computeIfAbsent(peer.getId(), __ -> doConnect(peer, network)); + } + + private SafeFuture doConnect(final MultiaddrPeerAddress peer, final Network network) { + LOG.debug("Connecting to {}", peer); + + return SafeFuture.of(() -> network.connect(peer.getMultiaddr())) + .thenApply( + connection -> { + final LibP2PNodeId nodeId = + new LibP2PNodeId(connection.secureSession().getRemoteId()); + final Peer connectedPeer = connectedPeerMap.get(nodeId); + if (connectedPeer == null) { + if (connection.closeFuture().isDone()) { + // Connection has been immediately closed and the peer already removed + // Since the connection is closed anyway, we can create a new peer to wrap it. + return new LibP2PPeer(connection, rpcHandlers, reputationManager); + } else { + // Theoretically this should never happen because removing from the map is done + // by the close future completing, but make a loud noise just in case. + throw new IllegalStateException( + "No peer registered for established connection to " + nodeId); + } + } + reputationManager.reportInitiatedConnectionSuccessful(peer); + return connectedPeer; + }) + .exceptionallyCompose(this::handleConcurrentConnectionInitiation) + .catchAndRethrow(error -> reputationManager.reportInitiatedConnectionFailed(peer)) + .whenComplete((result, error) -> pendingConnections.remove(peer.getId())); + } + + private CompletionStage handleConcurrentConnectionInitiation(final Throwable error) { + final Throwable rootCause = Throwables.getRootCause(error); + return rootCause instanceof PeerAlreadyConnectedException + ? SafeFuture.completedFuture(((PeerAlreadyConnectedException) rootCause).getPeer()) + : SafeFuture.failedFuture(error); + } + + public Optional getPeer(NodeId id) { + return Optional.ofNullable(connectedPeerMap.get(id)); + } + + @VisibleForTesting + void onConnectedPeer(Peer peer) { + final boolean wasAdded = connectedPeerMap.putIfAbsent(peer.getId(), peer) == null; + if (wasAdded) { + LOG.debug("onConnectedPeer() {}", peer.getId()); + peerHandlers.forEach(h -> h.onConnect(peer)); + connectSubscribers.forEach(c -> c.onConnected(peer)); + peer.subscribeDisconnect( + (reason, locallyInitiated) -> onDisconnectedPeer(peer, reason, locallyInitiated)); + } else { + LOG.trace("Disconnecting duplicate connection to {}", peer::getId); + peer.disconnectImmediately(Optional.empty(), true); + throw new PeerAlreadyConnectedException(peer); + } + } + + private void onDisconnectedPeer( + final Peer peer, final Optional reason, final boolean locallyInitiated) { + if (connectedPeerMap.remove(peer.getId()) != null) { + LOG.debug("Peer disconnected: {}", peer.getId()); + reputationManager.reportDisconnection(peer.getAddress(), reason, locallyInitiated); + peerHandlers.forEach(h -> h.onDisconnect(peer)); + } + } + + public Stream streamPeers() { + return connectedPeerMap.values().stream(); + } + + public int getPeerCount() { + return connectedPeerMap.size(); + } +} diff --git a/src/org/minima/system/network/base/peer/ReputationManager.java b/src/org/minima/system/network/base/peer/ReputationManager.java new file mode 100644 index 000000000..ad97c86e3 --- /dev/null +++ b/src/org/minima/system/network/base/peer/ReputationManager.java @@ -0,0 +1,166 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +// import static tech.pegasys.teku.networking.p2p.peer.DisconnectReason.TOO_MANY_PEERS; +// import static tech.pegasys.teku.networking.p2p.peer.DisconnectReason.UNRESPONSIVE; +import static org.minima.system.network.base.peer.DisconnectReason.TOO_MANY_PEERS; +import static org.minima.system.network.base.peer.DisconnectReason.UNRESPONSIVE; + +import java.util.EnumSet; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +// import org.hyperledger.besu.plugin.services.MetricsSystem; +// import tech.pegasys.teku.infrastructure.collections.cache.Cache; +// import tech.pegasys.teku.infrastructure.collections.cache.LRUCache; +// import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; +// import tech.pegasys.teku.infrastructure.time.TimeProvider; +// import tech.pegasys.teku.infrastructure.unsigned.UInt64; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.DisconnectReason; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; + +import org.minima.system.network.base.LRUCache; +import org.minima.system.network.base.TimeProvider; +import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.metrics.TekuMetricCategory; +import org.minima.system.network.base.ssz.Cache; +import org.minima.system.network.base.ssz.UInt64; + +public class ReputationManager { + static final UInt64 FAILURE_BAN_PERIOD = UInt64.valueOf(TimeUnit.MINUTES.toSeconds(2)); + + static final int LARGE_CHANGE = 10; + static final int SMALL_CHANGE = 3; + private static final int DISCONNECT_THRESHOLD = -LARGE_CHANGE; + private static final int MAX_REPUTATION_SCORE = 2 * LARGE_CHANGE; + + private final TimeProvider timeProvider; + private final Cache peerReputations; + + public ReputationManager( + final MetricsSystem metricsSystem, final TimeProvider timeProvider, final int capacity) { + this.timeProvider = timeProvider; + this.peerReputations = new LRUCache<>(capacity); + metricsSystem.createIntegerGauge( + TekuMetricCategory.NETWORK, + "peer_reputation_cache_size", + "Total number of peer reputations tracked", + peerReputations::size); + } + + public void reportInitiatedConnectionFailed(final PeerAddress peerAddress) { + getOrCreateReputation(peerAddress) + .reportInitiatedConnectionFailed(timeProvider.getTimeInSeconds()); + } + + public boolean isConnectionInitiationAllowed(final PeerAddress peerAddress) { + return peerReputations + .getCached(peerAddress.getId()) + .map(reputation -> reputation.shouldInitiateConnection(timeProvider.getTimeInSeconds())) + .orElse(true); + } + + public void reportInitiatedConnectionSuccessful(final PeerAddress peerAddress) { + getOrCreateReputation(peerAddress).reportInitiatedConnectionSuccessful(); + } + + public void reportDisconnection( + final PeerAddress peerAddress, + final Optional reason, + final boolean locallyInitiated) { + getOrCreateReputation(peerAddress) + .reportDisconnection(timeProvider.getTimeInSeconds(), reason, locallyInitiated); + } + + /** + * Adjust the reputation score for a peer either positively or negatively. + * + * @param peerAddress the address of the peer to adjust the score for. + * @param effect the reputation change to apply. + * @return true if the peer should be disconnected, otherwise false. + */ + public boolean adjustReputation( + final PeerAddress peerAddress, final ReputationAdjustment effect) { + return getOrCreateReputation(peerAddress).adjustReputation(effect); + } + + private Reputation getOrCreateReputation(final PeerAddress peerAddress) { + return peerReputations.get(peerAddress.getId(), key -> new Reputation()); + } + + private static class Reputation { + private static final EnumSet LOCAL_TEMPORARY_DISCONNECT_REASONS = + EnumSet.of( + // We're currently at limit so don't mark peer unsuitable + TOO_MANY_PEERS, + // Peer may have been unresponsive due to a temporary network issue. In particular + // our internet access may have failed and all peers could be unresponsive. + // If we consider them all permanently unsuitable we may not be able to rejoin the + // network once our internet access is restored. + UNRESPONSIVE); + + private volatile Optional lastInitiationFailure = Optional.empty(); + private volatile boolean unsuitable = false; + private final AtomicInteger score = new AtomicInteger(0); + + public void reportInitiatedConnectionFailed(final UInt64 failureTime) { + lastInitiationFailure = Optional.of(failureTime); + } + + public boolean shouldInitiateConnection(final UInt64 currentTime) { + return !unsuitable + && lastInitiationFailure + .map( + lastFailureTime -> + lastFailureTime.plus(FAILURE_BAN_PERIOD).compareTo(currentTime) < 0) + .orElse(true); + } + + public void reportInitiatedConnectionSuccessful() { + lastInitiationFailure = Optional.empty(); + } + + public void reportDisconnection( + final UInt64 disconnectTime, + final Optional reason, + final boolean locallyInitiated) { + if (isLocallyConsideredUnsuitable(reason, locallyInitiated) + || reason.map(DisconnectReason::isPermanent).orElse(false)) { + unsuitable = true; + } else { + lastInitiationFailure = Optional.of(disconnectTime); + } + } + + private boolean isLocallyConsideredUnsuitable( + final Optional reason, final boolean locallyInitiated) { + return locallyInitiated + && reason.map(r -> !LOCAL_TEMPORARY_DISCONNECT_REASONS.contains(r)).orElse(false); + } + + public boolean adjustReputation(final ReputationAdjustment effect) { + final int newScore = + score.updateAndGet( + current -> Math.min(MAX_REPUTATION_SCORE, current + effect.getScoreChange())); + final boolean shouldDisconnect = newScore <= DISCONNECT_THRESHOLD; + if (shouldDisconnect) { + // Prevent our node from connecting out to the remote peer. + unsuitable = true; + } + return shouldDisconnect; + } + } +} diff --git a/src/org/minima/system/network/base/peer/RpcHandler.java b/src/org/minima/system/network/base/peer/RpcHandler.java new file mode 100644 index 000000000..16a460087 --- /dev/null +++ b/src/org/minima/system/network/base/peer/RpcHandler.java @@ -0,0 +1,229 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +//import static tech.pegasys.teku.infrastructure.async.FutureUtil.ignoreFuture; +import static org.minima.system.network.base.FutureUtil.ignoreFuture; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; +import io.libp2p.core.Connection; +import io.libp2p.core.ConnectionClosedException; +import io.libp2p.core.P2PChannel; +import io.libp2p.core.Stream; +import io.libp2p.core.StreamPromise; +import io.libp2p.core.multistream.ProtocolBinding; +import io.libp2p.core.multistream.ProtocolDescriptor; +import io.libp2p.etc.util.netty.mux.RemoteWriteClosed; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import java.time.Duration; +import java.util.Optional; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.jetbrains.annotations.NotNull; +import org.minima.system.network.base.AsyncRunner; +import org.minima.system.network.base.SafeFuture; +import org.minima.system.network.base.SafeFuture.Interruptor; +import org.minima.system.network.base.peer.RpcHandler.Controller; + +// import tech.pegasys.teku.infrastructure.async.AsyncRunner; +// import tech.pegasys.teku.infrastructure.async.SafeFuture; +// import tech.pegasys.teku.infrastructure.async.SafeFuture.Interruptor; +// import tech.pegasys.teku.infrastructure.exceptions.ExceptionUtil; +// import tech.pegasys.teku.networking.p2p.libp2p.LibP2PNodeId; +// import tech.pegasys.teku.networking.p2p.libp2p.rpc.RpcHandler.Controller; +// import tech.pegasys.teku.networking.p2p.peer.NodeId; +// import tech.pegasys.teku.networking.p2p.peer.PeerDisconnectedException; +// import tech.pegasys.teku.networking.p2p.rpc.RpcMethod; +// import tech.pegasys.teku.networking.p2p.rpc.RpcRequestHandler; +// import tech.pegasys.teku.networking.p2p.rpc.RpcStream; +// import tech.pegasys.teku.networking.p2p.rpc.StreamClosedException; +// import tech.pegasys.teku.networking.p2p.rpc.StreamTimeoutException; + +public class RpcHandler implements ProtocolBinding { + private static final Duration TIMEOUT = Duration.ofSeconds(5); + private static final Logger LOG = LogManager.getLogger(); + + private final RpcMethod rpcMethod; + private final AsyncRunner asyncRunner; + + public RpcHandler(final AsyncRunner asyncRunner, RpcMethod rpcMethod) { + this.asyncRunner = asyncRunner; + this.rpcMethod = rpcMethod; + } + + @SuppressWarnings("unchecked") + public SafeFuture sendRequest( + Connection connection, Bytes initialPayload, RpcRequestHandler handler) { + + Interruptor closeInterruptor = + SafeFuture.createInterruptor(connection.closeFuture(), PeerDisconnectedException::new); + Interruptor timeoutInterruptor = + SafeFuture.createInterruptor( + asyncRunner.getDelayedFuture(TIMEOUT), + () -> + new StreamTimeoutException( + "Timed out waiting to initialize stream for method " + rpcMethod.getId())); + + return SafeFuture.notInterrupted(closeInterruptor) + .thenApply(__ -> connection.muxerSession().createStream(this)) + // waiting for a stream or interrupt + .thenWaitFor(StreamPromise::getStream) + .orInterrupt(closeInterruptor, timeoutInterruptor) + .thenCompose( + streamPromise -> + // waiting for controller, writing initial payload or interrupt + SafeFuture.of(streamPromise.getController()) + .orInterrupt(closeInterruptor, timeoutInterruptor) + .thenPeek(ctr -> ctr.setRequestHandler(handler)) + .thenApply(Controller::getRpcStream) + .thenWaitFor(rpcStream -> rpcStream.writeBytes(initialPayload)) + .orInterrupt(closeInterruptor, timeoutInterruptor) + // closing the stream in case of any errors or interruption + .whenException(err -> closeStreamAbruptly(streamPromise.getStream().join()))) + .catchAndRethrow( + err -> { + if (ExceptionUtil.getCause(err, ConnectionClosedException.class).isPresent()) { + throw new PeerDisconnectedException(err); + } + }); + } + + private void closeStreamAbruptly(Stream stream) { + SafeFuture.of(stream.close()).reportExceptions(); + } + + @NotNull + @Override + public ProtocolDescriptor getProtocolDescriptor() { + return new ProtocolDescriptor(rpcMethod.getId()); + } + + @NotNull + @Override + public SafeFuture initChannel(P2PChannel channel, String s) { + final Connection connection = ((io.libp2p.core.Stream) channel).getConnection(); + final NodeId nodeId = new LibP2PNodeId(connection.secureSession().getRemoteId()); + Controller controller = new Controller(nodeId, channel); + if (!channel.isInitiator()) { + controller.setRequestHandler(rpcMethod.createIncomingRequestHandler()); + } + channel.pushHandler(controller); + return controller.activeFuture; + } + + static class Controller extends SimpleChannelInboundHandler { + private final NodeId nodeId; + private final P2PChannel p2pChannel; + private Optional rpcRequestHandler = Optional.empty(); + private RpcStream rpcStream; + private boolean readCompleted = false; + + protected final SafeFuture activeFuture = new SafeFuture<>(); + + private Controller(final NodeId nodeId, final P2PChannel p2pChannel) { + this.nodeId = nodeId; + this.p2pChannel = p2pChannel; + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + rpcStream = new LibP2PRpcStream(nodeId, p2pChannel, ctx); + activeFuture.complete(this); + } + + public RpcStream getRpcStream() { + return rpcStream; + } + + @Override + protected void channelRead0(final ChannelHandlerContext ctx, final ByteBuf msg) { + runHandler(h -> h.processData(nodeId, rpcStream, msg)); + } + + public void setRequestHandler(RpcRequestHandler rpcRequestHandler) { + if (this.rpcRequestHandler.isPresent()) { + throw new IllegalStateException("Attempt to set an already set data handler"); + } + this.rpcRequestHandler = Optional.of(rpcRequestHandler); + + activeFuture.finish( + () -> { + rpcRequestHandler.active(nodeId, rpcStream); + }, + err -> { + if (err != null) { + if (Throwables.getRootCause(err) instanceof StreamClosedException) { + LOG.debug("Stream closed while processing rpc input", err); + } else { + LOG.error("Unhandled exception while processing rpc input", err); + } + } + }); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + LOG.error("Unhandled error while processes req/response", cause); + final IllegalStateException exception = new IllegalStateException("Channel exception", cause); + activeFuture.completeExceptionally(exception); + closeAbruptly(); + } + + @Override + public void handlerRemoved(ChannelHandlerContext ctx) { + onChannelClosed(); + } + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) { + if (evt instanceof RemoteWriteClosed) { + onRemoteWriteClosed(); + } + } + + private void onRemoteWriteClosed() { + if (!readCompleted) { + readCompleted = true; + runHandler(h -> h.readComplete(nodeId, rpcStream)); + } + } + + private void onChannelClosed() { + try { + onRemoteWriteClosed(); + runHandler(h -> h.closed(nodeId, rpcStream)); + } finally { + rpcRequestHandler = Optional.empty(); + } + } + + private void runHandler(final Consumer action) { + rpcRequestHandler.ifPresentOrElse(action, this::closeAbruptly); + } + + @VisibleForTesting + void closeAbruptly() { + // We're listening for the result of the close future above, so we can ignore this future + ignoreFuture(p2pChannel.close()); + + // Make sure to complete activation future in case we are never activated + activeFuture.completeExceptionally(new StreamClosedException()); + } + } +} diff --git a/src/org/minima/system/network/base/peer/RpcMethod.java b/src/org/minima/system/network/base/peer/RpcMethod.java new file mode 100644 index 000000000..636405d5d --- /dev/null +++ b/src/org/minima/system/network/base/peer/RpcMethod.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public interface RpcMethod { + + String getId(); + + RpcRequestHandler createIncomingRequestHandler(); +} diff --git a/src/org/minima/system/network/base/RpcRequestHandler.java b/src/org/minima/system/network/base/peer/RpcRequestHandler.java similarity index 91% rename from src/org/minima/system/network/base/RpcRequestHandler.java rename to src/org/minima/system/network/base/peer/RpcRequestHandler.java index 925ab7b2a..c8ae85d1c 100644 --- a/src/org/minima/system/network/base/RpcRequestHandler.java +++ b/src/org/minima/system/network/base/peer/RpcRequestHandler.java @@ -11,9 +11,7 @@ * specific language governing permissions and limitations under the License. */ -package org.minima.system.network.base; - -import org.minima.system.network.base.peer.NodeId; +package org.minima.system.network.base.peer; import io.netty.buffer.ByteBuf; //import tech.pegasys.teku.networking.p2p.peer.NodeId; diff --git a/src/org/minima/system/network/base/peer/RpcStream.java b/src/org/minima/system/network/base/peer/RpcStream.java new file mode 100644 index 000000000..528c99787 --- /dev/null +++ b/src/org/minima/system/network/base/peer/RpcStream.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import org.apache.tuweni.bytes.Bytes; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; +import org.minima.system.network.base.SafeFuture; + +public interface RpcStream { + + SafeFuture writeBytes(Bytes bytes) throws StreamClosedException; + + /** + * Close the stream altogether, allowing no further reads or writes. + * + * @return A future completing when the stream is closed. + */ + SafeFuture closeAbruptly(); + + /** + * Close the write side of the stream. When both sides of the stream close their write stream, the + * entire stream will be closed. + * + * @return A future completing when the write stream is closed. + */ + SafeFuture closeWriteStream(); +} diff --git a/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java b/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java new file mode 100644 index 000000000..2f9a12f01 --- /dev/null +++ b/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import static java.util.Collections.emptyList; +import static java.util.stream.Collectors.toList; +//import static tech.pegasys.teku.networking.p2p.connection.PeerPools.PeerPool.STATIC; +import static org.minima.system.network.base.peer.PeerPools.PeerPool.STATIC; + +import java.util.List; +import java.util.function.Supplier; +// import tech.pegasys.teku.networking.p2p.connection.PeerPools; +// import tech.pegasys.teku.networking.p2p.connection.PeerSelectionStrategy; +// import tech.pegasys.teku.networking.p2p.connection.TargetPeerRange; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.network.P2PNetwork; +// import tech.pegasys.teku.networking.p2p.network.PeerAddress; +// import tech.pegasys.teku.networking.p2p.peer.Peer; + +//import org.minima.system.network.base.; + +import org.minima.system.network.base.P2PNetwork; +import org.minima.system.network.base.DiscoveryPeer; + +public class SimplePeerSelectionStrategy implements PeerSelectionStrategy { + private final TargetPeerRange targetPeerRange; + + public SimplePeerSelectionStrategy(final TargetPeerRange targetPeerRange) { + this.targetPeerRange = targetPeerRange; + } + + @Override + public List selectPeersToConnect( + final P2PNetwork network, + final PeerPools peerPools, + final Supplier> candidates) { + final int peersToAdd = targetPeerRange.getPeersToAdd(network.getPeerCount()); + if (peersToAdd == 0) { + return emptyList(); + } + return candidates.get().stream() + .map(network::createPeerAddress) + .limit(peersToAdd) + .collect(toList()); + } + + @Override + public List selectPeersToDisconnect( + final P2PNetwork network, final PeerPools peerPools) { + final int peersToDrop = targetPeerRange.getPeersToDrop(network.getPeerCount()); + return network + .streamPeers() + .filter(peer -> peerPools.getPool(peer.getId()) != STATIC) + .limit(peersToDrop) + .collect(toList()); + } +} diff --git a/src/org/minima/system/network/base/peer/StreamClosedException.java b/src/org/minima/system/network/base/peer/StreamClosedException.java new file mode 100644 index 000000000..513163593 --- /dev/null +++ b/src/org/minima/system/network/base/peer/StreamClosedException.java @@ -0,0 +1,16 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public class StreamClosedException extends RuntimeException {} diff --git a/src/org/minima/system/network/base/peer/StreamTimeoutException.java b/src/org/minima/system/network/base/peer/StreamTimeoutException.java new file mode 100644 index 000000000..f90645409 --- /dev/null +++ b/src/org/minima/system/network/base/peer/StreamTimeoutException.java @@ -0,0 +1,21 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +public class StreamTimeoutException extends RuntimeException { + + public StreamTimeoutException(final String message) { + super(message); + } +} diff --git a/src/org/minima/system/network/base/peer/Subscribers.java b/src/org/minima/system/network/base/peer/Subscribers.java new file mode 100644 index 000000000..87841336b --- /dev/null +++ b/src/org/minima/system/network/base/peer/Subscribers.java @@ -0,0 +1,128 @@ +/* + * Copyright 2019 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Tracks subscribers that should be notified when some event occurred. This class is safe to use + * from multiple threads. + * + *

Each subscriber is assigned a unique ID which can be used to unsubscribe. This approach was + * chosen over using the subscriber's object identity to eliminate a common trap believing that + * method references are equal when they refer to the same method. For example, if object identity + * were used to track subscribers it would be possible to write the incorrect code: + * + * 

+ * subscribers.subscribe(this::onSomeEvent);
+ * subscribers.unsubscribe(this::onSomeEvent);
+ *
+ * + *

Since the two separate this:onSomeEvent are not equal, the subscriber wouldn't be + * removed. This bug is avoided by assigning each subscriber a unique ID and using that to + * unsubscribe. + * + * @param the type of subscribers + */ +public class Subscribers { + private static final Logger LOG = LogManager.getLogger(); + + private final AtomicLong subscriberId = new AtomicLong(); + private final Map subscribers = new ConcurrentHashMap<>(); + + private final boolean suppressCallbackExceptions; + + private Subscribers(final boolean suppressCallbackExceptions) { + this.suppressCallbackExceptions = suppressCallbackExceptions; + } + + public static Subscribers create(final boolean suppressCallbackExceptions) { + return new Subscribers<>(suppressCallbackExceptions); + } + + /** + * Add a subscriber to the list. + * + * @param subscriber the subscriber to add + * @return the ID assigned to this subscriber + */ + public long subscribe(final T subscriber) { + final long id = subscriberId.getAndIncrement(); + subscribers.put(id, subscriber); + return id; + } + + /** + * Remove a subscriber from the list. + * + * @param subscriberId the ID of the subscriber to remove + * @return true if a subscriber with that ID was found and removed, otherwise + * false + */ + public boolean unsubscribe(final long subscriberId) { + return subscribers.remove(subscriberId) != null; + } + + /** + * Iterate through the current list of subscribers. This is typically used to deliver events e.g.: + * + * 

+   * subscribers.forEach(subscriber -> subscriber.onEvent());
+   *
+ * + * @param action the action to perform for each subscriber + */ + public void forEach(final Consumer action) { + subscribers + .values() + .forEach( + subscriber -> { + try { + action.accept(subscriber); + } catch (Throwable throwable) { + if (suppressCallbackExceptions) { + LOG.error("Error in callback: ", throwable); + } else { + throw throwable; + } + } + }); + } + + /** + * Deliver an event to each subscriber by calling eventMethod. + * + * @param eventMethod the method to call + * @param event the event object to provide as a parameter + * @param the type of the event + */ + public void deliver(final BiConsumer eventMethod, final E event) { + forEach(subscriber -> eventMethod.accept(subscriber, event)); + } + + /** + * Get the current subscriber count. + * + * @return the current number of subscribers. + */ + public int getSubscriberCount() { + return subscribers.size(); + } +} diff --git a/src/org/minima/system/network/base/peer/TargetPeerRange.java b/src/org/minima/system/network/base/peer/TargetPeerRange.java new file mode 100644 index 000000000..571c8d9b3 --- /dev/null +++ b/src/org/minima/system/network/base/peer/TargetPeerRange.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base.peer; + +import com.google.common.base.Preconditions; + +public class TargetPeerRange { + private final int lowerBound; + private final int upperBound; + private final int minimumRandomlySelectedPeerCount; + + public TargetPeerRange( + final int lowerBound, final int upperBound, final int minimumRandomlySelectedPeerCount) { + Preconditions.checkArgument(lowerBound <= upperBound, "Invalid target peer count range"); + this.lowerBound = lowerBound; + this.upperBound = upperBound; + this.minimumRandomlySelectedPeerCount = minimumRandomlySelectedPeerCount; + } + + public int getPeersToAdd(final int currentPeerCount) { + return currentPeerCount < lowerBound ? upperBound - currentPeerCount : 0; + } + + public int getPeersToDrop(final int currentPeerCount) { + return currentPeerCount > upperBound ? currentPeerCount - upperBound : 0; + } + + public int getRandomlySelectedPeersToAdd(final int currentRandomlySelectedPeerCount) { + return currentRandomlySelectedPeerCount < minimumRandomlySelectedPeerCount + ? minimumRandomlySelectedPeerCount - currentRandomlySelectedPeerCount + : 0; + } + + public int getRandomlySelectedPeersToDrop( + final int currentRandomlySelectedPeerCount, final int currentTotalPeerCount) { + final int totalPeersToDrop = getPeersToDrop(currentTotalPeerCount); + if (totalPeersToDrop == 0) { + return 0; + } + return currentRandomlySelectedPeerCount > minimumRandomlySelectedPeerCount + ? Math.min( + currentRandomlySelectedPeerCount - minimumRandomlySelectedPeerCount, totalPeersToDrop) + : 0; + } +} From 52bb0dfa5a0697aeaf1769c14fad7cbdfd2adeb0 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Fri, 9 Apr 2021 10:53:08 +0100 Subject: [PATCH 11/55] p2p: added netty firewall rules and removed some duplicate rpc classes. --- .../network/base/ConnectionManager.java | 1 + .../network/base/DelegatingP2PNetwork.java | 6 +- .../system/network/base/DiscoveryNetwork.java | 4 +- .../network/base/DiscoveryNetworkFactory.java | 1 + .../system/network/base/DiscoveryPeer.java | 86 ------------ .../system/network/base/DiscoveryService.java | 1 + .../minima/system/network/base/Firewall.java | 66 +++++++++ .../system/network/base/LibP2PNetwork.java | 13 +- .../system/network/base/LibP2PNodeId.java | 37 ----- .../network/base/LibP2PParamsFactory.java | 1 + .../system/network/base/MplexFirewall.java | 128 ++++++++++++++++++ .../network/base/NoOpDiscoveryService.java | 1 + .../system/network/base/P2PNetwork.java | 4 +- .../minima/system/network/base/P2PStart.java | 37 ++--- .../minima/system/network/base/RpcStream.java | 37 ----- .../network/base/StreamClosedException.java | 16 --- .../network/base/gossip/GossipNetwork.java | 4 +- .../PrivateKeyGenerator.java} | 17 +-- .../libp2p/gossip/LibP2PGossipNetwork.java | 8 +- .../network/base/peer/MultiaddrUtil.java | 2 +- .../system/network/base/peer/PeerAddress.java | 4 +- .../base/peer/PeerSelectionStrategy.java | 2 +- .../peer/SimplePeerSelectionStrategy.java | 1 - .../network/base/ssz/ArrayIntCache.java | 6 + .../minima/system/network/base/ssz/Cache.java | 4 + .../system/network/base/ssz/IntCache.java | 5 + .../system/network/base/ssz/NoopIntCache.java | 4 + 27 files changed, 273 insertions(+), 223 deletions(-) delete mode 100644 src/org/minima/system/network/base/DiscoveryPeer.java create mode 100644 src/org/minima/system/network/base/Firewall.java delete mode 100644 src/org/minima/system/network/base/LibP2PNodeId.java create mode 100644 src/org/minima/system/network/base/MplexFirewall.java delete mode 100644 src/org/minima/system/network/base/RpcStream.java delete mode 100644 src/org/minima/system/network/base/StreamClosedException.java rename src/org/minima/system/network/base/{RpcMethod.java => libp2p/PrivateKeyGenerator.java} (62%) diff --git a/src/org/minima/system/network/base/ConnectionManager.java b/src/org/minima/system/network/base/ConnectionManager.java index 1400a40b9..da1f7fc82 100644 --- a/src/org/minima/system/network/base/ConnectionManager.java +++ b/src/org/minima/system/network/base/ConnectionManager.java @@ -30,6 +30,7 @@ import org.minima.system.network.base.metrics.MetricsSystem; import org.minima.system.network.base.metrics.TekuMetricCategory; import org.minima.system.network.base.peer.DisconnectReason; +import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.Peer; // // import org.hyperledger.besu.plugin.services.MetricsSystem; // // import org.hyperledger.besu.plugin.services.metrics.Counter; diff --git a/src/org/minima/system/network/base/DelegatingP2PNetwork.java b/src/org/minima/system/network/base/DelegatingP2PNetwork.java index a1419e585..7733df2bc 100644 --- a/src/org/minima/system/network/base/DelegatingP2PNetwork.java +++ b/src/org/minima/system/network/base/DelegatingP2PNetwork.java @@ -25,9 +25,11 @@ // import tech.pegasys.teku.networking.p2p.peer.NodeId; // import tech.pegasys.teku.networking.p2p.peer.Peer; +import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.gossip.TopicChannel; import org.minima.system.network.base.gossip.TopicHandler; import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; +import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; import org.minima.system.network.base.peer.PeerAddress; @@ -60,7 +62,7 @@ public boolean isConnected(final PeerAddress peerAddress) { } @Override - public byte[] getPrivateKey() { + public Bytes getPrivateKey() { return network.getPrivateKey(); } @@ -110,7 +112,7 @@ public SafeFuture stop() { } @Override - public SafeFuture gossip(final String topic, final byte[] data) { + public SafeFuture gossip(final String topic, final Bytes data) { return network.gossip(topic, data); } diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index 935eb33aa..7377c94fb 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -121,7 +121,9 @@ private static DiscoveryService createDiscoveryService( final Bytes privateKey) { final DiscoveryService discoveryService; if (discoConfig.isDiscoveryEnabled()) { - discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); + //TODO: activate V5 service + // discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); + discoveryService = new NoOpDiscoveryService(); } else { discoveryService = new NoOpDiscoveryService(); } diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index 14b37b20b..ae3daf866 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -23,6 +23,7 @@ import java.util.concurrent.TimeoutException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.minima.system.network.base.libp2p.PrivateKeyGenerator; // import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; // import tech.pegasys.teku.infrastructure.async.DelayedExecutorAsyncRunner; // import tech.pegasys.teku.infrastructure.async.SafeFuture; diff --git a/src/org/minima/system/network/base/DiscoveryPeer.java b/src/org/minima/system/network/base/DiscoveryPeer.java deleted file mode 100644 index 3a64ae598..000000000 --- a/src/org/minima/system/network/base/DiscoveryPeer.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.minima.system.network.base; - -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; - -import org.minima.system.network.base.ssz.SszBitvector; - -import java.net.InetSocketAddress; -import java.util.Optional; -// import org.apache.tuweni.bytes.Bytes; -// import tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc.EnrForkId; -// import tech.pegasys.teku.ssz.collections.SszBitvector; - -public class DiscoveryPeer { - private final byte[] publicKey; - private final InetSocketAddress nodeAddress; - private final Optional enrForkId; - private final SszBitvector persistentSubnets; - - public DiscoveryPeer( - final byte[] publicKey, - final InetSocketAddress nodeAddress, - final Optional enrForkId, - final SszBitvector persistentSubnets) { - this.publicKey = publicKey; - this.nodeAddress = nodeAddress; - this.enrForkId = enrForkId; - this.persistentSubnets = persistentSubnets; - } - - public byte[] getPublicKey() { - return publicKey; - } - - public InetSocketAddress getNodeAddress() { - return nodeAddress; - } - - public Optional getEnrForkId() { - return enrForkId; - } - - public SszBitvector getPersistentSubnets() { - return persistentSubnets; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof DiscoveryPeer)) return false; - DiscoveryPeer that = (DiscoveryPeer) o; - return Objects.equal(getPublicKey(), that.getPublicKey()) - && Objects.equal(getNodeAddress(), that.getNodeAddress()) - && Objects.equal(getEnrForkId(), that.getEnrForkId()) - && Objects.equal(getPersistentSubnets(), that.getPersistentSubnets()); - } - - @Override - public int hashCode() { - return Objects.hashCode( - getPublicKey(), getNodeAddress(), getEnrForkId(), getPersistentSubnets()); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("publicKey", publicKey) - .add("nodeAddress", nodeAddress) - .add("enrForkId", enrForkId) - .add("persistentSubnets", persistentSubnets) - .toString(); - } -} diff --git a/src/org/minima/system/network/base/DiscoveryService.java b/src/org/minima/system/network/base/DiscoveryService.java index 9f87e7f0f..184eb9935 100644 --- a/src/org/minima/system/network/base/DiscoveryService.java +++ b/src/org/minima/system/network/base/DiscoveryService.java @@ -17,6 +17,7 @@ import java.util.stream.Stream; import org.apache.tuweni.bytes.Bytes; //import tech.pegasys.teku.infrastructure.async.SafeFuture; +import org.minima.system.network.base.peer.DiscoveryPeer; public interface DiscoveryService { diff --git a/src/org/minima/system/network/base/Firewall.java b/src/org/minima/system/network/base/Firewall.java new file mode 100644 index 000000000..91bcaf37e --- /dev/null +++ b/src/org/minima/system/network/base/Firewall.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import io.netty.channel.ChannelHandler.Sharable; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import io.netty.channel.WriteBufferWaterMark; +import io.netty.handler.timeout.WriteTimeoutException; +import io.netty.handler.timeout.WriteTimeoutHandler; +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +//import tech.pegasys.teku.infrastructure.async.FutureUtil; + +/** + * The very first Netty handler in the Libp2p connection pipeline. Sets up Netty Channel options and + * doing other duties preventing DoS attacks + */ +@Sharable +public class Firewall extends ChannelInboundHandlerAdapter { + private static final Logger LOG = LogManager.getLogger(); + + private final Duration writeTimeout; + + public Firewall(Duration writeTimeout) { + this.writeTimeout = writeTimeout; + } + + @Override + public void handlerAdded(ChannelHandlerContext ctx) { + ctx.channel().config().setWriteBufferWaterMark(new WriteBufferWaterMark(100, 1024)); + ctx.pipeline().addLast(new WriteTimeoutHandler(writeTimeout.toMillis(), TimeUnit.MILLISECONDS)); + ctx.pipeline().addLast(new FirewallExceptionHandler()); + } + + @Override + public void channelWritabilityChanged(ChannelHandlerContext ctx) { + ctx.channel().config().setAutoRead(ctx.channel().isWritable()); + ctx.fireChannelWritabilityChanged(); + } + + class FirewallExceptionHandler extends ChannelInboundHandlerAdapter { + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + if (cause instanceof WriteTimeoutException) { + LOG.debug("Firewall closed channel by write timeout. No writes during " + writeTimeout); + } else { + LOG.debug("Error in Firewall, disconnecting" + cause); + FutureUtil.ignoreFuture(ctx.close()); + } + } + } +} diff --git a/src/org/minima/system/network/base/LibP2PNetwork.java b/src/org/minima/system/network/base/LibP2PNetwork.java index b3912e293..7f5cb8305 100644 --- a/src/org/minima/system/network/base/LibP2PNetwork.java +++ b/src/org/minima/system/network/base/LibP2PNetwork.java @@ -26,7 +26,7 @@ import io.libp2p.core.multistream.ProtocolBinding; import io.libp2p.core.mux.StreamMuxerProtocol; import io.libp2p.etc.types.ByteArrayExtKt; -import io.libp2p.etc.util.P2PService.PeerHandler; +//import io.libp2p.etc.util.P2PService.PeerHandler; import io.libp2p.protocol.Identify; import io.libp2p.protocol.Ping; import io.libp2p.security.noise.NoiseXXSecureChannel; @@ -45,9 +45,14 @@ import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.gossip.PreparedGossipMessageFactory; +import org.minima.system.network.base.gossip.TopicChannel; +import org.minima.system.network.base.gossip.TopicHandler; +import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; import org.minima.system.network.base.libp2p.gossip.GossipTopicFilter; import org.minima.system.network.base.libp2p.gossip.LibP2PGossipNetwork; import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.DiscoveryPeer; +import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.MultiaddrPeerAddress; import org.minima.system.network.base.peer.MultiaddrUtil; import org.minima.system.network.base.peer.NodeId; @@ -76,8 +81,10 @@ import org.minima.system.network.base.peer.PeerAddress; import org.minima.system.network.base.peer.PeerConnectedSubscriber; import org.minima.system.network.base.peer.PeerManager; +import org.minima.system.network.base.peer.PeerHandler; import org.minima.system.network.base.peer.ReputationManager; import org.minima.system.network.base.peer.RpcHandler; +import org.minima.system.network.base.peer.RpcMethod; public class LibP2PNetwork implements P2PNetwork { @@ -174,7 +181,7 @@ private List> getDefaultProtocols() { IdentifyOuterClass.Identify identifyMsg = IdentifyOuterClass.Identify.newBuilder() .setProtocolVersion("ipfs/0.1.0") - .setAgentVersion(VersionProvider.CLIENT_IDENTITY + "/" + VersionProvider.VERSION) + .setAgentVersion("Minima_0.98.0-testss-p2p-Zulu-OpenJDK-11-AARCH64") .setPublicKey(ByteArrayExtKt.toProtobuf(privKey.publicKey().bytes())) .addListenAddrs(ByteArrayExtKt.toProtobuf(advertisedAddr.getBytes())) .setObservedAddr(ByteArrayExtKt.toProtobuf(advertisedAddr.getBytes())) @@ -194,7 +201,7 @@ public SafeFuture start() { return SafeFuture.of(host.start()) .thenApply( i -> { - STATUS_LOG.listeningForLibP2P(getNodeAddress()); + //STATUS_LOG.listeningForLibP2P(getNodeAddress()); return null; }); } diff --git a/src/org/minima/system/network/base/LibP2PNodeId.java b/src/org/minima/system/network/base/LibP2PNodeId.java deleted file mode 100644 index 44093cfc7..000000000 --- a/src/org/minima/system/network/base/LibP2PNodeId.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2019 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.minima.system.network.base; - -import io.libp2p.core.PeerId; -import org.apache.tuweni.bytes.Bytes; -//import tech.pegasys.teku.networking.p2p.peer.NodeId; -import org.minima.system.network.base.peer.NodeId; - -public class LibP2PNodeId extends NodeId { - private final PeerId peerId; - - public LibP2PNodeId(final PeerId peerId) { - this.peerId = peerId; - } - - @Override - public Bytes toBytes() { - return Bytes.wrap(peerId.getBytes()); - } - - @Override - public String toBase58() { - return peerId.toBase58(); - } -} diff --git a/src/org/minima/system/network/base/LibP2PParamsFactory.java b/src/org/minima/system/network/base/LibP2PParamsFactory.java index efcba028e..1fe0cfbe7 100644 --- a/src/org/minima/system/network/base/LibP2PParamsFactory.java +++ b/src/org/minima/system/network/base/LibP2PParamsFactory.java @@ -27,6 +27,7 @@ import org.minima.system.network.base.gossip.config.GossipPeerScoringConfig; import org.minima.system.network.base.gossip.config.GossipScoringConfig; import org.minima.system.network.base.gossip.config.GossipTopicScoringConfig; +import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.gossip.config.GossipTopicScoringConfig; // import tech.pegasys.teku.networking.p2p.gossip.config.GossipConfig; diff --git a/src/org/minima/system/network/base/MplexFirewall.java b/src/org/minima/system/network/base/MplexFirewall.java new file mode 100644 index 000000000..117fc8f12 --- /dev/null +++ b/src/org/minima/system/network/base/MplexFirewall.java @@ -0,0 +1,128 @@ +/* + * Copyright 2021 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import com.google.common.annotations.VisibleForTesting; +import io.libp2p.core.ChannelVisitor; +import io.libp2p.core.Connection; +import io.libp2p.etc.util.netty.mux.MuxId; +import io.libp2p.mux.MuxFrame; +import io.netty.channel.ChannelDuplexHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelPromise; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Supplier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +//import tech.pegasys.teku.infrastructure.async.FutureUtil; + +public class MplexFirewall implements ChannelVisitor { + + private static final Logger LOG = LogManager.getLogger(); + private static final long ONE_SECOND = 1000; + + private final int remoteOpenStreamsRateLimit; + private final int remoteParallelOpenStreamsLimit; + private final Supplier currentTimeSupplier; + + public MplexFirewall(int remoteOpenStreamsRateLimit, int remoteParallelOpenStreamsLimit) { + this(remoteOpenStreamsRateLimit, remoteParallelOpenStreamsLimit, System::currentTimeMillis); + } + + @VisibleForTesting + MplexFirewall( + int remoteOpenStreamsRateLimit, + int remoteParallelOpenStreamsLimit, + Supplier currentTimeSupplier) { + this.remoteOpenStreamsRateLimit = remoteOpenStreamsRateLimit; + this.remoteParallelOpenStreamsLimit = remoteParallelOpenStreamsLimit; + this.currentTimeSupplier = currentTimeSupplier; + } + + protected void remoteParallelOpenStreamLimitExceeded(MplexFirewallHandler peerMplexHandler) { + LOG.debug("Abruptly closing peer connection due to exceeding parallel open streams limit"); + FutureUtil.ignoreFuture(peerMplexHandler.getConnection().close()); + } + + protected void remoteOpenFrameRateLimitExceeded(MplexFirewallHandler peerMplexHandler) { + LOG.debug("Abruptly closing peer connection due to exceeding open mplex frame rate limit"); + FutureUtil.ignoreFuture(peerMplexHandler.getConnection().close()); + } + + @Override + public void visit(@NotNull Connection connection) { + MplexFirewallHandler firewallHandler = new MplexFirewallHandler(connection); + connection.pushHandler(firewallHandler); + } + + private class MplexFirewallHandler extends ChannelDuplexHandler { + + private final Connection connection; + private int openFrameCounter = 0; + private long startCounterTime = 0; + private final Set remoteOpenedStreamIds = new HashSet<>(); + + public MplexFirewallHandler(Connection connection) { + this.connection = connection; + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) { + MuxFrame muxFrame = (MuxFrame) msg; + boolean blockFrame = false; + if (muxFrame.getFlag() == MuxFrame.Flag.OPEN) { + remoteOpenedStreamIds.add(muxFrame.getId()); + if (remoteOpenedStreamIds.size() > remoteParallelOpenStreamsLimit) { + remoteParallelOpenStreamLimitExceeded(this); + blockFrame = true; + } + + long curTime = currentTimeSupplier.get(); + if (curTime - startCounterTime > ONE_SECOND) { + startCounterTime = curTime; + openFrameCounter = 0; + } else { + openFrameCounter++; + if (openFrameCounter > remoteOpenStreamsRateLimit) { + remoteOpenFrameRateLimitExceeded(this); + blockFrame = true; + } + } + } else if (muxFrame.getFlag() == MuxFrame.Flag.CLOSE + || muxFrame.getFlag() == MuxFrame.Flag.RESET) { + remoteOpenedStreamIds.remove(muxFrame.getId()); + } + if (!blockFrame) { + ctx.fireChannelRead(msg); + } + } + + @Override + public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) { + MuxFrame muxFrame = (MuxFrame) msg; + if (muxFrame.getFlag() == MuxFrame.Flag.RESET) { + // Track only RESET since CLOSE from local doesn't close the stream for writing from remote + remoteOpenedStreamIds.remove(muxFrame.getId()); + } + // ignoring since the write() just returns `promise` instance + FutureUtil.ignoreFuture(ctx.write(msg, promise)); + } + + public Connection getConnection() { + return connection; + } + } +} diff --git a/src/org/minima/system/network/base/NoOpDiscoveryService.java b/src/org/minima/system/network/base/NoOpDiscoveryService.java index 828c91f50..e53779867 100644 --- a/src/org/minima/system/network/base/NoOpDiscoveryService.java +++ b/src/org/minima/system/network/base/NoOpDiscoveryService.java @@ -19,6 +19,7 @@ // import tech.pegasys.teku.infrastructure.async.SafeFuture; // import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; // import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService; +import org.minima.system.network.base.peer.DiscoveryPeer; public class NoOpDiscoveryService implements DiscoveryService { diff --git a/src/org/minima/system/network/base/P2PNetwork.java b/src/org/minima/system/network/base/P2PNetwork.java index cb02cbe88..7c70519ba 100644 --- a/src/org/minima/system/network/base/P2PNetwork.java +++ b/src/org/minima/system/network/base/P2PNetwork.java @@ -23,7 +23,9 @@ // import tech.pegasys.teku.networking.p2p.peer.Peer; // import tech.pegasys.teku.networking.p2p.peer.PeerConnectedSubscriber; +import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.gossip.GossipNetwork; +import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; import org.minima.system.network.base.peer.PeerAddress; @@ -76,7 +78,7 @@ enum State { boolean isConnected(PeerAddress peerAddress); - byte[] getPrivateKey(); + Bytes getPrivateKey(); Optional getPeer(NodeId id); diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index e89f47d74..3de118165 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -1,30 +1,10 @@ package org.minima.system.network.base; -import io.libp2p.core.Host; -import io.libp2p.core.PeerId; -import io.libp2p.core.crypto.PrivKey; -import io.libp2p.core.dsl.Builder.Defaults; -import io.libp2p.core.dsl.BuilderJKt; -import io.libp2p.core.multiformats.Multiaddr; -import io.libp2p.core.multistream.ProtocolBinding; -import io.libp2p.core.mux.StreamMuxerProtocol; -import io.libp2p.etc.types.ByteArrayExtKt; -import io.libp2p.protocol.Identify; -import io.libp2p.protocol.Ping; -import io.libp2p.security.noise.NoiseXXSecureChannel; -import io.libp2p.transport.tcp.TcpTransport; - -import io.libp2p.core.Host; -import io.libp2p.core.dsl.HostBuilder; -import io.libp2p.core.multiformats.Multiaddr; -import io.libp2p.protocol.Ping; -import io.libp2p.protocol.PingController; - import java.util.concurrent.ExecutionException; // Import log4j classes. import org.apache.logging.log4j.Logger; -import org.minima.system.network.base.libp2p.gossip.LibP2PGossipNetwork; +import org.minima.system.network.base.peer.Peer; import org.apache.logging.log4j.LogManager; public class P2PStart { @@ -35,8 +15,19 @@ public class P2PStart { public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); - // LibP2PGossipNetwork gossipNet = new LibP2PGossipNetwork(new MetricsSystem()); - + // attempt 1: start with DiscoveryNetworkFactory + DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); + try { + final DiscoveryNetwork network1 = factory.builder().buildAndStart(); + System.out.println("network1 node address: " + network1.getNodeAddress()); + } catch (Exception e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // attempt 2: start with Eth2P2PNetworkFactory and Eth2P2PNetwork + + } } diff --git a/src/org/minima/system/network/base/RpcStream.java b/src/org/minima/system/network/base/RpcStream.java deleted file mode 100644 index 2113f9f3b..000000000 --- a/src/org/minima/system/network/base/RpcStream.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.minima.system.network.base; - -//import org.apache.tuweni.bytes.Bytes; -//import tech.pegasys.teku.infrastructure.async.SafeFuture; - -public interface RpcStream { - - SafeFuture writeBytes(byte[] bytes) throws StreamClosedException; - - /** - * Close the stream altogether, allowing no further reads or writes. - * - * @return A future completing when the stream is closed. - */ - SafeFuture closeAbruptly(); - - /** - * Close the write side of the stream. When both sides of the stream close their write stream, the - * entire stream will be closed. - * - * @return A future completing when the write stream is closed. - */ - SafeFuture closeWriteStream(); -} diff --git a/src/org/minima/system/network/base/StreamClosedException.java b/src/org/minima/system/network/base/StreamClosedException.java deleted file mode 100644 index 1f14ed432..000000000 --- a/src/org/minima/system/network/base/StreamClosedException.java +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright 2020 ConsenSys AG. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package org.minima.system.network.base; - -public class StreamClosedException extends RuntimeException {} diff --git a/src/org/minima/system/network/base/gossip/GossipNetwork.java b/src/org/minima/system/network/base/gossip/GossipNetwork.java index 1386f1718..0f8ab65d7 100644 --- a/src/org/minima/system/network/base/gossip/GossipNetwork.java +++ b/src/org/minima/system/network/base/gossip/GossipNetwork.java @@ -15,6 +15,8 @@ import java.util.Collection; import java.util.Map; + +import org.apache.tuweni.bytes.Bytes; //import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.SafeFuture; import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; @@ -23,7 +25,7 @@ import org.minima.system.network.base.peer.NodeId; public interface GossipNetwork { - SafeFuture gossip(String topic, byte[] data); + SafeFuture gossip(String topic, Bytes data); TopicChannel subscribe(String topic, TopicHandler topicHandler); diff --git a/src/org/minima/system/network/base/RpcMethod.java b/src/org/minima/system/network/base/libp2p/PrivateKeyGenerator.java similarity index 62% rename from src/org/minima/system/network/base/RpcMethod.java rename to src/org/minima/system/network/base/libp2p/PrivateKeyGenerator.java index 3e30f57a4..25c6f216e 100644 --- a/src/org/minima/system/network/base/RpcMethod.java +++ b/src/org/minima/system/network/base/libp2p/PrivateKeyGenerator.java @@ -1,5 +1,5 @@ /* - * Copyright 2020 ConsenSys AG. + * Copyright 2021 ConsenSys AG. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at @@ -11,13 +11,14 @@ * specific language governing permissions and limitations under the License. */ -package org.minima.system.network.base; +package org.minima.system.network.base.libp2p; -import org.minima.system.network.base.peer.RpcRequestHandler; +import io.libp2p.core.crypto.KEY_TYPE; +import io.libp2p.core.crypto.KeyKt; +import io.libp2p.core.crypto.PrivKey; -public interface RpcMethod { - - String getId(); - - RpcRequestHandler createIncomingRequestHandler(); +public class PrivateKeyGenerator { + public static PrivKey generate() { + return KeyKt.generateKeyPair(KEY_TYPE.SECP256K1).component1(); + } } diff --git a/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java b/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java index 78d307809..28959f3e6 100644 --- a/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java +++ b/src/org/minima/system/network/base/libp2p/gossip/LibP2PGossipNetwork.java @@ -48,8 +48,9 @@ import kotlin.jvm.functions.Function0; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; import org.jetbrains.annotations.NotNull; -import org.minima.system.network.base.LibP2PNodeId; +//import org.minima.system.network.base.LibP2PNodeId; import org.minima.system.network.base.LibP2PParamsFactory; import org.minima.system.network.base.SafeFuture; // import org.apache.tuweni.bytes.Bytes; @@ -75,6 +76,7 @@ import org.minima.system.network.base.gossip.config.GossipConfig; import org.minima.system.network.base.gossip.config.GossipTopicsScoringConfig; import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.utils.Crypto; @@ -178,9 +180,9 @@ public LibP2PGossipNetwork( } @Override - public SafeFuture gossip(final String topic, final byte[] data) { + public SafeFuture gossip(final String topic, final Bytes data) { return SafeFuture.of( - publisher.publish(Unpooled.wrappedBuffer(data), new Topic(topic))); + publisher.publish(Unpooled.wrappedBuffer(data.toArrayUnsafe()), new Topic(topic))); } @Override diff --git a/src/org/minima/system/network/base/peer/MultiaddrUtil.java b/src/org/minima/system/network/base/peer/MultiaddrUtil.java index 60054d816..37247c2f4 100644 --- a/src/org/minima/system/network/base/peer/MultiaddrUtil.java +++ b/src/org/minima/system/network/base/peer/MultiaddrUtil.java @@ -34,7 +34,7 @@ public static Multiaddr fromDiscoveryPeerAsUdp(final DiscoveryPeer peer) { return addPeerId(fromInetSocketAddress(peer.getNodeAddress(), "udp"), getNodeId(peer)); } - static Multiaddr fromInetSocketAddress(final InetSocketAddress address) { + public static Multiaddr fromInetSocketAddress(final InetSocketAddress address) { return fromInetSocketAddress(address, "tcp"); } diff --git a/src/org/minima/system/network/base/peer/PeerAddress.java b/src/org/minima/system/network/base/peer/PeerAddress.java index 43786e6af..7b155143a 100644 --- a/src/org/minima/system/network/base/peer/PeerAddress.java +++ b/src/org/minima/system/network/base/peer/PeerAddress.java @@ -11,11 +11,11 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.networking.p2p.network; +package org.minima.system.network.base.peer; import java.util.Objects; import java.util.Optional; -import tech.pegasys.teku.networking.p2p.peer.NodeId; +//import tech.pegasys.teku.networking.p2p.peer.NodeId; public class PeerAddress { private final NodeId id; diff --git a/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java b/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java index 9768b0341..f4d0716b6 100644 --- a/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java +++ b/src/org/minima/system/network/base/peer/PeerSelectionStrategy.java @@ -20,7 +20,7 @@ // import tech.pegasys.teku.networking.p2p.network.PeerAddress; // import tech.pegasys.teku.networking.p2p.peer.Peer; -import org.minima.system.network.base.DiscoveryPeer; +//import org.minima.system.network.base.DiscoveryPeer; import org.minima.system.network.base.P2PNetwork; public interface PeerSelectionStrategy { diff --git a/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java b/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java index 2f9a12f01..7e26b9041 100644 --- a/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java +++ b/src/org/minima/system/network/base/peer/SimplePeerSelectionStrategy.java @@ -31,7 +31,6 @@ //import org.minima.system.network.base.; import org.minima.system.network.base.P2PNetwork; -import org.minima.system.network.base.DiscoveryPeer; public class SimplePeerSelectionStrategy implements PeerSelectionStrategy { private final TargetPeerRange targetPeerRange; diff --git a/src/org/minima/system/network/base/ssz/ArrayIntCache.java b/src/org/minima/system/network/base/ssz/ArrayIntCache.java index e5984d06e..8bebcac19 100644 --- a/src/org/minima/system/network/base/ssz/ArrayIntCache.java +++ b/src/org/minima/system/network/base/ssz/ArrayIntCache.java @@ -111,4 +111,10 @@ public void invalidateInt(int key) { public void clear() { values = createArray(initSize); } + + /** Returns the current number of items in the cache */ + @Override + public int size() { + return values.length; + } } diff --git a/src/org/minima/system/network/base/ssz/Cache.java b/src/org/minima/system/network/base/ssz/Cache.java index 33c2e4234..894d4b2d4 100644 --- a/src/org/minima/system/network/base/ssz/Cache.java +++ b/src/org/minima/system/network/base/ssz/Cache.java @@ -16,6 +16,10 @@ import java.util.Optional; import java.util.function.Function; +/*** Attention */ +// This source file comes from ./infrastructure/collections/src/main/java/tech/pegasys/teku/infrastructure/collections/cache/Cache.java +// and not ./ssz/src/main/java/tech/pegasys/teku/ssz/cache/Cache.java +// the only difference is the additional size() method which then needs to be overriden in all implementing classes. /** * Cache * diff --git a/src/org/minima/system/network/base/ssz/IntCache.java b/src/org/minima/system/network/base/ssz/IntCache.java index f5cf5a320..2774f3750 100644 --- a/src/org/minima/system/network/base/ssz/IntCache.java +++ b/src/org/minima/system/network/base/ssz/IntCache.java @@ -72,4 +72,9 @@ default void invalidateWithNewValue(Integer key, V newValue) { /** Clears all cached values */ @Override void clear(); + + + /** Returns the current number of items in the cache */ + @Override + int size(); } diff --git a/src/org/minima/system/network/base/ssz/NoopIntCache.java b/src/org/minima/system/network/base/ssz/NoopIntCache.java index 6fad4afb9..b8122e86b 100644 --- a/src/org/minima/system/network/base/ssz/NoopIntCache.java +++ b/src/org/minima/system/network/base/ssz/NoopIntCache.java @@ -43,4 +43,8 @@ public Optional getCached(Integer key) { return Optional.empty(); } + + /** Returns the current number of items in the cache */ + @Override + public int size() { return INSTANCE.size(); } } From fddeaa33cfe328349e35ed8aae261b8abfd507a8 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Tue, 13 Apr 2021 12:09:14 +0100 Subject: [PATCH 12/55] p2p: added ETH2 DiscoveryV5 protocol layer running on libp2p. --- build.gradle | 17 +- .../system/network/base/DiscV5Service.java | 161 ++++++++++++++++++ .../system/network/base/DiscoveryNetwork.java | 4 +- .../network/base/NodeRecordConverter.java | 66 +++++++ .../minima/system/network/base/P2PStart.java | 52 +++++- .../network/base/peer/DiscoveryPeer.java | 7 +- .../network/base/peer/MultiaddrUtil.java | 2 +- 7 files changed, 299 insertions(+), 10 deletions(-) create mode 100644 src/org/minima/system/network/base/DiscV5Service.java create mode 100644 src/org/minima/system/network/base/NodeRecordConverter.java diff --git a/build.gradle b/build.gradle index c3ca8071d..417fa4b5c 100644 --- a/build.gradle +++ b/build.gradle @@ -32,7 +32,14 @@ repositories { // You can declare any Maven/Ivy/file repository here. jcenter() - maven { url "https://dl.cloudsmith.io/public/libp2p/jvm-libp2p/maven/" } + maven { + url "https://dl.cloudsmith.io/public/libp2p/jvm-libp2p/maven/" + // we use consensys eth discovery v5 implementation for now for nodes discovery + url "https://artifacts.consensys.net/public/maven/maven/" + } + + + } @@ -54,7 +61,7 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0-M1") - implementation 'io.libp2p:jvm-libp2p-minimal:0.8.0-RELEASE' + implementation 'io.libp2p:jvm-libp2p-minimal:0.7.0-RELEASE' // implementation 'io.libp2p:jvm-libp2p-minimal' // compile files('libs/jvm-libp2p-minimal-0.8.0-RELEASE.jar') @@ -69,9 +76,15 @@ dependencies { implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0' // not sure we need the below, as we imported teku fork of tuweni ssz... But maybe we could switch to it. implementation group: 'org.apache.tuweni', name: 'tuweni-ssz', version: '1.1.0', ext: 'pom' + // https://mvnrepository.com/artifact/org.apache.tuweni/tuweni-units - needed for eth discovery v5 lib (tech.pegasys.discovery) + //implementation group: 'org.apache.tuweni', name: 'tuweni-units', version: '1.1.0', ext: 'pom' + implementation("org.apache.tuweni:units:1.3.0") // awaitility is used by Waiter.java implementation group: 'org.awaitility', name: 'awaitility', version: '4.0.3' + implementation 'tech.pegasys.discovery:discovery:0.4.3-dev-57c2fd81' + + } diff --git a/src/org/minima/system/network/base/DiscV5Service.java b/src/org/minima/system/network/base/DiscV5Service.java new file mode 100644 index 000000000..69585f84f --- /dev/null +++ b/src/org/minima/system/network/base/DiscV5Service.java @@ -0,0 +1,161 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +//import static tech.pegasys.teku.util.config.Constants.ATTESTATION_SUBNET_COUNT; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import org.apache.tuweni.bytes.Bytes; +import org.minima.system.network.base.peer.DiscoveryPeer; +import org.minima.system.network.base.peer.MultiaddrUtil; +import org.minima.system.network.base.ssz.SszBitvector; +import org.apache.tuweni.units.bigints.UInt64; +import org.ethereum.beacon.discovery.DiscoverySystem; +import org.ethereum.beacon.discovery.DiscoverySystemBuilder; +import org.ethereum.beacon.discovery.schema.EnrField; +import org.ethereum.beacon.discovery.schema.NodeRecord; +import org.ethereum.beacon.discovery.schema.NodeRecordBuilder; +import org.ethereum.beacon.discovery.schema.NodeRecordInfo; +import org.ethereum.beacon.discovery.schema.NodeStatus; +import org.ethereum.beacon.discovery.storage.NewAddressHandler; +//import tech.pegasys.teku.infrastructure.async.SafeFuture; + +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryConfig; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryPeer; +// import tech.pegasys.teku.networking.p2p.discovery.DiscoveryService; +// import tech.pegasys.teku.networking.p2p.libp2p.MultiaddrUtil; +// import tech.pegasys.teku.networking.p2p.network.config.NetworkConfig; +// import tech.pegasys.teku.service.serviceutils.Service; +// import tech.pegasys.teku.ssz.collections.SszBitvector; +// import tech.pegasys.teku.ssz.schema.collections.SszBitvectorSchema; +// import tech.pegasys.teku.storage.store.KeyValueStore; +import org.minima.system.network.base.ssz.SszBitvectorSchema; + +public class DiscV5Service extends Service implements DiscoveryService { + private static final String SEQ_NO_STORE_KEY = "local-enr-seqno"; + static final SszBitvectorSchema SUBNET_SUBSCRIPTIONS_SCHEMA = + SszBitvectorSchema.create(DiscoveryNetwork.ATTESTATION_SUBNET_COUNT); + + public static DiscoveryService create( + final DiscoveryConfig discoConfig, + final NetworkConfig p2pConfig, + final KeyValueStore kvStore, + final Bytes privateKey) { + return new DiscV5Service(discoConfig, p2pConfig, kvStore, privateKey); + } + + private final DiscoverySystem discoverySystem; + private final KeyValueStore kvStore; + + private DiscV5Service( + final DiscoveryConfig discoConfig, + NetworkConfig p2pConfig, + KeyValueStore kvStore, + final Bytes privateKey) { + final String listenAddress = p2pConfig.getNetworkInterface(); + final int listenPort = p2pConfig.getListenPort(); + final String advertisedAddress = p2pConfig.getAdvertisedIp(); + final int advertisedPort = p2pConfig.getAdvertisedPort(); + final List bootnodes = discoConfig.getBootnodes(); + final UInt64 seqNo = + kvStore.get(SEQ_NO_STORE_KEY).map(UInt64::fromBytes).orElse(UInt64.ZERO).add(1); + final NewAddressHandler maybeUpdateNodeRecordHandler = + maybeUpdateNodeRecord(p2pConfig.hasUserExplicitlySetAdvertisedIp()); + discoverySystem = + new DiscoverySystemBuilder() + .listen(listenAddress, listenPort) + .privateKey(privateKey) + .bootnodes(bootnodes.toArray(new String[0])) + .localNodeRecord( + new NodeRecordBuilder() + .privateKey(privateKey) + .address(advertisedAddress, advertisedPort) + .seq(seqNo) + .build()) + .newAddressHandler(maybeUpdateNodeRecordHandler) + .localNodeRecordListener(this::localNodeRecordUpdated) + .build(); + this.kvStore = kvStore; + } + + private NewAddressHandler maybeUpdateNodeRecord(boolean userExplicitlySetAdvertisedIpOrPort) { + return (oldRecord, proposedNewRecord) -> { + if (userExplicitlySetAdvertisedIpOrPort) { + return Optional.of(oldRecord); + } else { + return Optional.of(proposedNewRecord); + } + }; + } + + private void localNodeRecordUpdated(NodeRecord oldRecord, NodeRecord newRecord) { + kvStore.put(SEQ_NO_STORE_KEY, newRecord.getSeq().toBytes()); + } + + @Override + protected SafeFuture doStart() { + return SafeFuture.of(discoverySystem.start()); + } + + @Override + protected SafeFuture doStop() { + discoverySystem.stop(); + return SafeFuture.completedFuture(null); + } + + @Override + public Stream streamKnownPeers() { + return activeNodes().map(NodeRecordConverter::convertToDiscoveryPeer).flatMap(Optional::stream); + } + + @Override + public SafeFuture searchForPeers() { + return SafeFuture.of(discoverySystem.searchForNewPeers()); + } + + @Override + public Optional getEnr() { + return Optional.of(discoverySystem.getLocalNodeRecord().asEnr()); + } + + @Override + public Optional getDiscoveryAddress() { + final NodeRecord nodeRecord = discoverySystem.getLocalNodeRecord(); + if (nodeRecord.getUdpAddress().isEmpty()) { + return Optional.empty(); + } + final DiscoveryPeer discoveryPeer = + new DiscoveryPeer( + (Bytes) nodeRecord.get(EnrField.PKEY_SECP256K1), + nodeRecord.getUdpAddress().get(), + Optional.empty(), + SUBNET_SUBSCRIPTIONS_SCHEMA.getDefault()); + + return Optional.of(MultiaddrUtil.fromDiscoveryPeerAsUdp(discoveryPeer).toString()); + } + + @Override + public void updateCustomENRField(String fieldName, Bytes value) { + discoverySystem.updateCustomFieldValue(fieldName, value); + } + + private Stream activeNodes() { + return discoverySystem + .streamKnownNodes() + .filter(record -> record.getStatus() == NodeStatus.ACTIVE) + .map(NodeRecordInfo::getNode); + } +} diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index 7377c94fb..50fec6337 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -122,8 +122,8 @@ private static DiscoveryService createDiscoveryService( final DiscoveryService discoveryService; if (discoConfig.isDiscoveryEnabled()) { //TODO: activate V5 service - // discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); - discoveryService = new NoOpDiscoveryService(); + discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); + //discoveryService = new NoOpDiscoveryService(); } else { discoveryService = new NoOpDiscoveryService(); } diff --git a/src/org/minima/system/network/base/NodeRecordConverter.java b/src/org/minima/system/network/base/NodeRecordConverter.java new file mode 100644 index 000000000..e078f2464 --- /dev/null +++ b/src/org/minima/system/network/base/NodeRecordConverter.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 ConsenSys AG. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.minima.system.network.base; + +import static org.minima.system.network.base.DiscoveryNetwork.ATTESTATION_SUBNET_ENR_FIELD; +import static org.minima.system.network.base.DiscoveryNetwork.ETH2_ENR_FIELD; + +import java.net.InetSocketAddress; +import java.util.Optional; +import java.util.function.Function; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes; +import org.ethereum.beacon.discovery.schema.EnrField; +import org.ethereum.beacon.discovery.schema.NodeRecord; +import org.minima.system.network.base.peer.DiscoveryPeer; +//import org.minima.system.network.base.EnrForkId; +import org.minima.system.network.base.ssz.SszBitvector; + +public class NodeRecordConverter { + private static final Logger LOG = LogManager.getLogger(); + + static Optional convertToDiscoveryPeer(final NodeRecord nodeRecord) { + return nodeRecord + .getTcpAddress() + .map(address -> socketAddressToDiscoveryPeer(nodeRecord, address)); + } + + private static DiscoveryPeer socketAddressToDiscoveryPeer( + final NodeRecord nodeRecord, final InetSocketAddress address) { + + final Optional enrForkId = + parseField(nodeRecord, ETH2_ENR_FIELD, EnrForkId.SSZ_SCHEMA::sszDeserialize); + + final SszBitvector persistentSubnets = + parseField( + nodeRecord, + ATTESTATION_SUBNET_ENR_FIELD, + DiscV5Service.SUBNET_SUBSCRIPTIONS_SCHEMA::fromBytes) + .orElse(DiscV5Service.SUBNET_SUBSCRIPTIONS_SCHEMA.getDefault()); + + return new DiscoveryPeer( + ((Bytes) nodeRecord.get(EnrField.PKEY_SECP256K1)), address, enrForkId, persistentSubnets); + } + + private static Optional parseField( + final NodeRecord nodeRecord, final String fieldName, final Function parse) { + try { + return Optional.ofNullable((Bytes) nodeRecord.get(fieldName)).map(parse); + } catch (final Exception e) { + LOG.debug("Failed to parse ENR field {}", fieldName, e); + return Optional.empty(); + } + } +} diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 3de118165..41974bbf5 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -1,10 +1,17 @@ package org.minima.system.network.base; +import java.util.Optional; import java.util.concurrent.ExecutionException; // Import log4j classes. import org.apache.logging.log4j.Logger; +import org.minima.system.network.base.peer.LibP2PNodeId; +import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; + +import io.libp2p.core.PeerId; +import io.libp2p.etc.encode.Base58; + import org.apache.logging.log4j.LogManager; public class P2PStart { @@ -18,8 +25,49 @@ public static void main(String[] args) // attempt 1: start with DiscoveryNetworkFactory DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { - final DiscoveryNetwork network1 = factory.builder().buildAndStart(); - System.out.println("network1 node address: " + network1.getNodeAddress()); + final DiscoveryNetwork network; + if (args.length == 1) { // first is p2p addr + System.out.println("Found addr: " + args[0]); + String[] node1_addr_fields = args[0].split("/"); + String node1_id = node1_addr_fields[node1_addr_fields.length-1]; + + System.out.println("Found node1_id: " + node1_id); + // Multiaddr address = Multiaddr.fromString(args[0]); + network = factory.builder().staticPeer(args[0]).buildAndStart(); + //network.getEnr() + //network = factory.builder().bootnode(args[0]) + Thread.sleep(5000); + //Peer peer = network.getPeer(args[0]); + int peerCount = network.getPeerCount(); + String addr = network.getNodeAddress(); + NodeId id = network.getNodeId(); + System.out.println("peerCount = " + peerCount); + System.out.println("addr = " + addr); + System.out.println("id = " + id.toString()); + //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); + + LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); + Thread.sleep(5000); + Waiter.waitFor( + () -> { + Optional firstNode = network.getPeer(idfirst); + if(firstNode.isPresent()) { + System.out.println("Success! We found the first node."); + } else { + System.out.println("Not yet there."); + } + System.out.println("peerCount = " + peerCount); + + }); + + + } else { + System.out.println("Creating new address..."); + network = factory.builder().buildAndStart(); + NodeId id = network.getNodeId(); + System.out.println("id = " + id.toString()); + } + System.out.println("network node address: " + network.getNodeAddress()); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); diff --git a/src/org/minima/system/network/base/peer/DiscoveryPeer.java b/src/org/minima/system/network/base/peer/DiscoveryPeer.java index aedfc4322..44385c154 100644 --- a/src/org/minima/system/network/base/peer/DiscoveryPeer.java +++ b/src/org/minima/system/network/base/peer/DiscoveryPeer.java @@ -16,6 +16,7 @@ import com.google.common.base.MoreObjects; import com.google.common.base.Objects; +import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.EnrForkId; import org.minima.system.network.base.ssz.SszBitvector; @@ -26,13 +27,13 @@ // import tech.pegasys.teku.ssz.collections.SszBitvector; public class DiscoveryPeer { - private final byte[] publicKey; + private final Bytes publicKey; private final InetSocketAddress nodeAddress; private final Optional enrForkId; private final SszBitvector persistentSubnets; public DiscoveryPeer( - final byte[] publicKey, + final Bytes publicKey, final InetSocketAddress nodeAddress, final Optional enrForkId, final SszBitvector persistentSubnets) { @@ -42,7 +43,7 @@ public DiscoveryPeer( this.persistentSubnets = persistentSubnets; } - public byte[] getPublicKey() { + public Bytes getPublicKey() { return publicKey; } diff --git a/src/org/minima/system/network/base/peer/MultiaddrUtil.java b/src/org/minima/system/network/base/peer/MultiaddrUtil.java index 37247c2f4..a39358e74 100644 --- a/src/org/minima/system/network/base/peer/MultiaddrUtil.java +++ b/src/org/minima/system/network/base/peer/MultiaddrUtil.java @@ -59,7 +59,7 @@ private static Multiaddr addPeerId(final Multiaddr addr, final NodeId nodeId) { } private static LibP2PNodeId getNodeId(final DiscoveryPeer peer) { - final PubKey pubKey = unmarshalSecp256k1PublicKey(peer.getPublicKey()); + final PubKey pubKey = unmarshalSecp256k1PublicKey(peer.getPublicKey().toArray()); return new LibP2PNodeId(PeerId.fromPubKey(pubKey)); } From 4c4dc0438e1e4349dd2e958da44ada78548c9e98 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 15 Apr 2021 15:57:03 +0100 Subject: [PATCH 13/55] p2p: added 3rd node scenario. --- .../minima/system/network/base/P2PStart.java | 75 +++++++++++++++---- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 41974bbf5..9b31dcdc4 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -26,45 +26,92 @@ public static void main(String[] args) DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { final DiscoveryNetwork network; - if (args.length == 1) { // first is p2p addr + String node1_id; + String[] node1_addr_fields; + String node2_id; + String[] node2_addr_fields; + final int peerCount; + String addr; + NodeId id; + + if (args.length == 2) { // first is p2p addr + System.out.println("Found addr node 1: " + args[0]); + System.out.println("Found addr node 2: " + args[1]); + node1_addr_fields = args[0].split("/"); + node1_id = node1_addr_fields[node1_addr_fields.length-1]; + node2_addr_fields = args[1].split("/"); + node2_id = node2_addr_fields[node2_addr_fields.length-1]; + System.out.println("Found node2_id: " + node1_id); + // Multiaddr address = Multiaddr.fromString(args[0]); + network = factory.builder().staticPeer(args[0]).buildAndStart(); + //network.getEnr() + //network = factory.builder().bootnode(args[0]) + Thread.sleep(5000); + //Peer peer = network.getPeer(args[0]); + peerCount = network.getPeerCount(); + addr = network.getNodeAddress(); + id = network.getNodeId(); + System.out.println("peerCount = " + peerCount); + System.out.println("addr = " + addr); + System.out.println("id = " + id.toString()); + //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); + + LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); + LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); + //Thread.sleep(5000); + Waiter.waitFor( + () -> { + Optional firstNode = network.getPeer(id_1); + if(firstNode.isPresent()) { + System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); + } else { + System.out.println("First node not found."); + } + Optional secondNode = network.getPeer(id_2); + if(secondNode.isPresent()) { + System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); + } else { + System.out.println("Second node not found."); + } + System.out.println("peerCount = " + peerCount); + }); + } else if (args.length == 1) { // first is p2p addr System.out.println("Found addr: " + args[0]); - String[] node1_addr_fields = args[0].split("/"); - String node1_id = node1_addr_fields[node1_addr_fields.length-1]; + node1_addr_fields = args[0].split("/"); + node1_id = node1_addr_fields[node1_addr_fields.length-1]; System.out.println("Found node1_id: " + node1_id); // Multiaddr address = Multiaddr.fromString(args[0]); network = factory.builder().staticPeer(args[0]).buildAndStart(); //network.getEnr() //network = factory.builder().bootnode(args[0]) - Thread.sleep(5000); + Thread.sleep(1000); //Peer peer = network.getPeer(args[0]); - int peerCount = network.getPeerCount(); - String addr = network.getNodeAddress(); - NodeId id = network.getNodeId(); + peerCount = network.getPeerCount(); + addr = network.getNodeAddress(); + id = network.getNodeId(); System.out.println("peerCount = " + peerCount); System.out.println("addr = " + addr); System.out.println("id = " + id.toString()); //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - Thread.sleep(5000); + //Thread.sleep(5000); Waiter.waitFor( () -> { Optional firstNode = network.getPeer(idfirst); if(firstNode.isPresent()) { - System.out.println("Success! We found the first node."); + System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); } else { - System.out.println("Not yet there."); + System.out.println("First node not found."); } System.out.println("peerCount = " + peerCount); - }); - - + }); } else { System.out.println("Creating new address..."); network = factory.builder().buildAndStart(); - NodeId id = network.getNodeId(); + id = network.getNodeId(); System.out.println("id = " + id.toString()); } System.out.println("network node address: " + network.getNodeAddress()); From 8b8c8d03c6df49591ae54d781a92b054e6346b37 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 21 Apr 2021 17:59:56 +0100 Subject: [PATCH 14/55] p2p: added log4j2, config file, and extended P2PStart. --- build.gradle | 15 +++- resources/log4j2.xml | 14 ++++ .../system/network/base/DiscoveryNetwork.java | 9 +- .../network/base/DiscoveryNetworkFactory.java | 4 +- .../minima/system/network/base/P2PStart.java | 84 +++++++++++++++---- 5 files changed, 100 insertions(+), 26 deletions(-) create mode 100644 resources/log4j2.xml diff --git a/build.gradle b/build.gradle index 417fa4b5c..a099b691d 100644 --- a/build.gradle +++ b/build.gradle @@ -38,9 +38,6 @@ repositories { url "https://artifacts.consensys.net/public/maven/maven/" } - - - } dependencies { @@ -69,6 +66,11 @@ dependencies { implementation("org.apache.logging.log4j:log4j-api:2.11.2") implementation("org.apache.logging.log4j:log4j-core:2.11.2") + + // below is needed to actually capture the log messages from log4j + implementation group: 'org.slf4j', name: 'slf4j-simple', version: '2.0.0-alpha1' // '1.7.30' + + //implementation("org.slf4j:slf4j-api:2.0.0-alpha1") // Bytes object needed for collections such as KVS implementation("org.apache.tuweni:bytes:1.3.0") @@ -126,7 +128,12 @@ task(runp2p, dependsOn: 'classes', type: JavaExec) { classpath = sourceSets.main.runtimeClasspath // args ''/ip4/127.0.0.1/tcp/63407/ipfs/QmRGduDqyGXEsAGxAw9gM6tZrJbg1NEKSvmqwnjQqKwRVk' // use this line to hardcode args // systemProperty 'simple.message', 'Hello ' - // systemProperty 'log4j2.debug', 'true' + systemProperty 'log4j2.debug', 'true' +// systemProperty 'log4j.configurationFile', 'log4j2-p2p.properties' + systemProperty 'log4j.configurationFile', 'log4j2.xml' +// use below to log to file +// systemProperty "log4j.configurationFile", "log4j2-test-discovery.xml" + } jar { diff --git a/resources/log4j2.xml b/resources/log4j2.xml new file mode 100644 index 000000000..609151a79 --- /dev/null +++ b/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index 50fec6337..f00eb6615 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -26,7 +26,7 @@ import org.apache.logging.log4j.Logger; import org.minima.system.network.base.metrics.MetricsSystem; import org.minima.system.network.base.peer.NodeId; -//import org.apache.logging.log4j.status.StatusLogger; +import org.apache.logging.log4j.status.StatusLogger; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; // import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -121,10 +121,11 @@ private static DiscoveryService createDiscoveryService( final Bytes privateKey) { final DiscoveryService discoveryService; if (discoConfig.isDiscoveryEnabled()) { - //TODO: activate V5 service + System.out.println("P2P: Starting DiscV5 service"); discoveryService = DiscV5Service.create(discoConfig, p2pConfig, kvStore, privateKey); //discoveryService = new NoOpDiscoveryService(); } else { + System.out.println("P2P: Starting NoOp Disc service"); discoveryService = new NoOpDiscoveryService(); } return discoveryService; @@ -135,8 +136,8 @@ private static DiscoveryService createDiscoveryService( public SafeFuture start() { return SafeFuture.allOfFailFast(p2pNetwork.start(), discoveryService.start()) .thenCompose(__ -> connectionManager.start()) - .thenRun(() -> getEnr()); // TODO: log ENR info and discovery start - } // ifPresent(StatusLogger.STATUS_LOG::listeningForDiscv5) + .thenRun(() -> getEnr().ifPresent( enr -> { LOG.warn("logwarn: listening for discv5: " + enr); System.out.println("sysout: listening for discv5: " + enr);})); // TODO: log ENR info and discovery start + } //::listeningForDiscv5 @Override public SafeFuture stop() { diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index ae3daf866..61fbf80d1 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -99,7 +99,7 @@ public DiscoveryNetwork buildAndStart() throws Exception { final DiscoveryConfig discoveryConfig = DiscoveryConfig.builder().staticPeers(staticPeers).bootnodes(bootnodes).build(); final NetworkConfig config = - NetworkConfig.builder().listenPort(port).networkInterface("127.0.0.1").build(); + NetworkConfig.builder().listenPort(port).networkInterface("0.0.0.0").build(); final NoOpMetricsSystem metricsSystem = new NoOpMetricsSystem(); final ReputationManager reputationManager = new ReputationManager( @@ -129,7 +129,7 @@ public DiscoveryNetwork buildAndStart() throws Exception { discoveryConfig, config); try { - network.start().get(30, TimeUnit.SECONDS); + network.start().get(5, TimeUnit.SECONDS); networks.add(network); return network; } catch (final ExecutionException e) { diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 9b31dcdc4..c1923aff9 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -1,10 +1,20 @@ package org.minima.system.network.base; +import java.net.InetSocketAddress; +import java.util.HashSet; +import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +// Import log4j classes. // Import log4j classes. import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; +import org.ethereum.beacon.discovery.schema.NodeRecord; +import org.ethereum.beacon.discovery.schema.NodeRecordInfo; import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; @@ -24,8 +34,9 @@ public static void main(String[] args) System.out.println("Hello world!"); // attempt 1: start with DiscoveryNetworkFactory DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); + final DiscoveryNetwork network; try { - final DiscoveryNetwork network; + String node1_id; String[] node1_addr_fields; String node2_id; @@ -34,16 +45,18 @@ public static void main(String[] args) String addr; NodeId id; - if (args.length == 2) { // first is p2p addr + if (args.length == 2) { // first is p2p addr, second is enr System.out.println("Found addr node 1: " + args[0]); - System.out.println("Found addr node 2: " + args[1]); + System.out.println("Found boot node 1: " + args[1]); node1_addr_fields = args[0].split("/"); node1_id = node1_addr_fields[node1_addr_fields.length-1]; - node2_addr_fields = args[1].split("/"); - node2_id = node2_addr_fields[node2_addr_fields.length-1]; - System.out.println("Found node2_id: " + node1_id); +// node2_addr_fields = args[1].split("/"); +// node2_id = node2_addr_fields[node2_addr_fields.length-1]; +// System.out.println("Found node2_id: " + node1_id); + String bootnode_1 = args[1]; + System.out.println("Found boot node addr: " + bootnode_1); // Multiaddr address = Multiaddr.fromString(args[0]); - network = factory.builder().staticPeer(args[0]).buildAndStart(); + network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(); //network.getEnr() //network = factory.builder().bootnode(args[0]) Thread.sleep(5000); @@ -57,7 +70,8 @@ public static void main(String[] args) //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); + +// LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); //Thread.sleep(5000); Waiter.waitFor( () -> { @@ -67,12 +81,12 @@ public static void main(String[] args) } else { System.out.println("First node not found."); } - Optional secondNode = network.getPeer(id_2); - if(secondNode.isPresent()) { - System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); - } else { - System.out.println("Second node not found."); - } +// Optional secondNode = network.getPeer(id_2); + // if(secondNode.isPresent()) { + // System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); + // } else { + // System.out.println("Second node not found."); + // } System.out.println("peerCount = " + peerCount); }); } else if (args.length == 1) { // first is p2p addr @@ -112,16 +126,54 @@ public static void main(String[] args) System.out.println("Creating new address..."); network = factory.builder().buildAndStart(); id = network.getNodeId(); + Optional discAddr = network.getDiscoveryAddress(); + Optional enr = network.getEnr(); System.out.println("id = " + id.toString()); + System.out.println("discAddr = " + discAddr.toString()); + System.out.println("enr = " + enr.get().substring(3)); + } System.out.println("network node address: " + network.getNodeAddress()); + Optional discAddr = network.getDiscoveryAddress(); + System.out.println("network discovery address: " + discAddr.get()); + System.out.println("network node id: " + network.getNodeId()); + logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + " , discovery address: " + discAddr.get() ); + System.out.println("Starting discovery loop info"); + + if(network != null) { + Set activeKnownNodes = new HashSet<>(); + while (true) { + // List newActiveNodes = + network + .streamPeers() + .filter(peer -> peer.getId() != null) + .forEach( + peer -> { + System.out.println("peer " + peer.getId()); + System.out.println("peer " + peer.getAddress().toString()); + } + ); + // .map(NodeRecordInfo::getNode) + // .filter(r -> !activeKnownNodes.contains(r)) + // .collect(Collectors.toList()); + + // activeKnownNodes.addAll(newActiveNodes); + // newActiveNodes.forEach( + // n -> { + // System.out.println( + // "New active node: " + // + n.getNodeId() + // + " @ " + // + n.getUdpAddress().map(InetSocketAddress::toString).orElse("")); + // }); + Thread.sleep(5000); + } + } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } - // attempt 2: start with Eth2P2PNetworkFactory and Eth2P2PNetwork - } From 652221188e0c33a67ee66855e0b997916bd96ab4 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 22 Apr 2021 10:51:43 +0100 Subject: [PATCH 15/55] p2p: discovery network with traces and periodic peers logging. --- build.gradle | 2 +- src/log4j.properties | 8 ++ src/log4j2.xml | 14 ++ .../network/base/ConnectionManager.java | 10 +- .../system/network/base/DiscoveryNetwork.java | 11 ++ .../system/network/base/LibP2PNetwork.java | 1 + .../minima/system/network/base/P2PStart.java | 123 ++++++++++-------- .../system/network/base/peer/PeerManager.java | 2 +- 8 files changed, 107 insertions(+), 64 deletions(-) create mode 100644 src/log4j.properties create mode 100644 src/log4j2.xml diff --git a/build.gradle b/build.gradle index a099b691d..66b5647ee 100644 --- a/build.gradle +++ b/build.gradle @@ -128,7 +128,7 @@ task(runp2p, dependsOn: 'classes', type: JavaExec) { classpath = sourceSets.main.runtimeClasspath // args ''/ip4/127.0.0.1/tcp/63407/ipfs/QmRGduDqyGXEsAGxAw9gM6tZrJbg1NEKSvmqwnjQqKwRVk' // use this line to hardcode args // systemProperty 'simple.message', 'Hello ' - systemProperty 'log4j2.debug', 'true' + systemProperty 'log4j2.debug', 'false' // systemProperty 'log4j.configurationFile', 'log4j2-p2p.properties' systemProperty 'log4j.configurationFile', 'log4j2.xml' // use below to log to file diff --git a/src/log4j.properties b/src/log4j.properties new file mode 100644 index 000000000..9a538dcc7 --- /dev/null +++ b/src/log4j.properties @@ -0,0 +1,8 @@ +# This sets the global logging level and specifies the appenders +log4j.rootLogger=INFO, theConsoleAppender + +# settings for the console appender +log4j.appender.theConsoleAppender=org.apache.log4j.ConsoleAppender +log4j.appender.theConsoleAppender.layout=org.apache.log4j.PatternLayout +log4j.appender.theConsoleAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + diff --git a/src/log4j2.xml b/src/log4j2.xml new file mode 100644 index 000000000..609151a79 --- /dev/null +++ b/src/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/org/minima/system/network/base/ConnectionManager.java b/src/org/minima/system/network/base/ConnectionManager.java index da1f7fc82..8bbc6224d 100644 --- a/src/org/minima/system/network/base/ConnectionManager.java +++ b/src/org/minima/system/network/base/ConnectionManager.java @@ -122,10 +122,10 @@ private void connectToKnownPeers() { private void searchForPeers() { if (!isRunning()) { - LOG.trace("Not running so not searching for peers"); + LOG.debug("Not running so not searching for peers"); return; } - LOG.trace("Searching for peers"); + LOG.debug("Searching for peers"); discoveryService .searchForPeers() .orTimeout(10, TimeUnit.SECONDS) @@ -138,19 +138,19 @@ private void searchForPeers() { } private void attemptConnection(final PeerAddress peerAddress) { - LOG.trace("Attempting to connect to {}", peerAddress.getId()); + LOG.debug("Attempting to connect to {}", peerAddress.getId()); attemptedConnectionCounter.inc(); network .connect(peerAddress) .finish( peer -> { - LOG.trace("Successfully connected to peer {}", peer.getId()); + LOG.debug("Successfully connected to peer {}", peer.getId()); successfulConnectionCounter.inc(); peer.subscribeDisconnect( (reason, locallyInitiated) -> peerPools.forgetPeer(peer.getId())); }, error -> { - LOG.trace(() -> "Failed to connect to peer: " + peerAddress.getId(), error); + LOG.debug(() -> "Failed to connect to peer: " + peerAddress.getId(), error); failedConnectionCounter.inc(); peerPools.forgetPeer(peerAddress.getId()); }); diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index f00eb6615..60bad80e0 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -25,6 +25,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.minima.system.network.base.metrics.MetricsSystem; +import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.NodeId; import org.apache.logging.log4j.status.StatusLogger; import org.apache.tuweni.bytes.Bytes; @@ -231,5 +232,15 @@ public Optional

getPeer(final NodeId id) { public Stream

streamPeers() { return p2pNetwork.streamPeers(); } + + public Stream streamKnownDiscoveryPeers() { + return discoveryService.streamKnownPeers(); + } + + public int getP2PPeerCount() { + LibP2PNetwork net = (LibP2PNetwork) p2pNetwork; + return net.getPeerCount(); + } + } diff --git a/src/org/minima/system/network/base/LibP2PNetwork.java b/src/org/minima/system/network/base/LibP2PNetwork.java index 7f5cb8305..e96b96982 100644 --- a/src/org/minima/system/network/base/LibP2PNetwork.java +++ b/src/org/minima/system/network/base/LibP2PNetwork.java @@ -202,6 +202,7 @@ public SafeFuture start() { .thenApply( i -> { //STATUS_LOG.listeningForLibP2P(getNodeAddress()); + LOG.debug("Listening for LibP2P - " + getNodeAddress()); return null; }); } diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index c1923aff9..b7353b344 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -54,43 +54,44 @@ public static void main(String[] args) // node2_id = node2_addr_fields[node2_addr_fields.length-1]; // System.out.println("Found node2_id: " + node1_id); String bootnode_1 = args[1]; - System.out.println("Found boot node addr: " + bootnode_1); + // System.out.println("Found boot node addr: " + bootnode_1); // Multiaddr address = Multiaddr.fromString(args[0]); network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(); //network.getEnr() //network = factory.builder().bootnode(args[0]) - Thread.sleep(5000); + //Thread.sleep(5000); //Peer peer = network.getPeer(args[0]); - peerCount = network.getPeerCount(); - addr = network.getNodeAddress(); - id = network.getNodeId(); - System.out.println("peerCount = " + peerCount); - System.out.println("addr = " + addr); - System.out.println("id = " + id.toString()); + // peerCount = network.getPeerCount(); + // addr = network.getNodeAddress(); + // id = network.getNodeId(); + // // System.out.println("peerCount = " + peerCount); + // System.out.println("addr = " + addr); + // System.out.println("id = " + id.toString()); //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); // LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); //Thread.sleep(5000); - Waiter.waitFor( - () -> { - Optional firstNode = network.getPeer(id_1); - if(firstNode.isPresent()) { - System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); - } else { - System.out.println("First node not found."); - } -// Optional secondNode = network.getPeer(id_2); - // if(secondNode.isPresent()) { - // System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); - // } else { - // System.out.println("Second node not found."); - // } - System.out.println("peerCount = " + peerCount); - }); - } else if (args.length == 1) { // first is p2p addr - System.out.println("Found addr: " + args[0]); +// Waiter.waitFor( +// () -> { +// Optional firstNode = network.getPeer(id_1); +// if(firstNode.isPresent()) { +// System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); +// } else { +// System.out.println("First node not found."); +// } +// // Optional secondNode = network.getPeer(id_2); +// // if(secondNode.isPresent()) { +// // System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); +// // } else { +// // System.out.println("Second node not found."); +// // } +// System.out.println("peerCount = " + peerCount); +// }); + } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR (application level address) + logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); + System.out.println("Found addr: " + args[0]); node1_addr_fields = args[0].split("/"); node1_id = node1_addr_fields[node1_addr_fields.length-1]; @@ -99,44 +100,44 @@ public static void main(String[] args) network = factory.builder().staticPeer(args[0]).buildAndStart(); //network.getEnr() //network = factory.builder().bootnode(args[0]) - Thread.sleep(1000); +// Thread.sleep(1000); //Peer peer = network.getPeer(args[0]); - peerCount = network.getPeerCount(); - addr = network.getNodeAddress(); - id = network.getNodeId(); - System.out.println("peerCount = " + peerCount); - System.out.println("addr = " + addr); - System.out.println("id = " + id.toString()); + // peerCount = network.getPeerCount(); + // addr = network.getNodeAddress(); + // id = network.getNodeId(); +// System.out.println("peerCount = " + peerCount); +// System.out.println("addr = " + addr); +// System.out.println("id = " + id.toString()); //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); - LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); + //LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); //Thread.sleep(5000); - Waiter.waitFor( - () -> { - Optional firstNode = network.getPeer(idfirst); - if(firstNode.isPresent()) { - System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); - } else { - System.out.println("First node not found."); - } - System.out.println("peerCount = " + peerCount); - - }); + // Waiter.waitFor( + // () -> { + // Optional firstNode = network.getPeer(idfirst); + // if(firstNode.isPresent()) { + // System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); + // } else { + // System.out.println("First node not found."); + // } + // System.out.println("peerCount = " + peerCount); + + // }); } else { - System.out.println("Creating new address..."); + // server mode only - no peer to connect to network = factory.builder().buildAndStart(); id = network.getNodeId(); - Optional discAddr = network.getDiscoveryAddress(); - Optional enr = network.getEnr(); - System.out.println("id = " + id.toString()); - System.out.println("discAddr = " + discAddr.toString()); - System.out.println("enr = " + enr.get().substring(3)); + //Optional discAddr = network.getDiscoveryAddress(); + //Optional enr = network.getEnr(); + //System.out.println("id = " + id.toString()); + //System.out.println("discAddr = " + discAddr.toString()); + //System.out.println("enr = " + enr.get().substring(3)); } - System.out.println("network node address: " + network.getNodeAddress()); +// System.out.println("network node address: " + network.getNodeAddress()); Optional discAddr = network.getDiscoveryAddress(); - System.out.println("network discovery address: " + discAddr.get()); - System.out.println("network node id: " + network.getNodeId()); +// System.out.println("network discovery address: " + discAddr.get()); +// System.out.println("network node id: " + network.getNodeId()); logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + " , discovery address: " + discAddr.get() ); System.out.println("Starting discovery loop info"); @@ -149,14 +150,22 @@ public static void main(String[] args) .filter(peer -> peer.getId() != null) .forEach( peer -> { - System.out.println("peer " + peer.getId()); - System.out.println("peer " + peer.getAddress().toString()); - } + logger.debug("peer: id=" + peer.getId() + " address=" + peer.getAddress() + " isConnected=" + peer.isConnected()); + } ); // .map(NodeRecordInfo::getNode) // .filter(r -> !activeKnownNodes.contains(r)) // .collect(Collectors.toList()); + network + .streamKnownDiscoveryPeers() + .filter(discoPeer -> discoPeer.getNodeAddress() != null) + .forEach( + discoPeer -> { + //System.out.println("peer: id=" + peer.getId() + " address=" + peer.getAddress() + " isConnected=" + peer.isConnected()); + logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey="+discoPeer.getPublicKey()); + } + ); // activeKnownNodes.addAll(newActiveNodes); // newActiveNodes.forEach( // n -> { diff --git a/src/org/minima/system/network/base/peer/PeerManager.java b/src/org/minima/system/network/base/peer/PeerManager.java index 3e2b01be0..bc239d42a 100644 --- a/src/org/minima/system/network/base/peer/PeerManager.java +++ b/src/org/minima/system/network/base/peer/PeerManager.java @@ -139,7 +139,7 @@ void onConnectedPeer(Peer peer) { peer.subscribeDisconnect( (reason, locallyInitiated) -> onDisconnectedPeer(peer, reason, locallyInitiated)); } else { - LOG.trace("Disconnecting duplicate connection to {}", peer::getId); + LOG.debug("Disconnecting duplicate connection to {}", peer::getId); peer.disconnectImmediately(Optional.empty(), true); throw new PeerAlreadyConnectedException(peer); } From 63e2a1f832937b3c6cc749a39871483fa49b8037 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 22 Apr 2021 14:25:41 +0100 Subject: [PATCH 16/55] p2p: added log messages to discovery classes, cleaned up P2PStart. --- .../system/network/base/DiscV5Service.java | 13 ++ .../system/network/base/DiscoveryNetwork.java | 3 +- .../minima/system/network/base/P2PStart.java | 212 ++++++++---------- .../system/network/base/peer/PeerManager.java | 2 +- 4 files changed, 112 insertions(+), 118 deletions(-) diff --git a/src/org/minima/system/network/base/DiscV5Service.java b/src/org/minima/system/network/base/DiscV5Service.java index 69585f84f..aee8f19bb 100644 --- a/src/org/minima/system/network/base/DiscV5Service.java +++ b/src/org/minima/system/network/base/DiscV5Service.java @@ -18,6 +18,11 @@ import java.util.List; import java.util.Optional; import java.util.stream.Stream; + + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import org.apache.tuweni.bytes.Bytes; import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.MultiaddrUtil; @@ -60,6 +65,8 @@ public static DiscoveryService create( private final DiscoverySystem discoverySystem; private final KeyValueStore kvStore; + private static final Logger LOG = LogManager.getLogger(); + private DiscV5Service( final DiscoveryConfig discoConfig, NetworkConfig p2pConfig, @@ -102,27 +109,32 @@ private NewAddressHandler maybeUpdateNodeRecord(boolean userExplicitlySetAdverti } private void localNodeRecordUpdated(NodeRecord oldRecord, NodeRecord newRecord) { + LOG.info("Updating NodeRecord for " + newRecord.getNodeId() + "(" + newRecord.getTcpAddress() + ")"); kvStore.put(SEQ_NO_STORE_KEY, newRecord.getSeq().toBytes()); } @Override protected SafeFuture doStart() { + LOG.info("Starting discovery system"); return SafeFuture.of(discoverySystem.start()); } @Override protected SafeFuture doStop() { + LOG.info("Stopping discovery system"); discoverySystem.stop(); return SafeFuture.completedFuture(null); } @Override public Stream streamKnownPeers() { + LOG.info("Returning all active nodes as known peers - " + activeNodes().count()); return activeNodes().map(NodeRecordConverter::convertToDiscoveryPeer).flatMap(Optional::stream); } @Override public SafeFuture searchForPeers() { + LOG.info("Searching for new peers"); return SafeFuture.of(discoverySystem.searchForNewPeers()); } @@ -153,6 +165,7 @@ public void updateCustomENRField(String fieldName, Bytes value) { } private Stream activeNodes() { + LOG.info("Returning all nodes known by discovery system and active"); return discoverySystem .streamKnownNodes() .filter(record -> record.getStatus() == NodeStatus.ACTIVE) diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index 60bad80e0..b102e1a9f 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -27,7 +27,6 @@ import org.minima.system.network.base.metrics.MetricsSystem; import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.NodeId; -import org.apache.logging.log4j.status.StatusLogger; import org.apache.tuweni.bytes.Bytes; import org.apache.tuweni.bytes.Bytes32; // import org.hyperledger.besu.plugin.services.MetricsSystem; @@ -241,6 +240,6 @@ public int getP2PPeerCount() { LibP2PNetwork net = (LibP2PNetwork) p2pNetwork; return net.getPeerCount(); } - + } diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index b7353b344..ac7965c48 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -26,17 +26,15 @@ public class P2PStart { - private static final Logger logger = LogManager.getLogger(P2PStart.class); - public static void main(String[] args) - throws ExecutionException, InterruptedException { + public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); // attempt 1: start with DiscoveryNetworkFactory DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); final DiscoveryNetwork network; try { - + String node1_id; String[] node1_addr_fields; String node2_id; @@ -49,142 +47,126 @@ public static void main(String[] args) System.out.println("Found addr node 1: " + args[0]); System.out.println("Found boot node 1: " + args[1]); node1_addr_fields = args[0].split("/"); - node1_id = node1_addr_fields[node1_addr_fields.length-1]; -// node2_addr_fields = args[1].split("/"); -// node2_id = node2_addr_fields[node2_addr_fields.length-1]; -// System.out.println("Found node2_id: " + node1_id); + node1_id = node1_addr_fields[node1_addr_fields.length - 1]; + // node2_addr_fields = args[1].split("/"); + // node2_id = node2_addr_fields[node2_addr_fields.length-1]; + // System.out.println("Found node2_id: " + node1_id); String bootnode_1 = args[1]; // System.out.println("Found boot node addr: " + bootnode_1); // Multiaddr address = Multiaddr.fromString(args[0]); network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(); - //network.getEnr() - //network = factory.builder().bootnode(args[0]) - //Thread.sleep(5000); - //Peer peer = network.getPeer(args[0]); + // network.getEnr() + // network = factory.builder().bootnode(args[0]) + // Thread.sleep(5000); + // Peer peer = network.getPeer(args[0]); // peerCount = network.getPeerCount(); // addr = network.getNodeAddress(); // id = network.getNodeId(); // // System.out.println("peerCount = " + peerCount); // System.out.println("addr = " + addr); // System.out.println("id = " + id.toString()); - //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); + // discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); -// LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); - //Thread.sleep(5000); -// Waiter.waitFor( -// () -> { -// Optional firstNode = network.getPeer(id_1); -// if(firstNode.isPresent()) { -// System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); -// } else { -// System.out.println("First node not found."); -// } -// // Optional secondNode = network.getPeer(id_2); -// // if(secondNode.isPresent()) { -// // System.out.println("Success! We found the second node: " + secondNode.get().getAddress()); -// // } else { -// // System.out.println("Second node not found."); -// // } -// System.out.println("peerCount = " + peerCount); -// }); - } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR (application level address) - logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); - System.out.println("Found addr: " + args[0]); - node1_addr_fields = args[0].split("/"); - node1_id = node1_addr_fields[node1_addr_fields.length-1]; - - System.out.println("Found node1_id: " + node1_id); - // Multiaddr address = Multiaddr.fromString(args[0]); - network = factory.builder().staticPeer(args[0]).buildAndStart(); - //network.getEnr() - //network = factory.builder().bootnode(args[0]) -// Thread.sleep(1000); - //Peer peer = network.getPeer(args[0]); - // peerCount = network.getPeerCount(); - // addr = network.getNodeAddress(); - // id = network.getNodeId(); -// System.out.println("peerCount = " + peerCount); -// System.out.println("addr = " + addr); -// System.out.println("id = " + id.toString()); - //discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); - - //LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - //Thread.sleep(5000); - // Waiter.waitFor( - // () -> { - // Optional firstNode = network.getPeer(idfirst); - // if(firstNode.isPresent()) { - // System.out.println("Success! We found the first node: " + firstNode.get().getAddress()); - // } else { - // System.out.println("First node not found."); - // } - // System.out.println("peerCount = " + peerCount); - - // }); + // LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); + // Thread.sleep(5000); + // Waiter.waitFor( + // () -> { + // Optional firstNode = network.getPeer(id_1); + // if(firstNode.isPresent()) { + // System.out.println("Success! We found the first node: " + + // firstNode.get().getAddress()); + // } else { + // System.out.println("First node not found."); + // } + // // Optional secondNode = network.getPeer(id_2); + // // if(secondNode.isPresent()) { + // // System.out.println("Success! We found the second node: " + + // secondNode.get().getAddress()); + // // } else { + // // System.out.println("Second node not found."); + // // } + // System.out.println("peerCount = " + peerCount); + // }); + } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR + // (application level address) + logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); + System.out.println("Found addr: " + args[0]); + node1_addr_fields = args[0].split("/"); + node1_id = node1_addr_fields[node1_addr_fields.length - 1]; + + System.out.println("Found node1_id: " + node1_id); + // Multiaddr address = Multiaddr.fromString(args[0]); + network = factory.builder().staticPeer(args[0]).buildAndStart(); + // network.getEnr() + // network = factory.builder().bootnode(args[0]) + // Thread.sleep(1000); + // Peer peer = network.getPeer(args[0]); + // peerCount = network.getPeerCount(); + // addr = network.getNodeAddress(); + // id = network.getNodeId(); + // System.out.println("peerCount = " + peerCount); + // System.out.println("addr = " + addr); + // System.out.println("id = " + id.toString()); + // discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); + + // LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); + // Thread.sleep(5000); + // Waiter.waitFor( + // () -> { + // Optional firstNode = network.getPeer(idfirst); + // if(firstNode.isPresent()) { + // System.out.println("Success! We found the first node: " + + // firstNode.get().getAddress()); + // } else { + // System.out.println("First node not found."); + // } + // System.out.println("peerCount = " + peerCount); + + // }); } else { // server mode only - no peer to connect to network = factory.builder().buildAndStart(); id = network.getNodeId(); - //Optional discAddr = network.getDiscoveryAddress(); - //Optional enr = network.getEnr(); - //System.out.println("id = " + id.toString()); - //System.out.println("discAddr = " + discAddr.toString()); - //System.out.println("enr = " + enr.get().substring(3)); - } -// System.out.println("network node address: " + network.getNodeAddress()); - Optional discAddr = network.getDiscoveryAddress(); -// System.out.println("network discovery address: " + discAddr.get()); -// System.out.println("network node id: " + network.getNodeId()); - logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + " , discovery address: " + discAddr.get() ); + + Optional discAddr = network.getDiscoveryAddress(); + logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + + " , discovery address: " + discAddr.get()); System.out.println("Starting discovery loop info"); - if(network != null) { - Set activeKnownNodes = new HashSet<>(); - while (true) { - // List newActiveNodes = - network - .streamPeers() - .filter(peer -> peer.getId() != null) - .forEach( - peer -> { - logger.debug("peer: id=" + peer.getId() + " address=" + peer.getAddress() + " isConnected=" + peer.isConnected()); - } - ); - // .map(NodeRecordInfo::getNode) - // .filter(r -> !activeKnownNodes.contains(r)) - // .collect(Collectors.toList()); - - network - .streamKnownDiscoveryPeers() - .filter(discoPeer -> discoPeer.getNodeAddress() != null) - .forEach( - discoPeer -> { - //System.out.println("peer: id=" + peer.getId() + " address=" + peer.getAddress() + " isConnected=" + peer.isConnected()); - logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey="+discoPeer.getPublicKey()); - } - ); - // activeKnownNodes.addAll(newActiveNodes); - // newActiveNodes.forEach( - // n -> { - // System.out.println( - // "New active node: " - // + n.getNodeId() - // + " @ " - // + n.getUdpAddress().map(InetSocketAddress::toString).orElse("")); - // }); - Thread.sleep(5000); - } - } + if (network != null) { + Set activeKnownNodes = new HashSet<>(); + while (true) { + network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { + logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true + }); + + logger.debug("trying to stream dicovery peers"); + network.streamKnownDiscoveryPeers() + .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr + logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + + discoPeer.getPublicKey()); + }); + + // activeKnownNodes.addAll(newActiveNodes); + // newActiveNodes.forEach( + // n -> { + // System.out.println( + // "New active node: " + // + n.getNodeId() + // + " @ " + // + n.getUdpAddress().map(InetSocketAddress::toString).orElse("")); + // }); + Thread.sleep(5000); + } + } } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } - } } - diff --git a/src/org/minima/system/network/base/peer/PeerManager.java b/src/org/minima/system/network/base/peer/PeerManager.java index bc239d42a..5e6089c5a 100644 --- a/src/org/minima/system/network/base/peer/PeerManager.java +++ b/src/org/minima/system/network/base/peer/PeerManager.java @@ -90,7 +90,7 @@ public SafeFuture connect(final MultiaddrPeerAddress peer, final Network n } private SafeFuture doConnect(final MultiaddrPeerAddress peer, final Network network) { - LOG.debug("Connecting to {}", peer); + LOG.debug("PeerMgr - Connecting to {}", peer); return SafeFuture.of(() -> network.connect(peer.getMultiaddr())) .thenApply( From b4ad6077cb69f6d315a423aca0efcfc829e5dff6 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 22 Apr 2021 15:39:52 +0100 Subject: [PATCH 17/55] p2p: build.gradle updated libp2p to 0.8.1 and discovery to 0.4.6 --- build.gradle | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 66b5647ee..c6ff08f6c 100644 --- a/build.gradle +++ b/build.gradle @@ -34,7 +34,8 @@ repositories { maven { url "https://dl.cloudsmith.io/public/libp2p/jvm-libp2p/maven/" - // we use consensys eth discovery v5 implementation for now for nodes discovery + } + maven { url "https://artifacts.consensys.net/public/maven/maven/" } @@ -58,7 +59,8 @@ dependencies { implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.0-M1") - implementation 'io.libp2p:jvm-libp2p-minimal:0.7.0-RELEASE' + //implementation('io.libp2p:jvm-libp2p-minimal:0.8.1-RELEASE') + implementation 'io.libp2p:jvm-libp2p-minimal:0.8.1-RELEASE' // implementation 'io.libp2p:jvm-libp2p-minimal' // compile files('libs/jvm-libp2p-minimal-0.8.0-RELEASE.jar') @@ -84,7 +86,8 @@ dependencies { // awaitility is used by Waiter.java implementation group: 'org.awaitility', name: 'awaitility', version: '4.0.3' - implementation 'tech.pegasys.discovery:discovery:0.4.3-dev-57c2fd81' + implementation 'tech.pegasys.discovery:discovery:0.4.6' + // implementation 'tech.pegasys.discovery:discovery:0.4.3-dev-57c2fd81' From ad2ae380862f54e71c368342307fc44275a16bae Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 22 Apr 2021 15:41:47 +0100 Subject: [PATCH 18/55] p2p: detects new peers every 5 seconds in P2PStart, and commented out logs in DiscV5Service. --- .../system/network/base/DiscV5Service.java | 4 +- .../minima/system/network/base/P2PStart.java | 92 ++++--------------- 2 files changed, 18 insertions(+), 78 deletions(-) diff --git a/src/org/minima/system/network/base/DiscV5Service.java b/src/org/minima/system/network/base/DiscV5Service.java index aee8f19bb..aa9937c82 100644 --- a/src/org/minima/system/network/base/DiscV5Service.java +++ b/src/org/minima/system/network/base/DiscV5Service.java @@ -128,7 +128,7 @@ protected SafeFuture doStop() { @Override public Stream streamKnownPeers() { - LOG.info("Returning all active nodes as known peers - " + activeNodes().count()); +// LOG.info("Returning all active nodes as known peers - " + activeNodes().count()); return activeNodes().map(NodeRecordConverter::convertToDiscoveryPeer).flatMap(Optional::stream); } @@ -165,7 +165,7 @@ public void updateCustomENRField(String fieldName, Bytes value) { } private Stream activeNodes() { - LOG.info("Returning all nodes known by discovery system and active"); + // LOG.info("Returning all nodes known by discovery system and active"); return discoverySystem .streamKnownNodes() .filter(record -> record.getStatus() == NodeStatus.ACTIVE) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index ac7965c48..5dd171ad5 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -2,6 +2,7 @@ import java.net.InetSocketAddress; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; @@ -48,47 +49,10 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc System.out.println("Found boot node 1: " + args[1]); node1_addr_fields = args[0].split("/"); node1_id = node1_addr_fields[node1_addr_fields.length - 1]; - // node2_addr_fields = args[1].split("/"); - // node2_id = node2_addr_fields[node2_addr_fields.length-1]; - // System.out.println("Found node2_id: " + node1_id); String bootnode_1 = args[1]; - // System.out.println("Found boot node addr: " + bootnode_1); - // Multiaddr address = Multiaddr.fromString(args[0]); network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(); - // network.getEnr() - // network = factory.builder().bootnode(args[0]) - // Thread.sleep(5000); - // Peer peer = network.getPeer(args[0]); - // peerCount = network.getPeerCount(); - // addr = network.getNodeAddress(); - // id = network.getNodeId(); - // // System.out.println("peerCount = " + peerCount); - // System.out.println("addr = " + addr); - // System.out.println("id = " + id.toString()); - // discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); - LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - // LibP2PNodeId id_2 = new LibP2PNodeId(PeerId.fromBase58(node2_id)); - // Thread.sleep(5000); - // Waiter.waitFor( - // () -> { - // Optional firstNode = network.getPeer(id_1); - // if(firstNode.isPresent()) { - // System.out.println("Success! We found the first node: " + - // firstNode.get().getAddress()); - // } else { - // System.out.println("First node not found."); - // } - // // Optional secondNode = network.getPeer(id_2); - // // if(secondNode.isPresent()) { - // // System.out.println("Success! We found the second node: " + - // secondNode.get().getAddress()); - // // } else { - // // System.out.println("Second node not found."); - // // } - // System.out.println("peerCount = " + peerCount); - // }); } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR // (application level address) logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); @@ -99,32 +63,6 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc System.out.println("Found node1_id: " + node1_id); // Multiaddr address = Multiaddr.fromString(args[0]); network = factory.builder().staticPeer(args[0]).buildAndStart(); - // network.getEnr() - // network = factory.builder().bootnode(args[0]) - // Thread.sleep(1000); - // Peer peer = network.getPeer(args[0]); - // peerCount = network.getPeerCount(); - // addr = network.getNodeAddress(); - // id = network.getNodeId(); - // System.out.println("peerCount = " + peerCount); - // System.out.println("addr = " + addr); - // System.out.println("id = " + id.toString()); - // discoveryNetworkFactory.builder().bootnode(network1.getEnr().orElseThrow()).buildAndStart(); - - // LibP2PNodeId idfirst = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - // Thread.sleep(5000); - // Waiter.waitFor( - // () -> { - // Optional firstNode = network.getPeer(idfirst); - // if(firstNode.isPresent()) { - // System.out.println("Success! We found the first node: " + - // firstNode.get().getAddress()); - // } else { - // System.out.println("First node not found."); - // } - // System.out.println("peerCount = " + peerCount); - - // }); } else { // server mode only - no peer to connect to network = factory.builder().buildAndStart(); @@ -137,28 +75,30 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc System.out.println("Starting discovery loop info"); if (network != null) { - Set activeKnownNodes = new HashSet<>(); + Set activeKnownNodes = new HashSet<>(); while (true) { network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true }); - logger.debug("trying to stream dicovery peers"); + Set newActiveNodes = new HashSet<>(); + //logger.debug("trying to stream dicovery peers"); network.streamKnownDiscoveryPeers() .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr - logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" - + discoPeer.getPublicKey()); + // logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey()); + newActiveNodes.add(discoPeer.getNodeAddress()); }); - // activeKnownNodes.addAll(newActiveNodes); - // newActiveNodes.forEach( - // n -> { - // System.out.println( - // "New active node: " - // + n.getNodeId() - // + " @ " - // + n.getUdpAddress().map(InetSocketAddress::toString).orElse("")); - // }); + Set delta = new HashSet(newActiveNodes); + delta.removeAll(activeKnownNodes); //now contains only new sockets + + for(InetSocketAddress i: delta) { + logger.info("New peer address: " + i.toString()); + } + + // update known nodes + activeKnownNodes = newActiveNodes; + Thread.sleep(5000); } } From 52259bace419b8a7750aff4c7867630438ac95a3 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 22 Apr 2021 17:08:20 +0100 Subject: [PATCH 19/55] p2p: implemented MessageProcessor in P2PStart and integrated within Minima. --- build.gradle | 2 + src/org/minima/system/Main.java | 8 ++ .../network/base/ConnectionManager.java | 2 +- .../system/network/base/DiscV5Service.java | 2 +- .../system/network/base/DiscoveryNetwork.java | 2 +- .../network/base/DiscoveryNetworkFactory.java | 2 +- .../system/network/base/LibP2PNetwork.java | 2 +- .../system/network/base/NetworkConfig.java | 2 +- .../network/base/NodeRecordConverter.java | 2 +- .../minima/system/network/base/P2PStart.java | 107 ++++++++++++++++-- .../system/network/base/peer/LibP2PPeer.java | 2 +- .../system/network/base/peer/PeerManager.java | 2 +- 12 files changed, 118 insertions(+), 17 deletions(-) diff --git a/build.gradle b/build.gradle index c6ff08f6c..c4e6b9fb7 100644 --- a/build.gradle +++ b/build.gradle @@ -41,6 +41,7 @@ repositories { } + dependencies { // This dependency is used by the application. implementation 'com.google.guava:guava:29.0-jre' @@ -142,5 +143,6 @@ task(runp2p, dependsOn: 'classes', type: JavaExec) { jar { manifest { attributes 'Main-Class': 'org.minima.Start' + attributes "Multi-Release": true } } diff --git a/src/org/minima/system/Main.java b/src/org/minima/system/Main.java index 4ebf1934b..530416f02 100644 --- a/src/org/minima/system/Main.java +++ b/src/org/minima/system/Main.java @@ -10,6 +10,7 @@ import org.minima.system.brains.SendManager; import org.minima.system.input.InputHandler; import org.minima.system.network.NetworkHandler; +import org.minima.system.network.base.P2PStart; import org.minima.system.txpow.TxPoWMiner; import org.minima.utils.MinimaLogger; import org.minima.utils.SQLHandler; @@ -64,7 +65,13 @@ public static Main getMainHandler() { * The Backup Manager - runs in a separate thread */ private BackupManager mBackup; + + /** + * P2P nodes discovery layer + */ + private P2PStart mP2P; + /** * Are we creating a network from scratch */ @@ -106,6 +113,7 @@ public Main(String zHost, int zPort, boolean zGenesis, String zConfFolder) { mTXMiner = new TxPoWMiner(); mConsensus = new ConsensusHandler(); mSendManager = new SendManager(); + mP2P = new P2PStart(null, null); //Are we the genesis mGenesis = zGenesis; diff --git a/src/org/minima/system/network/base/ConnectionManager.java b/src/org/minima/system/network/base/ConnectionManager.java index 8bbc6224d..b412193c5 100644 --- a/src/org/minima/system/network/base/ConnectionManager.java +++ b/src/org/minima/system/network/base/ConnectionManager.java @@ -53,7 +53,7 @@ import org.minima.system.network.base.peer.PeerPools.PeerPool; public class ConnectionManager extends Service { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(ConnectionManager.class); private static final Duration RECONNECT_TIMEOUT = Duration.ofSeconds(20); private static final Duration DISCOVERY_INTERVAL = Duration.ofSeconds(30); private final AsyncRunner asyncRunner; diff --git a/src/org/minima/system/network/base/DiscV5Service.java b/src/org/minima/system/network/base/DiscV5Service.java index aa9937c82..3e85ae9cf 100644 --- a/src/org/minima/system/network/base/DiscV5Service.java +++ b/src/org/minima/system/network/base/DiscV5Service.java @@ -65,7 +65,7 @@ public static DiscoveryService create( private final DiscoverySystem discoverySystem; private final KeyValueStore kvStore; - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(DiscV5Service.class); private DiscV5Service( final DiscoveryConfig discoConfig, diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index b102e1a9f..a89fcb04c 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -54,7 +54,7 @@ // import tech.pegasys.teku.storage.store.KeyValueStore; public class DiscoveryNetwork

extends DelegatingP2PNetwork

{ - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(DiscoveryNetwork.class); public static final String ATTESTATION_SUBNET_ENR_FIELD = "attnets"; public static final String ETH2_ENR_FIELD = "eth2"; diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index 61fbf80d1..78c9a9623 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -52,7 +52,7 @@ public class DiscoveryNetworkFactory { - protected static final Logger LOG = LogManager.getLogger(); + protected static final Logger LOG = LogManager.getLogger(DiscoveryNetworkFactory.class); protected static final NoOpMetricsSystem METRICS_SYSTEM = new NoOpMetricsSystem(); private static final int MIN_PORT = 9000; private static final int MAX_PORT = 12000; diff --git a/src/org/minima/system/network/base/LibP2PNetwork.java b/src/org/minima/system/network/base/LibP2PNetwork.java index e96b96982..721646ef1 100644 --- a/src/org/minima/system/network/base/LibP2PNetwork.java +++ b/src/org/minima/system/network/base/LibP2PNetwork.java @@ -88,7 +88,7 @@ public class LibP2PNetwork implements P2PNetwork { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(LibP2PNetwork.class); private static final int REMOTE_OPEN_STREAMS_RATE_LIMIT = 256; private static final int REMOTE_PARALLEL_OPEN_STREAMS_COUNT_LIMIT = 256; diff --git a/src/org/minima/system/network/base/NetworkConfig.java b/src/org/minima/system/network/base/NetworkConfig.java index 45a584636..d580bfab5 100644 --- a/src/org/minima/system/network/base/NetworkConfig.java +++ b/src/org/minima/system/network/base/NetworkConfig.java @@ -29,7 +29,7 @@ import org.minima.system.network.base.gossip.config.GossipConfig; public class NetworkConfig { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(NetworkConfig.class); private final GossipConfig gossipConfig; private final WireLogsConfig wireLogsConfig; diff --git a/src/org/minima/system/network/base/NodeRecordConverter.java b/src/org/minima/system/network/base/NodeRecordConverter.java index e078f2464..aa773e106 100644 --- a/src/org/minima/system/network/base/NodeRecordConverter.java +++ b/src/org/minima/system/network/base/NodeRecordConverter.java @@ -29,7 +29,7 @@ import org.minima.system.network.base.ssz.SszBitvector; public class NodeRecordConverter { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(NodeRecordConverter.class); static Optional convertToDiscoveryPeer(final NodeRecord nodeRecord) { return nodeRecord diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 5dd171ad5..92deb311a 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -19,15 +19,110 @@ import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; +import org.minima.utils.messages.Message; +import org.minima.utils.messages.MessageProcessor; import io.libp2p.core.PeerId; import io.libp2p.etc.encode.Base58; import org.apache.logging.log4j.LogManager; -public class P2PStart { +public class P2PStart extends MessageProcessor { private static final Logger logger = LogManager.getLogger(P2PStart.class); + public static final String P2P_THREAD = "P2P"; + // these messages only control LIBP2P+DISCV5 <> Minima comms at the moment + public static final String P2P_START_SCAN = "P2P_START_SCAN"; + public static final String P2P_STOP_SCAN = "P2P_STOP_SCAN"; + + + private DiscoveryNetwork network; + Set activeKnownNodes; + + // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt + // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr + public P2PStart(String[] staticPeers, String[] bootnodes) { + super(P2P_THREAD); + if(staticPeers == null || staticPeers.length == 0) { + logger.info("P2P layer - no static peer."); + } + if(bootnodes == null || bootnodes.length == 0) { + logger.info("P2P layer - no bootnode."); + } + DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); + try { + if(staticPeers != null && staticPeers.length > 0 && bootnodes != null && bootnodes.length > 0) { + network = factory.builder().staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); + } else if(staticPeers == null || staticPeers.length == 0) { + logger.info("P2P: starting in standalone mode"); + network = factory.builder().buildAndStart(); + } + } catch (Exception e) { + logger.error("P2P failed to start through DiscoveyrNetworkFactory."); + e.printStackTrace(); + } + + if(network != null) { + // P2P initialization complete + Optional discAddr = network.getDiscoveryAddress(); + logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + + " , discovery address: " + discAddr.get()); + System.out.println("Starting discovery loop info"); + activeKnownNodes = new HashSet<>(); + // TODO: send to yourself a message 5 seconds in the future + PostMessage(P2P_START_SCAN); // could also be a TimerMessage + } else { + // initialization failed - what do we do? + logger.error("Failed to start P2P network."); + } + + } + + + @Override + protected void processMessage(Message zMessage) throws Exception { + // TODO Auto-generated method stub + logger.warn("P2PStart received message: " + zMessage.toString()); + + if(zMessage.isMessageType(P2P_START_SCAN)) { + + if(network != null) { + Set activeKnownNodes = new HashSet<>(); + while (true) { + network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { + logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true + }); + + Set newActiveNodes = new HashSet<>(); + //logger.debug("trying to stream discovery peers"); + network.streamKnownDiscoveryPeers() + .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr + // logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey()); + newActiveNodes.add(discoPeer.getNodeAddress()); + }); + + Set delta = new HashSet(newActiveNodes); + delta.removeAll(activeKnownNodes); //now contains only new sockets + + for(InetSocketAddress i: delta) { + logger.info("New peer address: " + i.toString()); + } + + // update known nodes + activeKnownNodes = newActiveNodes; + + try { + Thread.sleep(5000); + PostMessage(P2P_START_SCAN); + } catch(Exception e) { + + } + } + } + } + + + } public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); @@ -38,11 +133,6 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc String node1_id; String[] node1_addr_fields; - String node2_id; - String[] node2_addr_fields; - final int peerCount; - String addr; - NodeId id; if (args.length == 2) { // first is p2p addr, second is enr System.out.println("Found addr node 1: " + args[0]); @@ -55,6 +145,7 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR // (application level address) + // DiscV5 cant start without at least one boot ndoe logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); System.out.println("Found addr: " + args[0]); node1_addr_fields = args[0].split("/"); @@ -66,7 +157,7 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc } else { // server mode only - no peer to connect to network = factory.builder().buildAndStart(); - id = network.getNodeId(); + //id = network.getNodeId(); } Optional discAddr = network.getDiscoveryAddress(); @@ -82,7 +173,7 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc }); Set newActiveNodes = new HashSet<>(); - //logger.debug("trying to stream dicovery peers"); + //logger.debug("trying to stream discovery peers"); network.streamKnownDiscoveryPeers() .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr // logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey()); diff --git a/src/org/minima/system/network/base/peer/LibP2PPeer.java b/src/org/minima/system/network/base/peer/LibP2PPeer.java index f27b4e7a7..604cf229b 100644 --- a/src/org/minima/system/network/base/peer/LibP2PPeer.java +++ b/src/org/minima/system/network/base/peer/LibP2PPeer.java @@ -43,7 +43,7 @@ import org.minima.system.network.base.peer.RpcStream; public class LibP2PPeer implements Peer { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(LibP2PPeer.class); private final Map rpcHandlers; private final ReputationManager reputationManager; diff --git a/src/org/minima/system/network/base/peer/PeerManager.java b/src/org/minima/system/network/base/peer/PeerManager.java index 5e6089c5a..be57cef49 100644 --- a/src/org/minima/system/network/base/peer/PeerManager.java +++ b/src/org/minima/system/network/base/peer/PeerManager.java @@ -46,7 +46,7 @@ public class PeerManager implements ConnectionHandler { - private static final Logger LOG = LogManager.getLogger(); + private static final Logger LOG = LogManager.getLogger(PeerManager.class); private final Map rpcHandlers; From 285b2c2a9eb7dc2246c7ac7832c42e8ae26514ee Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Sun, 9 May 2021 12:24:56 +0100 Subject: [PATCH 20/55] p2p: modified Start process to disable existing networking stack if p2p enabled. --- src/org/minima/Start.java | 29 ++++++++++++++++--- src/org/minima/system/Main.java | 6 ++-- .../minima/system/network/base/P2PStart.java | 16 ++++++++-- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/src/org/minima/Start.java b/src/org/minima/Start.java index bb804e376..73c9aabaf 100644 --- a/src/org/minima/Start.java +++ b/src/org/minima/Start.java @@ -9,6 +9,8 @@ import java.io.InputStreamReader; import java.util.ArrayList; import java.util.Random; +import java.util.Set; +import java.util.HashSet; import org.minima.objects.base.MiniNumber; import org.minima.objects.greet.SyncPacket; @@ -27,7 +29,7 @@ public class Start { /** * A list of default valid nodes to connect to at startup.. */ - public static final String[] VALID_BOOTSTRAP_NODES = + public static String[] VALID_BOOTSTRAP_NODES = {"35.204.181.120", "35.204.119.15", "34.91.220.49", @@ -116,6 +118,9 @@ public static void main(String[] zArgs){ boolean genesis = false; boolean daemon = false; boolean automine = false; + + Set p2pStaticSet = new HashSet(); + Set p2pBootnodeSet = new HashSet(); //Configuration folder File conf = new File(System.getProperty("user.home"),".minima"); @@ -149,6 +154,9 @@ public static void main(String[] zArgs){ MinimaLogger.log(" -connect [host] [port] : Don't connect to MainNet but connect to this node instead."); MinimaLogger.log(" -daemon : Accepts no input from STDIN. Can run in background process."); MinimaLogger.log(" -externalurl : Send a POST request to this URL with Minima JSON information."); + MinimaLogger.log(" -p2p-static : static peer address for p2p network discovery."); + MinimaLogger.log(" -p2p-bootnode : bootnode record for p2p network discovery."); + MinimaLogger.log(" -help : Show this help"); MinimaLogger.log(""); MinimaLogger.log("With zero parameters Minima will start and connect to a set of default nodes."); @@ -210,6 +218,16 @@ public static void main(String[] zArgs){ System.exit(0); + } else if(arg.equals("-p2p-static")) { + p2pStaticSet.add(zArgs[counter++]); + // if this set is not empty we must ignore valid_bootstrap_nodes and connecthost / connectport + VALID_BOOTSTRAP_NODES = new String[0]; + connecthost = ""; + } else if(arg.equals("-p2p-bootnode")) { + p2pBootnodeSet.add(zArgs[counter++]); + // if this set is not empty we must ignore valid_bootstrap_nodes and connecthost / connectport + VALID_BOOTSTRAP_NODES = new String[0]; + connecthost = ""; }else if(arg.equals("")) { //Do nothing.. @@ -235,15 +253,18 @@ public static void main(String[] zArgs){ //Wipe webroot too.. BackupManager.deleteWebRoot(conffile); } - + // TODO: replace array with dynamic set and extract array when calling Main + // TODO: manage more than one staticpeer / bootnode from commandline by pushing to set + // TODO: read staticpeers and bootnodes from config file depending on some flag - maybe combined with command line args + //Start the main Minima server - Main rcmainserver = new Main(host, port, genesis, conffile.getAbsolutePath()); + Main rcmainserver = new Main(host, port, genesis, conffile.getAbsolutePath(), p2pStaticSet.toArray(new String[0]), p2pBootnodeSet.toArray(new String[0])); //Link it. mMainServer = rcmainserver; //Have we added any connect hosts.. - if(connectlist.size() == 0 && connect) { + if((connectlist.size() == 0) && connecthost != null && !connecthost.isEmpty() && connect) { rcmainserver.addAutoConnectHostPort(connecthost+":"+connectport); }else { for(String hostport : connectlist) { diff --git a/src/org/minima/system/Main.java b/src/org/minima/system/Main.java index 530416f02..b29b7642b 100644 --- a/src/org/minima/system/Main.java +++ b/src/org/minima/system/Main.java @@ -93,7 +93,7 @@ public static Main getMainHandler() { * @param zPort * @param zGenesis */ - public Main(String zHost, int zPort, boolean zGenesis, String zConfFolder) { + public Main(String zHost, int zPort, boolean zGenesis, String zConfFolder, String[] p2pStaticNodes, String[] p2pBootnodes) { super("MAIN"); mMainHandler = this; @@ -113,8 +113,8 @@ public Main(String zHost, int zPort, boolean zGenesis, String zConfFolder) { mTXMiner = new TxPoWMiner(); mConsensus = new ConsensusHandler(); mSendManager = new SendManager(); - mP2P = new P2PStart(null, null); - + mP2P = new P2PStart(mNetwork, p2pStaticNodes, p2pBootnodes); + //Are we the genesis mGenesis = zGenesis; diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 92deb311a..d5f26b953 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.ethereum.beacon.discovery.schema.NodeRecord; import org.ethereum.beacon.discovery.schema.NodeRecordInfo; +import org.minima.system.network.NetworkHandler; import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; @@ -35,14 +36,15 @@ public class P2PStart extends MessageProcessor { public static final String P2P_START_SCAN = "P2P_START_SCAN"; public static final String P2P_STOP_SCAN = "P2P_STOP_SCAN"; - + private NetworkHandler mNetwork; private DiscoveryNetwork network; Set activeKnownNodes; // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr - public P2PStart(String[] staticPeers, String[] bootnodes) { + public P2PStart(NetworkHandler minimaNet, String[] staticPeers, String[] bootnodes) { super(P2P_THREAD); + mNetwork = minimaNet; if(staticPeers == null || staticPeers.length == 0) { logger.info("P2P layer - no static peer."); } @@ -52,9 +54,11 @@ public P2PStart(String[] staticPeers, String[] bootnodes) { DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { if(staticPeers != null && staticPeers.length > 0 && bootnodes != null && bootnodes.length > 0) { + System.out.println("Building p2p layer using provided params: staticpeer=" + staticPeers[0] + " and bootnode=" + bootnodes[0]); network = factory.builder().staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); } else if(staticPeers == null || staticPeers.length == 0) { logger.info("P2P: starting in standalone mode"); + System.out.println("P2P: starting in standalone mode"); network = factory.builder().buildAndStart(); } } catch (Exception e) { @@ -67,6 +71,8 @@ public P2PStart(String[] staticPeers, String[] bootnodes) { Optional discAddr = network.getDiscoveryAddress(); logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + " , discovery address: " + discAddr.get()); + System.out.println("P2P nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() + + " , discovery address: " + discAddr.get()); System.out.println("Starting discovery loop info"); activeKnownNodes = new HashSet<>(); // TODO: send to yourself a message 5 seconds in the future @@ -105,7 +111,10 @@ protected void processMessage(Message zMessage) throws Exception { delta.removeAll(activeKnownNodes); //now contains only new sockets for(InetSocketAddress i: delta) { - logger.info("New peer address: " + i.toString()); + logger.info("New peer address: " + i.toString().substring(1)); + System.out.println("Starting MinimaClient: " + i.toString().substring(1) + ":9001"); + MinimaClient mclient = new MinimaClient(i.getAddress().toString().substring(1), 9001, mNetwork); // hardcode port for now + mNetwork.PostMessage(new Message(NetworkHandler.NETWORK_NEWCLIENT).addObject("client", mclient)); } // update known nodes @@ -124,6 +133,7 @@ protected void processMessage(Message zMessage) throws Exception { } + //TODO: refactor below code to use above object and constructor instead - if possible public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); // attempt 1: start with DiscoveryNetworkFactory From 59908a47e7a7d3d56681c68da7d78500db130d2b Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Mon, 10 May 2021 10:51:16 +0100 Subject: [PATCH 21/55] p2p: made node key persistent (-> persistent nodeID). --- src/org/minima/system/Main.java | 2 +- .../network/base/DiscoveryNetworkFactory.java | 12 ++- .../system/network/base/LibP2PNetwork.java | 7 +- .../minima/system/network/base/P2PStart.java | 82 ++++++++++++++++++- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/src/org/minima/system/Main.java b/src/org/minima/system/Main.java index b29b7642b..8bed400fa 100644 --- a/src/org/minima/system/Main.java +++ b/src/org/minima/system/Main.java @@ -113,7 +113,7 @@ public Main(String zHost, int zPort, boolean zGenesis, String zConfFolder, Strin mTXMiner = new TxPoWMiner(); mConsensus = new ConsensusHandler(); mSendManager = new SendManager(); - mP2P = new P2PStart(mNetwork, p2pStaticNodes, p2pBootnodes); + mP2P = new P2PStart(zConfFolder, mNetwork, p2pStaticNodes, p2pBootnodes); //Are we the genesis mGenesis = zGenesis; diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index 78c9a9623..c24063c2e 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -23,7 +23,9 @@ import java.util.concurrent.TimeoutException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.minima.system.network.base.libp2p.PrivateKeyGenerator; + +import io.libp2p.core.crypto.PrivKey; + // import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem; // import tech.pegasys.teku.infrastructure.async.DelayedExecutorAsyncRunner; // import tech.pegasys.teku.infrastructure.async.SafeFuture; @@ -77,6 +79,7 @@ public void stopAll() throws InterruptedException, ExecutionException, TimeoutEx public class DiscoveryNetworkBuilder { private final List staticPeers = new ArrayList<>(); private final List bootnodes = new ArrayList<>(); + private PrivKey privKey; private DiscoveryNetworkBuilder() {} @@ -90,6 +93,10 @@ public DiscoveryNetworkBuilder bootnode(final String bootnode) { return this; } + public DiscoveryNetworkBuilder setPrivKey(final PrivKey privKey) { + this.privKey = privKey; + return this; + } public DiscoveryNetwork buildAndStart() throws Exception { int attempt = 1; while (true) { @@ -108,6 +115,7 @@ public DiscoveryNetwork buildAndStart() throws Exception { Constants.REPUTATION_MANAGER_CAPACITY); final PeerSelectionStrategy peerSelectionStrategy = new SimplePeerSelectionStrategy(new TargetPeerRange(20, 30, 0)); + final DiscoveryNetwork network = DiscoveryNetwork.create( metricsSystem, @@ -116,7 +124,7 @@ public DiscoveryNetwork buildAndStart() throws Exception { new LibP2PNetwork( DelayedExecutorAsyncRunner.create(), config, - PrivateKeyGenerator::generate, + privKey, reputationManager, METRICS_SYSTEM, Collections.emptyList(), diff --git a/src/org/minima/system/network/base/LibP2PNetwork.java b/src/org/minima/system/network/base/LibP2PNetwork.java index 721646ef1..51381dcfd 100644 --- a/src/org/minima/system/network/base/LibP2PNetwork.java +++ b/src/org/minima/system/network/base/LibP2PNetwork.java @@ -107,16 +107,19 @@ public class LibP2PNetwork implements P2PNetwork { public LibP2PNetwork( final AsyncRunner asyncRunner, final NetworkConfig config, - final PrivateKeyProvider privateKeyProvider, + final PrivKey privKey, final ReputationManager reputationManager, final MetricsSystem metricsSystem, final List rpcMethods, final List peerHandlers, final PreparedGossipMessageFactory defaultMessageFactory, final GossipTopicFilter gossipTopicFilter) { - this.privKey = privateKeyProvider.get(); + + this.privKey = privKey; this.nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); + System.out.println("LibP2PNetwork - privKey = " + privKey.toString()); + System.out.println("LibP2PNetwork - nodeId = " + nodeId); advertisedAddr = MultiaddrUtil.fromInetSocketAddress( new InetSocketAddress(config.getAdvertisedIp(), config.getAdvertisedPort()), nodeId); diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index d5f26b953..b76beee85 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -1,5 +1,10 @@ package org.minima.system.network.base; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; import java.net.InetSocketAddress; import java.util.HashSet; import java.util.Iterator; @@ -17,6 +22,8 @@ import org.ethereum.beacon.discovery.schema.NodeRecord; import org.ethereum.beacon.discovery.schema.NodeRecordInfo; import org.minima.system.network.NetworkHandler; +import org.minima.system.network.base.LibP2PNetwork.PrivateKeyProvider; +import org.minima.system.network.base.libp2p.PrivateKeyGenerator; import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; @@ -24,6 +31,11 @@ import org.minima.utils.messages.MessageProcessor; import io.libp2p.core.PeerId; +import io.libp2p.core.crypto.PrivKey; +import io.libp2p.core.crypto.KEY_TYPE; +import io.libp2p.core.crypto.KeyKt; +import io.libp2p.crypto.keys.Secp256k1PrivateKey; + import io.libp2p.etc.encode.Base58; import org.apache.logging.log4j.LogManager; @@ -36,14 +48,16 @@ public class P2PStart extends MessageProcessor { public static final String P2P_START_SCAN = "P2P_START_SCAN"; public static final String P2P_STOP_SCAN = "P2P_STOP_SCAN"; + private String mConfFolder; private NetworkHandler mNetwork; private DiscoveryNetwork network; Set activeKnownNodes; // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr - public P2PStart(NetworkHandler minimaNet, String[] staticPeers, String[] bootnodes) { + public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPeers, String[] bootnodes) { super(P2P_THREAD); + this.mConfFolder = zConfFolder; mNetwork = minimaNet; if(staticPeers == null || staticPeers.length == 0) { logger.info("P2P layer - no static peer."); @@ -51,15 +65,68 @@ public P2PStart(NetworkHandler minimaNet, String[] staticPeers, String[] bootnod if(bootnodes == null || bootnodes.length == 0) { logger.info("P2P layer - no bootnode."); } + + + // check config file for SECP256K1 private key + File mRoot = ensureFolder(new File(mConfFolder)); + String mRootPath = mRoot.getAbsolutePath(); + + //Current used TxPOW + File mP2PDir = ensureFolder(new File(mRoot,"p2p")); + File mP2PNodePrivKeyFile = new File(mP2PDir, "NodePrivKey.pkey"); + // two lines below just to get rid of temporary not initialized issue + PrivateKeyProvider provider = PrivateKeyGenerator::generate; + PrivKey privKey = provider.get(); + if(mP2PNodePrivKeyFile.exists()) { + // try loading file from private key + // if error, bailout? + FileInputStream inputStream; + try { + inputStream = new FileInputStream(mP2PNodePrivKeyFile); + byte[] keyBuffer; + keyBuffer = inputStream.readAllBytes(); + privKey = KeyKt.unmarshalPrivateKey(keyBuffer); + } catch (FileNotFoundException e) { + System.out.println("Failed to read private key from disk - " + e.getMessage()); + e.printStackTrace(); + } catch (IOException e) { + System.out.println("Failed to read private key from disk - " + e.getMessage()); + e.printStackTrace(); + } + } else { + // generate a SECP256K1 private key + //PrivateKeyProvider keyProvider = PrivateKeyGenerator::generate; + privKey = KeyKt.generateKeyPair(KEY_TYPE.SECP256K1).component1(); + // privKey = keyProvider.get(); + // save priv key to file + try { + FileOutputStream outputStream = new FileOutputStream(mP2PNodePrivKeyFile); + outputStream.write(KeyKt.marshalPrivateKey(privKey)); + } catch (Exception e) { + System.out.println("Failed to save private key to disk - " + e.getMessage()); + } + } + if(privKey == null) { + System.out.println("P2P Error - priv key uninitialized!"); + return; + } + // privKey.toString(); + logger.warn("P2P layer - generated node private key: " + privKey.toString()); + System.out.println("P2P layer - generated node private key: " + privKey.toString()); + //System.out.println("P2P layer - generated node private key bytes: " + String. privKey.raw(); + NodeId nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); + System.out.println("P2P layer - nodeid: " + nodeId.toString()); + System.out.println("P2P layer - nodeid base58: " + nodeId.toBase58()); + DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { if(staticPeers != null && staticPeers.length > 0 && bootnodes != null && bootnodes.length > 0) { System.out.println("Building p2p layer using provided params: staticpeer=" + staticPeers[0] + " and bootnode=" + bootnodes[0]); - network = factory.builder().staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); + network = factory.builder().setPrivKey(privKey).staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); } else if(staticPeers == null || staticPeers.length == 0) { logger.info("P2P: starting in standalone mode"); System.out.println("P2P: starting in standalone mode"); - network = factory.builder().buildAndStart(); + network = factory.builder().setPrivKey(privKey).buildAndStart(); } } catch (Exception e) { logger.error("P2P failed to start through DiscoveyrNetworkFactory."); @@ -210,4 +277,13 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc } + + private static File ensureFolder(File zFolder) { + if(!zFolder.exists()) { + zFolder.mkdirs(); + } + + return zFolder; + } + } From fa66443eb1107702e7e18b1978f077d8ce842070 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 13 May 2021 16:26:27 +0100 Subject: [PATCH 22/55] p2p: added new fields to REST API status for p2p. --- src/org/minima/system/Main.java | 4 +++ .../minima/system/brains/ConsensusPrint.java | 7 +++++ .../system/network/base/DiscoveryNetwork.java | 12 ++++++++- .../minima/system/network/base/P2PStart.java | 26 ++++++++++++++++--- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/org/minima/system/Main.java b/src/org/minima/system/Main.java index 8bed400fa..9128d61c6 100644 --- a/src/org/minima/system/Main.java +++ b/src/org/minima/system/Main.java @@ -181,6 +181,10 @@ public BackupManager getBackupManager() { return mBackup; } + public P2PStart getP2P() { + return mP2P; + } + public TxPoWMiner getMiner() { return mTXMiner; } diff --git a/src/org/minima/system/brains/ConsensusPrint.java b/src/org/minima/system/brains/ConsensusPrint.java index 67d7db83a..9ba810baf 100644 --- a/src/org/minima/system/brains/ConsensusPrint.java +++ b/src/org/minima/system/brains/ConsensusPrint.java @@ -8,6 +8,7 @@ import java.util.Date; import java.util.Enumeration; import java.util.Hashtable; +import java.util.Optional; import org.minima.GlobalParams; import org.minima.database.MinimaDB; @@ -1068,6 +1069,12 @@ public int compare(JSONObject o1, JSONObject o2) { ArrayList nets = main.getNetworkHandler().getNetClients(); status.put("connections", nets.size()); + status.put("p2pnodeid", main.getP2P().getNodeId()); + Optional discAddr = main.getP2P().getDiscoveryAddress(); + status.put("p2pdiscoveryaddr", discAddr.isPresent()?discAddr.get():"unavailable"); + String enr = main.getP2P().getENR(); + status.put("p2penr", enr!=null?enr:"unavailable"); + //Add it to the output InputHandler.endResponse(zMessage, true, ""); diff --git a/src/org/minima/system/network/base/DiscoveryNetwork.java b/src/org/minima/system/network/base/DiscoveryNetwork.java index a89fcb04c..24ee7cfa8 100644 --- a/src/org/minima/system/network/base/DiscoveryNetwork.java +++ b/src/org/minima/system/network/base/DiscoveryNetwork.java @@ -68,6 +68,8 @@ public class DiscoveryNetwork

extends DelegatingP2PNetwork

{ private volatile Optional enrForkId = Optional.empty(); + private String mENR; + DiscoveryNetwork( final P2PNetwork

p2pNetwork, final DiscoveryService discoveryService, @@ -131,12 +133,20 @@ private static DiscoveryService createDiscoveryService( return discoveryService; } + public String getENR() { + return mENR; + } @Override public SafeFuture start() { return SafeFuture.allOfFailFast(p2pNetwork.start(), discoveryService.start()) .thenCompose(__ -> connectionManager.start()) - .thenRun(() -> getEnr().ifPresent( enr -> { LOG.warn("logwarn: listening for discv5: " + enr); System.out.println("sysout: listening for discv5: " + enr);})); // TODO: log ENR info and discovery start + .thenRun(() -> getEnr().ifPresent( + enr -> { + LOG.warn("logwarn: listening for discv5: " + enr); + System.out.println("sysout: listening for discv5: " + enr); + mENR = enr; + })); } //::listeningForDiscv5 @Override diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index b76beee85..87bd782ca 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -32,6 +32,7 @@ import io.libp2p.core.PeerId; import io.libp2p.core.crypto.PrivKey; +import io.libp2p.core.crypto.PubKey; import io.libp2p.core.crypto.KEY_TYPE; import io.libp2p.core.crypto.KeyKt; import io.libp2p.crypto.keys.Secp256k1PrivateKey; @@ -52,6 +53,9 @@ public class P2PStart extends MessageProcessor { private NetworkHandler mNetwork; private DiscoveryNetwork network; Set activeKnownNodes; + private NodeId nodeId; + private PubKey pubKey; + // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr @@ -109,15 +113,16 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee if(privKey == null) { System.out.println("P2P Error - priv key uninitialized!"); return; - } + } // privKey.toString(); logger.warn("P2P layer - generated node private key: " + privKey.toString()); System.out.println("P2P layer - generated node private key: " + privKey.toString()); //System.out.println("P2P layer - generated node private key bytes: " + String. privKey.raw(); - NodeId nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); + nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); System.out.println("P2P layer - nodeid: " + nodeId.toString()); System.out.println("P2P layer - nodeid base58: " + nodeId.toBase58()); - + pubKey = privKey.publicKey(); + DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { if(staticPeers != null && staticPeers.length > 0 && bootnodes != null && bootnodes.length > 0) { @@ -151,7 +156,22 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee } + public NodeId getNodeId() { + return nodeId; + } + public PubKey getPubKey() { + return pubKey; + } + + public String getENR() { + return network.getENR(); + } + + public Optional getDiscoveryAddress() { + return network.getDiscoveryAddress(); + } + @Override protected void processMessage(Message zMessage) throws Exception { // TODO Auto-generated method stub From d62211ae09479263360f4ef32c77a8171a9e85bc Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 13 May 2021 16:26:42 +0100 Subject: [PATCH 23/55] e2e: upgraded node to v16. --- endtoend/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endtoend/Dockerfile b/endtoend/Dockerfile index f25830f60..ef87e8f05 100644 --- a/endtoend/Dockerfile +++ b/endtoend/Dockerfile @@ -1,5 +1,5 @@ #FROM keymetrics/pm2:latest-alpine -FROM node:15-alpine +FROM node:16-alpine #FROM node:10 #FROM mhart/alpine-node:15 @@ -7,7 +7,7 @@ WORKDIR /app/ # install some packages on Alpine RUN apk --no-cache add \ - python \ + python2 \ bash # Copy nodejs package files From e0ad4d46b9344e97c7d786aaba42a829a6fd80f4 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Fri, 14 May 2021 16:43:52 +0100 Subject: [PATCH 24/55] ConsensusPrint: fixed error that made status answer not JSON compliant (missing String quotes). --- src/org/minima/system/brains/ConsensusPrint.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/minima/system/brains/ConsensusPrint.java b/src/org/minima/system/brains/ConsensusPrint.java index 9ba810baf..236cae922 100644 --- a/src/org/minima/system/brains/ConsensusPrint.java +++ b/src/org/minima/system/brains/ConsensusPrint.java @@ -1068,8 +1068,8 @@ public int compare(JSONObject o1, JSONObject o2) { //Add the network connections ArrayList nets = main.getNetworkHandler().getNetClients(); status.put("connections", nets.size()); - - status.put("p2pnodeid", main.getP2P().getNodeId()); + String nodeid = main.getP2P().getNodeId().toString(); + status.put("p2pnodeid", nodeid==null?"unavailable":nodeid); Optional discAddr = main.getP2P().getDiscoveryAddress(); status.put("p2pdiscoveryaddr", discAddr.isPresent()?discAddr.get():"unavailable"); String enr = main.getP2P().getENR(); From d3a23b5d7b5c54f22f97fc34262f25e6c231cf0b Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 20 May 2021 10:27:35 +0100 Subject: [PATCH 25/55] p2p: cleaned up code and added ENR and NodeID to support filtering when connecting to new nodes. --- Dockerfile.arm64v8 | 5 +- endtoend/src/staticTests.js | 9 +- endtoend/src/test_star_static.js | 6 +- resources/log4j2.xml | 2 +- .../minima/system/brains/ConsensusPrint.java | 7 +- .../system/network/base/MinimaClient.java | 28 +++++- .../network/base/NodeRecordConverter.java | 5 +- .../minima/system/network/base/P2PStart.java | 86 ++++++++++++++++--- .../network/base/peer/DiscoveryPeer.java | 21 +++++ 9 files changed, 145 insertions(+), 24 deletions(-) diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index ffea53f0d..9cf5752f8 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -21,6 +21,8 @@ RUN stat build/libs/minima-all.jar #FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.9_11-slim as production-stage FROM arm64v8/adoptopenjdk:11-jdk-hotspot-focal as production-stage +#COPY --from=build-stage /usr/src/minima/build/resources/main/log4j2.xml /opt/minima/log4j2.xml +COPY --from=build-stage /usr/src/minima/src/resources/log4j2.xml /opt/minima/log4j2.xml COPY --from=build-stage /usr/src/minima/build/libs/minima-all.jar /opt/minima/minima.jar #COPY --from=build-stage /usr/src/minima/minimajar.tar /opt/minima/minimajar.tar WORKDIR /opt/minima @@ -33,4 +35,5 @@ RUN ls -l *.jar RUN stat minima.jar # 9001 minima protocol 9002 REST 9003 WebSocket 9004 MiniDapp Server EXPOSE 9001 9002 9003 9004 -ENTRYPOINT ["java", "-jar", "minima.jar"] +#ENTRYPOINT ["java", "-jar", "minima.jar"] +ENTRYPOINT ["java", "-Dlog4j.configurationFile=log4j2.xml", "-jar", "minima.jar"] diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index a661c325a..b4f11c750 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -59,7 +59,7 @@ const start_docker_node_1 = async function (topology, nbNodes, tests_collection) await containers["1"].start(); process.stdout.write("Trying to sleep for 5 seconds..."); await sleep(5000); - process.stdout.write("Did I sleep 5 seconds?"); +// process.stdout.write("Did I sleep 5 seconds?"); containers["1"].inspect(function (err, data) { ip_addrs["1"] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; @@ -157,14 +157,15 @@ get_node_p2p_params = function(host, cb) { console.log("response.data.response: " + response.data.response); //data = JSON.parse(response.data); console.log("response.data.status: " + response.data.status); - console.log("response.data.response.p2penr: " + response.data.response.p2penr); + console.log("response.data.response.p2pEnr: " + response.data.response.p2pEnr); + //TODO: check values for p2pEnr and p2pDiscoveryAddr, block if empty //console.log("data: " + data); //console.log("data.status: " + data.status); //console.log("data.response.p2penr: " + data.response.p2penr); if(response.data.status == true) { console.log("received data with status = true, extracting p2p fields"); - disc = response.data.response.p2pdiscoveryaddr; - enr = response.data.response.p2penr; + disc = response.data.response.p2pDiscoveryaddr; + enr = response.data.response.p2pEnr; console.log("disc=" + disc + " enr=" + enr); p2pdiscoveryaddr = disc; p2penr = enr; diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 867f20cd3..2e448d3f5 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -22,12 +22,12 @@ test_star_static = function () { // curl -s 127.0.0.1:9002/status | jq '.response.connections' staticTests.run_some_tests_get(ip_addrs["1"], '/status', "", function (response) { response.connections.should.be.above(0); - response.connections.should.be.equal((nbNodes-1)*2); + response.connections.should.be.above((nbNodes-1)); // be at least connected to each node }); staticTests.run_some_tests_get(ip_addrs["3"], '/status', "", function (response) { response.connections.should.be.above(0); - response.connections.should.be.equal((nbNodes-1)*2); + response.connections.should.be.above((nbNodes-1)); }); //1. send funds with no money and assert failure //staticTests.run_some_tests_get(ip_addrs["1"], '/send', {"amount": 1, "address": "0xFF", "tokenid": "0x00"}, @@ -62,7 +62,7 @@ test_star_static = function () { setTimeout(function() { staticTests.run_some_tests_get(ip_addrs["2"], '/status', "", function (response) { response.connections.should.be.above(0); - response.connections.should.be.equal((nbNodes-1)*2); + response.connections.should.be.above((nbNodes-1)); }); }, 10000 diff --git a/resources/log4j2.xml b/resources/log4j2.xml index 609151a79..d953bc3fc 100644 --- a/resources/log4j2.xml +++ b/resources/log4j2.xml @@ -1,5 +1,5 @@ - + diff --git a/src/org/minima/system/brains/ConsensusPrint.java b/src/org/minima/system/brains/ConsensusPrint.java index 236cae922..c6cd3b9cd 100644 --- a/src/org/minima/system/brains/ConsensusPrint.java +++ b/src/org/minima/system/brains/ConsensusPrint.java @@ -1069,11 +1069,12 @@ public int compare(JSONObject o1, JSONObject o2) { ArrayList nets = main.getNetworkHandler().getNetClients(); status.put("connections", nets.size()); String nodeid = main.getP2P().getNodeId().toString(); - status.put("p2pnodeid", nodeid==null?"unavailable":nodeid); + status.put("p2pNodeid", nodeid==null?"unavailable":nodeid); Optional discAddr = main.getP2P().getDiscoveryAddress(); - status.put("p2pdiscoveryaddr", discAddr.isPresent()?discAddr.get():"unavailable"); + status.put("p2pDiscoveryaddr", discAddr.isPresent()?discAddr.get():"unavailable"); String enr = main.getP2P().getENR(); - status.put("p2penr", enr!=null?enr:"unavailable"); + status.put("p2pEnr", enr!=null?enr:"unavailable"); + status.put("p2pPeercount", main.getP2P().getP2PPeerCount()); //Add it to the output InputHandler.endResponse(zMessage, true, ""); diff --git a/src/org/minima/system/network/base/MinimaClient.java b/src/org/minima/system/network/base/MinimaClient.java index 4862e5194..dbe9d0a17 100644 --- a/src/org/minima/system/network/base/MinimaClient.java +++ b/src/org/minima/system/network/base/MinimaClient.java @@ -71,6 +71,10 @@ public class MinimaClient extends MessageProcessor { //The UID String mUID; + // NodeID + private String nodeID; // this should never change + private String nodeRecord; // this is updated when node IP changes + //The Host and Port String mHost; int mPort; @@ -84,7 +88,21 @@ public class MinimaClient extends MessageProcessor { */ boolean mReconnect = false; int mReconnectAttempts = 0; - + + /** + * Constructor + * + * @param zSock + * @param zNetwork + * @throws IOException + * @throws UnknownHostException + */ + public MinimaClient(String zHost, int zPort, NetworkHandler zNetwork, String nodeID, String nodeRecord) { + this(zHost, zPort, zNetwork); + this.nodeID = nodeID; + this.nodeRecord = nodeRecord; + } + /** * Constructor * @@ -160,6 +178,14 @@ public String getUID() { return mUID; } + public String getNodeID() { + return nodeID; + } + + public String getNodeRecord() { + return nodeRecord; + } + public NetworkHandler getNetworkHandler() { return mNetworkMain; } diff --git a/src/org/minima/system/network/base/NodeRecordConverter.java b/src/org/minima/system/network/base/NodeRecordConverter.java index aa773e106..2a55f619e 100644 --- a/src/org/minima/system/network/base/NodeRecordConverter.java +++ b/src/org/minima/system/network/base/NodeRecordConverter.java @@ -50,8 +50,11 @@ private static DiscoveryPeer socketAddressToDiscoveryPeer( DiscV5Service.SUBNET_SUBSCRIPTIONS_SCHEMA::fromBytes) .orElse(DiscV5Service.SUBNET_SUBSCRIPTIONS_SCHEMA.getDefault()); + + Bytes nodeId = nodeRecord.getNodeId(); + String enr = nodeRecord.asEnr(); return new DiscoveryPeer( - ((Bytes) nodeRecord.get(EnrField.PKEY_SECP256K1)), address, enrForkId, persistentSubnets); + ((Bytes) nodeRecord.get(EnrField.PKEY_SECP256K1)), address, enrForkId, persistentSubnets, nodeId, enr); } private static Optional parseField( diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 87bd782ca..050cd1acc 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -6,13 +6,18 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.InetSocketAddress; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import java.util.stream.Stream; // Import log4j classes. @@ -24,6 +29,7 @@ import org.minima.system.network.NetworkHandler; import org.minima.system.network.base.LibP2PNetwork.PrivateKeyProvider; import org.minima.system.network.base.libp2p.PrivateKeyGenerator; +import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; @@ -56,6 +62,16 @@ public class P2PStart extends MessageProcessor { private NodeId nodeId; private PubKey pubKey; + class MinimaNodeInfo { + final public InetSocketAddress socket; + final public String nodeRecord; + final public String nodeID; + public MinimaNodeInfo(String nodeID, String nodeRecord, InetSocketAddress socket) { + this.nodeID = nodeID; + this.nodeRecord = nodeRecord; + this.socket = socket; + } + } // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr @@ -118,6 +134,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee logger.warn("P2P layer - generated node private key: " + privKey.toString()); System.out.println("P2P layer - generated node private key: " + privKey.toString()); //System.out.println("P2P layer - generated node private key bytes: " + String. privKey.raw(); + // generate ENR nodeid - not to be confused with libp2pnodeid which is session specific! nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); System.out.println("P2P layer - nodeid: " + nodeId.toString()); System.out.println("P2P layer - nodeid base58: " + nodeId.toBase58()); @@ -153,7 +170,6 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee // initialization failed - what do we do? logger.error("Failed to start P2P network."); } - } public NodeId getNodeId() { @@ -167,14 +183,27 @@ public PubKey getPubKey() { public String getENR() { return network.getENR(); } - + public Optional getDiscoveryAddress() { return network.getDiscoveryAddress(); } + public int getP2PPeerCount() { + // return network.get + return network.getP2PPeerCount(); + } + + public Stream streamPeers() { + return network.streamPeers(); + } + + public Stream streamDiscoveryPeers() { + return network.streamKnownDiscoveryPeers(); + } + + @Override protected void processMessage(Message zMessage) throws Exception { - // TODO Auto-generated method stub logger.warn("P2PStart received message: " + zMessage.toString()); if(zMessage.isMessageType(P2P_START_SCAN)) { @@ -182,26 +211,63 @@ protected void processMessage(Message zMessage) throws Exception { if(network != null) { Set activeKnownNodes = new HashSet<>(); while (true) { + + // we dont really care about this list... network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true + }); + + ArrayList mClients = mNetwork.getNetClients(); + + Set knownNodeIDs = new HashSet<>(); + + for(MinimaClient mClient: mClients) { + logger.debug(" mclient nodeid=" + mClient.getNodeID() + ", nodeRecord=" + mClient.getNodeRecord()); + knownNodeIDs.add(mClient.getNodeID()); + } Set newActiveNodes = new HashSet<>(); + Set unconnectedNewNodes = new HashSet<>(); //logger.debug("trying to stream discovery peers"); network.streamKnownDiscoveryPeers() .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr - // logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey()); + PeerId peerid = new PeerId(discoPeer.getNodeID().toArray()); + logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey() + + " peerid: " + peerid + + " nodeid:" + discoPeer.getNodeID().toHexString() + " enr: " + discoPeer.getNodeRecord()); + //TODO: establish link between Bytes nodeID and libp2p nodeid / peerid + //TODO: verify values for nodeid and enr and filter existing nodes vs new based on nodeid newActiveNodes.add(discoPeer.getNodeAddress()); + + if(!knownNodeIDs.contains(discoPeer.getNodeID().toString())) { + logger.debug("FOUND NEW NODE: nodeid:" + discoPeer.getNodeID().toString() + " " + discoPeer.getNodeRecord().toString()); + unconnectedNewNodes.add(new MinimaNodeInfo(discoPeer.getNodeID().toHexString(), + discoPeer.getNodeRecord().toString(), + discoPeer.getNodeAddress())); + } else { + logger.debug("SKIPPING an already connected node: nodeid:" + discoPeer.getNodeID().toHexString() + " " + discoPeer.getNodeRecord().toString()); + } }); + Set delta = new HashSet(newActiveNodes); delta.removeAll(activeKnownNodes); //now contains only new sockets - - for(InetSocketAddress i: delta) { - logger.info("New peer address: " + i.toString().substring(1)); - System.out.println("Starting MinimaClient: " + i.toString().substring(1) + ":9001"); - MinimaClient mclient = new MinimaClient(i.getAddress().toString().substring(1), 9001, mNetwork); // hardcode port for now - mNetwork.PostMessage(new Message(NetworkHandler.NETWORK_NEWCLIENT).addObject("client", mclient)); + + for(MinimaNodeInfo i: unconnectedNewNodes) { + logger.info("New peer address: " + i.socket.toString().substring(1)); + // TODO: replace ENR with nodeID, but P2PStart.nodeID is not the correct value (16... and not the bytes) + if (i.nodeRecord.compareTo(network.getENR())==0) { + logger.warn("IGNORING node ENR in list of new peers."); + } else if(i.nodeRecord == null || i.nodeID == null) { + logger.warn("IGNORING empty ndoeRecord or nodeID."); + } else { + logger.info("CONNECTING to new ENR " + i.nodeRecord); + System.out.println("Starting MinimaClient: " + i.socket.toString().substring(1) + ":9001"); + String nodeRecord = i.nodeRecord, nodeID = i.nodeID; + MinimaClient mclient = new MinimaClient(i.socket.getAddress().toString().substring(1), 9001, mNetwork, nodeID, nodeRecord); // hardcode port for now + mNetwork.PostMessage(new Message(NetworkHandler.NETWORK_NEWCLIENT).addObject("client", mclient)); + } } // update known nodes diff --git a/src/org/minima/system/network/base/peer/DiscoveryPeer.java b/src/org/minima/system/network/base/peer/DiscoveryPeer.java index 44385c154..e5acecffd 100644 --- a/src/org/minima/system/network/base/peer/DiscoveryPeer.java +++ b/src/org/minima/system/network/base/peer/DiscoveryPeer.java @@ -31,6 +31,8 @@ public class DiscoveryPeer { private final InetSocketAddress nodeAddress; private final Optional enrForkId; private final SszBitvector persistentSubnets; + private String nodeRecord; + private Bytes nodeId; public DiscoveryPeer( final Bytes publicKey, @@ -43,6 +45,25 @@ public DiscoveryPeer( this.persistentSubnets = persistentSubnets; } + public DiscoveryPeer( + final Bytes publicKey, + final InetSocketAddress nodeAddress, + final Optional enrForkId, + final SszBitvector persistentSubnets, + Bytes nodeId, String nodeRecord) { + this(publicKey, nodeAddress, enrForkId, persistentSubnets); + this.nodeId = nodeId; + this.nodeRecord = nodeRecord; + } + + public String getNodeRecord() { + return nodeRecord; + } + + public Bytes getNodeID() { + return nodeId; + } + public Bytes getPublicKey() { return publicKey; } From 26dcd1a1a95622141de7727607ba1993cec4edc4 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 26 May 2021 09:20:56 +0100 Subject: [PATCH 26/55] e2e: disabled host autoremove to enable docker container inspection after shutdown for debugging. --- endtoend/README.md | 7 +++-- endtoend/src/staticTests.js | 8 +++--- endtoend/src/test_star_static.js | 45 ++++++++++++++++++++++---------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/endtoend/README.md b/endtoend/README.md index 27e0cb578..1d1f13910 100644 --- a/endtoend/README.md +++ b/endtoend/README.md @@ -7,7 +7,7 @@ Quick Start cd minima_root_dir docker network create minima-e2e-testnet docker build -t minima:latest . # OR on ARM: docker build -t minima:latest -f Dockerfile.arm64v8 . - cd end2end + cd endtoend docker build -t minima-e2e . && docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e Setup @@ -19,8 +19,7 @@ build minima docker image (or pull it) - default image used is minima:latest #ARM: docker build -t minima:latest -f Dockerfile.arm64v8 . build nodejs tests docker image (same for ARM and x64): - cd end2end - docker build -t minima-e2e . + docker build -t minima-e2e endtoend stop all running docker images (useful to stop instances manually, otherwise script stops automatically old instances at restart): docker stop $(docker ps -a -q) @@ -29,5 +28,5 @@ run docker instance to create network and perform network connectivity check (re docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e All in one: - docker build -t minima-e2e . && docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e + docker build -t minima-e2e endtoend && docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index b4f11c750..790110fd8 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -20,10 +20,12 @@ const cfg = { node_prefix: "minima-node-", HTTP_TIMEOUT: 30000, DELAY_BEFORE_TESTS: 5000, - hostConfig1: { AutoRemove: true, NetworkMode: "minima-e2e-testnet", 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'] }, - hostConfig: { AutoRemove: true, NetworkMode: "minima-e2e-testnet" }, + hostConfig1: { // AutoRemove: true, // comment this out to inspect stopped containers + NetworkMode: "minima-e2e-testnet", 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'] }, + hostConfig: { // AutoRemove: true, // comment this out to inspect stopped containers + NetworkMode: "minima-e2e-testnet" }, // unused - can be applied on a node to expose its RPC port on localhost - not needed for our tests - hostCfgExpose: { AutoRemove: true, NetworkMode: "minima-e2e-testnet", PortBindings: {"9002/tcp": [ { "HostPort": "9002"} ] } }, + hostCfgExpose: { NetworkMode: "minima-e2e-testnet", PortBindings: {"9002/tcp": [ { "HostPort": "9002"} ] } }, host_port: 9002, TOPO_STAR: "star", TOPO_LINE: "line" diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 2e448d3f5..14bd5a7a9 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -4,7 +4,7 @@ require('chai') .use(require('chai-as-promised')); require('chai').assert; -const nbNodes = 3; +const nbNodes = 9; function sleep(ms) { @@ -22,12 +22,12 @@ test_star_static = function () { // curl -s 127.0.0.1:9002/status | jq '.response.connections' staticTests.run_some_tests_get(ip_addrs["1"], '/status', "", function (response) { response.connections.should.be.above(0); - response.connections.should.be.above((nbNodes-1)); // be at least connected to each node + //response.connections.should.be.equal((nbNodes-1)); // be at least connected to each node }); staticTests.run_some_tests_get(ip_addrs["3"], '/status', "", function (response) { response.connections.should.be.above(0); - response.connections.should.be.above((nbNodes-1)); + //response.connections.should.be.equal((nbNodes-1)); }); //1. send funds with no money and assert failure //staticTests.run_some_tests_get(ip_addrs["1"], '/send', {"amount": 1, "address": "0xFF", "tokenid": "0x00"}, @@ -47,25 +47,42 @@ test_star_static = function () { // 3. send funds with money //staticTests.run_some_tests_get(ip_addrs["1"], '/send', {"amount": 1, "address": "0xFF", "tokenid": "0x00"}, - setTimeout(function () { - staticTests.run_some_tests_get(ip_addrs["1"], '/send', params="+1+0xFF", + setTimeout( + function () { + staticTests.run_some_tests_get( + ip_addrs["1"], + '/send', + params="+1+0xFF", tests=function (response) { - console.log("send response: " + JSON.stringify(response.txpow.body.txn)); + //console.log("send response: " + JSON.stringify(response.txpow.body.txn)); + console.log("received minima response to send tx, verifying tx fields."); response.txpow.body.txn.inputs[0].amount.should.be.equal("25"); response.txpow.body.txn.outputs[0].amount.should.be.equal("1"); response.txpow.body.txn.outputs[1].amount.should.be.equal("24"); response.txpow.body.txn.outputs[0].address.should.be.equal("0xFF"); - })}, 10000); + } + ) + }, + 10000); - // await sleep(30*1000); - setTimeout(function() { - staticTests.run_some_tests_get(ip_addrs["2"], '/status', "", function (response) { - response.connections.should.be.above(0); - response.connections.should.be.above((nbNodes-1)); - }); + setTimeout( + function() { + for(child = 1; child < nbNodes; child++) { + console.log("connecting to node " + child + " to verify status."); + staticTests.run_some_tests_get( + ip_addrs[child.toString()], + '/status', + "", + function (response) { + console.log("status for node " + child + " received, verifying correct number of Minima socket connections."); + response.connections.should.be.above(0); + response.connections.should.be.equal((nbNodes-1)*2); + } + ); + } }, - 10000 + 30000 ); }); From e62b9f61e4a010b23039ea2cf36c58a20d8107dd Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 26 May 2021 09:21:57 +0100 Subject: [PATCH 27/55] p2p: removed unused imports. --- src/main/resources/log4j2-test.properties | 3 +++ src/main/resources/log4j2.xml | 14 ++++++++++++++ src/org/minima/system/network/base/P2PStart.java | 15 --------------- src/resources/log4j2-test.properties | 3 +++ src/resources/log4j2.xml | 14 ++++++++++++++ 5 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 src/main/resources/log4j2-test.properties create mode 100644 src/main/resources/log4j2.xml create mode 100644 src/resources/log4j2-test.properties create mode 100644 src/resources/log4j2.xml diff --git a/src/main/resources/log4j2-test.properties b/src/main/resources/log4j2-test.properties new file mode 100644 index 000000000..24bdc81ee --- /dev/null +++ b/src/main/resources/log4j2-test.properties @@ -0,0 +1,3 @@ +# Root logger option +log4j.rootLogger=TRACE, stdout + diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 000000000..609151a79 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 050cd1acc..7917035e7 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -6,26 +6,16 @@ import java.io.FileOutputStream; import java.io.IOException; import java.net.InetSocketAddress; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import java.util.stream.Stream; -// Import log4j classes. - // Import log4j classes. import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; -import org.ethereum.beacon.discovery.schema.NodeRecord; -import org.ethereum.beacon.discovery.schema.NodeRecordInfo; import org.minima.system.network.NetworkHandler; import org.minima.system.network.base.LibP2PNetwork.PrivateKeyProvider; import org.minima.system.network.base.libp2p.PrivateKeyGenerator; @@ -41,11 +31,6 @@ import io.libp2p.core.crypto.PubKey; import io.libp2p.core.crypto.KEY_TYPE; import io.libp2p.core.crypto.KeyKt; -import io.libp2p.crypto.keys.Secp256k1PrivateKey; - -import io.libp2p.etc.encode.Base58; - -import org.apache.logging.log4j.LogManager; public class P2PStart extends MessageProcessor { diff --git a/src/resources/log4j2-test.properties b/src/resources/log4j2-test.properties new file mode 100644 index 000000000..24bdc81ee --- /dev/null +++ b/src/resources/log4j2-test.properties @@ -0,0 +1,3 @@ +# Root logger option +log4j.rootLogger=TRACE, stdout + diff --git a/src/resources/log4j2.xml b/src/resources/log4j2.xml new file mode 100644 index 000000000..d953bc3fc --- /dev/null +++ b/src/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + From 258742a693518ab84dc0361e73ed5dde46fa0a12 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 26 May 2021 09:22:36 +0100 Subject: [PATCH 28/55] gradle: added gradle.properties file to configure log4j2 reliably. --- gradle.properties | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 gradle.properties diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 000000000..1f218a8c7 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,2 @@ +org.gradle.jvmargs='-Dlog4j.configurationFile=resources/log4j2.xml' + From d6276e147d5a3625470c8957cd03a822e53638f3 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 27 May 2021 10:06:14 +0100 Subject: [PATCH 29/55] e2e: refactored code and re-enabled auto-remove containers at termination. --- endtoend/src/staticTests.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 790110fd8..541f5bab0 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -20,10 +20,14 @@ const cfg = { node_prefix: "minima-node-", HTTP_TIMEOUT: 30000, DELAY_BEFORE_TESTS: 5000, - hostConfig1: { // AutoRemove: true, // comment this out to inspect stopped containers - NetworkMode: "minima-e2e-testnet", 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'] }, - hostConfig: { // AutoRemove: true, // comment this out to inspect stopped containers - NetworkMode: "minima-e2e-testnet" }, + hostConfig1: { + AutoRemove: true, // comment this out to inspect stopped containers + NetworkMode: "minima-e2e-testnet", 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'] + }, + hostConfig: { + AutoRemove: true, // comment this out to inspect stopped containers + NetworkMode: "minima-e2e-testnet" + }, // unused - can be applied on a node to expose its RPC port on localhost - not needed for our tests hostCfgExpose: { NetworkMode: "minima-e2e-testnet", PortBindings: {"9002/tcp": [ { "HostPort": "9002"} ] } }, host_port: 9002, From a443b3aaa1d903dc0fc080589bf9f5223d393f3a Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 27 May 2021 13:27:41 +0100 Subject: [PATCH 30/55] p2p: added cpushares option and increased nbNodes to 10. --- endtoend/src/staticTests.js | 7 +++++-- endtoend/src/test_star_static.js | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 541f5bab0..32a137c67 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -22,11 +22,14 @@ const cfg = { DELAY_BEFORE_TESTS: 5000, hostConfig1: { AutoRemove: true, // comment this out to inspect stopped containers - NetworkMode: "minima-e2e-testnet", 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'] + NetworkMode: "minima-e2e-testnet", + 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'], + CpuShares: 10, // node 1 in private mode uses auto-mining and aim for 100% CPU usage, so we throttle it }, hostConfig: { AutoRemove: true, // comment this out to inspect stopped containers - NetworkMode: "minima-e2e-testnet" + NetworkMode: "minima-e2e-testnet", + CpuShares: 10, }, // unused - can be applied on a node to expose its RPC port on localhost - not needed for our tests hostCfgExpose: { NetworkMode: "minima-e2e-testnet", PortBindings: {"9002/tcp": [ { "HostPort": "9002"} ] } }, diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 14bd5a7a9..5b2dcb53c 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -4,7 +4,7 @@ require('chai') .use(require('chai-as-promised')); require('chai').assert; -const nbNodes = 9; +const nbNodes = 10; function sleep(ms) { From 7f9461e71485dd0ab8c82a378ff600668c148e4c Mon Sep 17 00:00:00 2001 From: panda Date: Thu, 3 Jun 2021 17:28:49 -0400 Subject: [PATCH 31/55] implemented docker run with params --- endtoend/Dockerfile | 6 ++++++ endtoend/README.md | 4 ++-- endtoend/src/test_star_static.js | 3 ++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/endtoend/Dockerfile b/endtoend/Dockerfile index ef87e8f05..2d5d935fb 100644 --- a/endtoend/Dockerfile +++ b/endtoend/Dockerfile @@ -26,4 +26,10 @@ COPY src /app/src CMD ["node", "src/index.js"] +ARG nbNodes + +ENV nbNodes=$nbNodes + +RUN echo $nbNodes + diff --git a/endtoend/README.md b/endtoend/README.md index 1d1f13910..5498ca30f 100644 --- a/endtoend/README.md +++ b/endtoend/README.md @@ -25,8 +25,8 @@ stop all running docker images (useful to stop instances manually, otherwise scr docker stop $(docker ps -a -q) run docker instance to create network and perform network connectivity check (requires at least one connection): - docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e + docker run -v /var/run/docker.sock:/var/run/docker.sock --env nbNodes=3 --network minima-e2e-testnet minima-e2e All in one: - docker build -t minima-e2e endtoend && docker run -v /var/run/docker.sock:/var/run/docker.sock --network minima-e2e-testnet minima-e2e + docker build -t minima-e2e endtoend && docker run -v /var/run/docker.sock:/var/run/docker.sock --env nbNodes=3 --network minima-e2e-testnet minima-e2e diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 5b2dcb53c..5318bc562 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -4,8 +4,9 @@ require('chai') .use(require('chai-as-promised')); require('chai').assert; -const nbNodes = 10; +const nbNodes = parseInt(process.env.nbNodes); +console.log("test===>", nbNodes) function sleep(ms) { return new Promise((resolve) => { From 68ed9f47fe0ee7473529fe34e17b4488312872a2 Mon Sep 17 00:00:00 2001 From: panda Date: Wed, 9 Jun 2021 02:14:03 -0400 Subject: [PATCH 32/55] updated endtoend with minima API --- endtoend/Dockerfile | 4 +- endtoend/minima-api-1.0.0.tgz | Bin 0 -> 1366 bytes endtoend/package-lock.json | 19 ++++++- endtoend/package.json | 3 +- endtoend/src/test_star_static.js | 83 ++++++------------------------- 5 files changed, 38 insertions(+), 71 deletions(-) create mode 100644 endtoend/minima-api-1.0.0.tgz diff --git a/endtoend/Dockerfile b/endtoend/Dockerfile index 2d5d935fb..1e1c4f13b 100644 --- a/endtoend/Dockerfile +++ b/endtoend/Dockerfile @@ -11,7 +11,7 @@ RUN apk --no-cache add \ bash # Copy nodejs package files -COPY package-lock.json package.json /app/ +COPY package-lock.json package.json minima-api-1.0.0.tgz /app/ # Install packages #ENV NPM_CONFIG_LOGLEVEL warn @@ -26,7 +26,7 @@ COPY src /app/src CMD ["node", "src/index.js"] -ARG nbNodes +ARG nbNodes=3 ENV nbNodes=$nbNodes diff --git a/endtoend/minima-api-1.0.0.tgz b/endtoend/minima-api-1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..79df29f2d18584121471e3827ae1ee58fcf0f999 GIT binary patch literal 1366 zcmV-c1*!TUiwFP!00002|LvObPvbTe$N9N`g%wSAMc1_{P|%7xX*!)Wq&w}Tv7dGu zXOh>(M`Gt}hZdp!?|Y7uw19h>*aK}7Mk3Vq`+WTx`}rlNLo5?+HpA z$HzxUEL0!IhVVGnC3$u5l;?PGVu*^*>S)C5>f# zC6r+^rr~QPH1wmoBHBOg^?Zk9zQE?&9%DbBpMShKKl||Rm*38(?9e&eTlad`T(b{C zih^JK^zp3ji=B0uWD3ovEXu4cr=!vM;A9x%&v^Pn93Mmj*9*<;l+7!dSfOO!S?t$Z zYzsMTSu^&zVSRV&D%LgWGj?Csv<&ici&k>Y3p0zN7OBBjTCxS$i^9`Erlk_n_8+PZ zSTu6LXuv#ZztgdmRa)?TOY6}REemCEpq&IxOz;sBUe8ejiX7J*Qh@H8Y>A;@p?N~6RO13N66{iGlI7f>?yY9r4J0I%aQ1m% zz?W1QazxAzbYN8~ZLyK=&UMIuoG8fU zI%H@=Mms|p5)rpc2FH;Du~y9t&JHMD2RgZy7}_E6el87gon(ajrF3eUSWCSy#frjOFM5=$Nw>wm=WC}TuhN%`Amh)LGshmfK<$M-PD(8_q+v`Zro=zM z4Q0tib6z*Z!S+#Uz;3N5K)JY(%ooG$6S_4vVz%;DNej}x2f47BKefXacH4r_Z%Vv( zuW>j+zFzM);>~3=!YoauIWSWt2^iDVi+R5r*sE9FAcy{>iOpv0AdWY$ty*_f4#Qk6 znh7%4*x|kA%{TBspl;kETV26jbT`NDnzWy`x$EBcbajIXX~>`v`{Z3=!*ZCa#Cfi=^I&2a&F3d0$0yg`9c z#r16e$HdpR@35xLa*HhBV*qvojMD5=^v*4-DU*u124+e~#9To7DRR*URu~4yk>3%} z_zG^8N~aF@`yX`;?pB%74(RoAkpKn)S8?|1+t!s5gEPoPz}7(gKR;l", nbNodes) @@ -16,78 +18,25 @@ function sleep(ms) { test_star_static = function () { // number of nodes, list of tests - staticTests.start_static_network_tests("star", nbNodes, function (ip_addrs) { + staticTests.start_static_network_tests("star", nbNodes, async function (ip_addrs) { console.log("tests collection"); - // test 0: healthcheck - are we connected? - // curl -s 127.0.0.1:9002/status | jq '.response.connections' - staticTests.run_some_tests_get(ip_addrs["1"], '/status', "", function (response) { - response.connections.should.be.above(0); - //response.connections.should.be.equal((nbNodes-1)); // be at least connected to each node - }); - - staticTests.run_some_tests_get(ip_addrs["3"], '/status', "", function (response) { - response.connections.should.be.above(0); - //response.connections.should.be.equal((nbNodes-1)); - }); - //1. send funds with no money and assert failure - //staticTests.run_some_tests_get(ip_addrs["1"], '/send', {"amount": 1, "address": "0xFF", "tokenid": "0x00"}, - staticTests.run_some_tests_get(ip_addrs["1"], '/send', params="+1+0xFF", - tests=function (response) { - console.log("chai tests not written yet for this test, just printing answer."); - console.log("send response: " + JSON.stringify(response)); - }); + await sleep(30000); - // 2. generate 50 coins - staticTests.run_some_tests_get(ip_addrs["1"], '/gimme50', "", tests = function (response) { - // response.connections.should.be.above(0); - // response.chainlength.should.be.above(1); - console.log("chai tests not written yet for this test, just printing answer."); - console.log("gimme50 response: " + JSON.stringify(response)); - }); + for(child = 1; child < nbNodes+1; child++) { + console.log("connecting to node " + child + " to verify status."); + await Minima_API.init(ip_addrs[child.toString()]); - // 3. send funds with money - //staticTests.run_some_tests_get(ip_addrs["1"], '/send', {"amount": 1, "address": "0xFF", "tokenid": "0x00"}, - setTimeout( - function () { - staticTests.run_some_tests_get( - ip_addrs["1"], - '/send', - params="+1+0xFF", - tests=function (response) { - //console.log("send response: " + JSON.stringify(response.txpow.body.txn)); - console.log("received minima response to send tx, verifying tx fields."); - response.txpow.body.txn.inputs[0].amount.should.be.equal("25"); - response.txpow.body.txn.outputs[0].amount.should.be.equal("1"); - response.txpow.body.txn.outputs[1].amount.should.be.equal("24"); - response.txpow.body.txn.outputs[0].address.should.be.equal("0xFF"); - } - ) - }, - 10000); - - // await sleep(30*1000); - setTimeout( - function() { - for(child = 1; child < nbNodes; child++) { - console.log("connecting to node " + child + " to verify status."); - staticTests.run_some_tests_get( - ip_addrs[child.toString()], - '/status', - "", - function (response) { - console.log("status for node " + child + " received, verifying correct number of Minima socket connections."); - response.connections.should.be.above(0); - response.connections.should.be.equal((nbNodes-1)*2); - } - ); - } - }, - 30000 - ); - - }); + await Minima_API.status().then(res => { + console.log("status====>", res) + }); + await Minima_API.network().then(res => { + console.log("network====>", res) + }); + } + }) } + module.exports = test_star_static From 2b0e9c8c83ebee71b0d8f26c15e5e20f76e8fc98 Mon Sep 17 00:00:00 2001 From: panda Date: Thu, 10 Jun 2021 17:09:10 -0400 Subject: [PATCH 33/55] saved result to the text file --- endtoend/Dockerfile | 8 +- endtoend/package-lock.json | 11 ++ endtoend/package.json | 1 + endtoend/src/staticTests.js | 166 ++++++++++++++----------------- endtoend/src/test_star_static.js | 50 ++++++++-- 5 files changed, 130 insertions(+), 106 deletions(-) diff --git a/endtoend/Dockerfile b/endtoend/Dockerfile index 1e1c4f13b..d94694840 100644 --- a/endtoend/Dockerfile +++ b/endtoend/Dockerfile @@ -26,10 +26,14 @@ COPY src /app/src CMD ["node", "src/index.js"] +ARG topology=star ARG nbNodes=3 +ARG nodeFailure=2 +ENV topology=$topology ENV nbNodes=$nbNodes +ENV nodeFailure=$nodeFailure +RUN echo $topology RUN echo $nbNodes - - +RUN echo $nodeFailure diff --git a/endtoend/package-lock.json b/endtoend/package-lock.json index 8c0e06113..887ea7158 100644 --- a/endtoend/package-lock.json +++ b/endtoend/package-lock.json @@ -14,6 +14,7 @@ "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", "dockerode": "^3.2.1", + "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" } }, @@ -236,6 +237,11 @@ } } }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -595,6 +601,11 @@ "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.2.tgz", "integrity": "sha512-6mPTgLxYm3r6Bkkg0vNM0HTjfGrOEtsfbhagQvbxDEsEkpNhw582upBaoRZylzen6krEmxXJgt9Ju6HiI4O7BA==" }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ=" + }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", diff --git a/endtoend/package.json b/endtoend/package.json index 68933bd7b..c7e964e09 100644 --- a/endtoend/package.json +++ b/endtoend/package.json @@ -14,6 +14,7 @@ "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", "dockerode": "^3.2.1", + "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" } } diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 32a137c67..9ad92e628 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -67,19 +67,23 @@ const start_docker_node_1 = async function (topology, nbNodes, tests_collection) // Start the container. await containers["1"].start(); process.stdout.write("Trying to sleep for 5 seconds..."); - await sleep(5000); + await sleep(10000); // process.stdout.write("Did I sleep 5 seconds?"); - containers["1"].inspect(function (err, data) { + console.log("Hello1"); + await runContainerInspect(topology, nbNodes, tests_collection); +} + +const runContainerInspect = (topology, nbNodes, tests_collection) => { + return new Promise((resolve, reject) => { + containers["1"].inspect(async function(err, data) { ip_addrs["1"] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; - //TODO: add curl status and retrieve discovery address and ENR -// process.stdout.write("Trying to sleep for 10 seconds..."); -// await sleep(10000); -// process.stdout.write("Did I sleep 10 seconds?"); + console.log("Started node 1," + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); get_node_p2p_params(ip_addrs["1"], function() { - start_other_nodes(topology, nbNodes, tests_collection); - }); - }); + resolve(start_other_nodes(topology, nbNodes, tests_collection)); + }); + }) + }) } get_node_args = function(topology, pos) { @@ -108,16 +112,28 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { console.log("topo star node " + pos + " args: " + node_args); containers[pos] = await createMinimaContainer(node_args, cfg.node_prefix + pos, cfg.hostConfig); // Start the container. + await containers[pos].start(); - containers[pos].inspect(function (err, data) { + + await sleep(10000); + console.log("node " + pos); + await starContainerInspect(pos, nbNodes, tests_collection); + } +} + +const starContainerInspect = (pos, nbNodes, tests_collection) => { + return new Promise((resolve) => { + containers[''+pos].inspect(async function(err, data) { console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; - if(pos == nbNodes) { // run tests after we created last node - // need to sleep to let node sync with others - setTimeout(function () { tests_collection(ip_addrs) }, cfg.DELAY_BEFORE_TESTS); + if(pos == nbNodes) { + await sleep(5000); + resolve(tests_collection(0, ip_addrs)) + } else { + resolve(null) } - }); - } + }) + }) } start_other_nodes_line = async function (nbNodes, pos, tests_collection) { @@ -130,23 +146,31 @@ start_other_nodes_line = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - containers[pos].inspect(function (err, data) { - console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); - ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; - if (pos == nbNodes) { // run tests after we created last node - // need to sleep to let node sync with others - setTimeout(function () { tests_collection(ip_addrs) }, cfg.DELAY_BEFORE_TESTS); - } else { - start_other_nodes_line(nbNodes, pos+1, tests_collection); - } - }); + await sleep(10000); + + await lineContainerInspect(pos, nbNodes, tests_collection); +} + +const lineContainerInspect = (pos, nbNodes, tests_collection) => { + return new Promise((resolve) => { + containers[''+pos].inspect(async function(err, data) { + console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); + ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; + if(pos == nbNodes) { + await sleep(5000); + resolve(tests_collection(0, ip_addrs)) + } else { + resolve(start_other_nodes_line(nbNodes, pos+1, tests_collection)) + } + }) + }) } start_other_nodes = async function (topology, nbNodes, tests_collection) { if(topology === cfg.TOPO_STAR) { - start_other_nodes_star(nbNodes, tests_collection); + await start_other_nodes_star(nbNodes, tests_collection); } else if(topology === cfg.TOPO_LINE) { - start_other_nodes_line(nbNodes, 2, tests_collection); + await start_other_nodes_line(nbNodes, 2, tests_collection); } else { console.log("Unsupported topology! This error should be caught earlier."); console.log(" topology=" + topology); @@ -195,77 +219,25 @@ get_node_p2p_params = function(host, cb) { }); } - -// this function calls HTTP GET on host:endpoint, expects a minima answer, and runs tests_to_run if server success. -run_some_tests_get = async function(host, endpoint, params="", tests) { - const url = "http://" + host + ":" + cfg.host_port + endpoint + params; - - axios.get(url, {timeout: cfg.HTTP_TIMEOUT}, {maxContentLength: 3000}, {responseType: 'plain'}) - .then(function (response) { - // handle success - if(response && response.status == 200) { - console.log(response.data); - if(response.data.status == true) { - // console.log("received data with status = true, calling tests"); - tests(response.data.response); - } else { - console.log("json data incorrect, skipping tests"); - } - } - }) - .catch(function (error) { - // handle error - console.log("axios error handler:"); - console.log(error); - }) - .then(function () { - // always executed - }); -} - -// TODO / DO NOT USE -// this function calls HTTP POST on host:endpoint, expects a minima answer, and runs tests_to_run if server success. -run_some_tests_post = async function(host, endpoint, tests_to_run) { - const url = "http://" + host + ":" + cfg.host_port + endpoint; - axios.post(url, {timeout: cfg.HTTP_TIMEOUT}, {maxContentLength: 3000}, {responseType: 'plain'}) - .then(function (response) { - // handle success - if(response && response.status == 200) { - console.log(response.data); - if(response.data.status == true) { - tests_to_run(response.data.response); - } - } - }) - .catch(function (error) { - // handle error - console.log(error); - }) - .then(function () { - // always executed - }); -} - stop_docker_nodes = async function() { console.log("stop_docker_nodes"); // iterate over all running containers and stop them if their name starts with node_prefix - docker.listContainers({ all:false }, - function (err, containers) { - if(containers) { - containers.forEach(function (containerInfo) { - if(containerInfo.Names[0].startsWith("/" + cfg.node_prefix)) { - console.log ("Found a minima node(" + containerInfo.Names[0] + "), stopping."); - docker.getContainer(containerInfo.Id).stop(); - } - }); - } else { - console.log("Found no running docker instances\n"); - } - }); + docker.listContainers({ all:false }, function (err, containers) { + if(containers) { + containers.forEach(function (containerInfo) { + if(containerInfo.Names[0].startsWith("/" + cfg.node_prefix)) { + console.log ("Found a minima node(" + containerInfo.Names[0] + "), stopping."); + docker.getContainer(containerInfo.Id).stop(); + } + }); + } else { + console.log("Found no running docker instances\n"); + } + }); } // setup a network of nbNodes minima nodes in star topology and runs tests_collection on it with argument ip_addrs[node_prefix+ "01"] . -start_static_network_tests = async function (topology, nbNodes, tests_collection) { +start_static_network_tests = async function (topology, nbNodes, nodeFailure, tests_collection) { if(!(topology === cfg.TOPO_STAR || topology === cfg.TOPO_LINE)) { console.log("Error! Unsupported topology: " + topology); return; @@ -283,9 +255,15 @@ start_static_network_tests = async function (topology, nbNodes, tests_collection } await stop_docker_nodes(); // give 5 seconds to stop all docker nodes (should depend on nbNodes but also system performance) - setTimeout(function() { start_docker_node_1(topology, nbNodes, tests_collection); }, 5000); + await sleep(5000); + await start_docker_node_1(topology, nbNodes, tests_collection); + + //stop one node and run tests + await containers[''+nodeFailure].stop(); + console.log("node " + nodeFailure + " stopping..."); + await sleep(10000); + await tests_collection(1, ip_addrs); } exports.cfg = cfg; exports.start_static_network_tests = start_static_network_tests; -exports.run_some_tests_get = run_some_tests_get; diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index dbda7e080..806bda6b3 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -5,10 +5,13 @@ require('chai') require('chai').assert; var { Minima_API } = require('minima-api'); +var fs = require('fs'); +const topology = process.env.topology; const nbNodes = parseInt(process.env.nbNodes); +const nodeFailure = parseInt(process.env.nodeFailure); -console.log("test===>", nbNodes) +console.log("test===>", topology, nbNodes, nodeFailure) function sleep(ms) { return new Promise((resolve) => { @@ -16,24 +19,51 @@ function sleep(ms) { }); } +function writeFunction(data) { + + return new Promise((resolve, reject) => { + fs.appendFile("./log/result.txt", data, function(err) { + if (err) { + reject(err); + } else { + resolve(null) + } + }); + }) +} + test_star_static = function () { - // number of nodes, list of tests - staticTests.start_static_network_tests("star", nbNodes, async function (ip_addrs) { + staticTests.start_static_network_tests(topology, nbNodes, nodeFailure, async function (flag, ip_addrs) { console.log("tests collection"); - await sleep(30000); + //wait for processing(should depend on nbNodes but also system performance) + await sleep(60000); + + if (!fs.existsSync("./log")){ + fs.mkdirSync("./log"); + } + + if (flag == 0) { + await writeFunction("The Result before stopping node.\n") + } + + if(flag == 1) { + await writeFunction("\n\n\nThe Result after stopping node " + nodeFailure + "\n"); + } for(child = 1; child < nbNodes+1; child++) { + if (flag == 1 && child == nodeFailure) continue; console.log("connecting to node " + child + " to verify status."); + await Minima_API.init(ip_addrs[child.toString()]); - await Minima_API.status().then(res => { - console.log("status====>", res) - }); + var status = await Minima_API.status(); + var data = "\n " + ip_addrs[child.toString()] + ":\nStatus: \n" + JSON.stringify(status) + "\n"; + await writeFunction(data); - await Minima_API.network().then(res => { - console.log("network====>", res) - }); + var network = await Minima_API.network(); + data = "network: \n" + JSON.stringify(network) + "\n"; + await writeFunction(data); } }) } From 4a9ce47c992eca23e66c6b4f5b841fd02474bc26 Mon Sep 17 00:00:00 2001 From: panda Date: Fri, 11 Jun 2021 12:41:35 -0400 Subject: [PATCH 34/55] updated e2e test --- .gitignore | 2 ++ endtoend/package-lock.json | 11 ++++++ endtoend/package.json | 1 + endtoend/src/test_star_static.js | 58 +++++++++++++++++--------------- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 197e472e0..8745da007 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ build endtoend/node_modules +endtoend/results + diff --git a/endtoend/package-lock.json b/endtoend/package-lock.json index 887ea7158..3595c06a0 100644 --- a/endtoend/package-lock.json +++ b/endtoend/package-lock.json @@ -13,6 +13,7 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "csv-writer": "^1.6.0", "dockerode": "^3.2.1", "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" @@ -157,6 +158,11 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "node_modules/csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" + }, "node_modules/debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -552,6 +558,11 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" + }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", diff --git a/endtoend/package.json b/endtoend/package.json index c7e964e09..e79cddd20 100644 --- a/endtoend/package.json +++ b/endtoend/package.json @@ -13,6 +13,7 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "csv-writer": "^1.6.0", "dockerode": "^3.2.1", "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 806bda6b3..fdfa940a5 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -6,6 +6,7 @@ require('chai').assert; var { Minima_API } = require('minima-api'); var fs = require('fs'); +const createCsvWriter = require("csv-writer").createObjectCsvWriter; const topology = process.env.topology; const nbNodes = parseInt(process.env.nbNodes); @@ -17,19 +18,6 @@ function sleep(ms) { return new Promise((resolve) => { setTimeout(resolve, ms); }); -} - -function writeFunction(data) { - - return new Promise((resolve, reject) => { - fs.appendFile("./log/result.txt", data, function(err) { - if (err) { - reject(err); - } else { - resolve(null) - } - }); - }) } test_star_static = function () { @@ -39,17 +27,24 @@ test_star_static = function () { //wait for processing(should depend on nbNodes but also system performance) await sleep(60000); - if (!fs.existsSync("./log")){ - fs.mkdirSync("./log"); + if (!fs.existsSync("./results")){ + fs.mkdirSync("./results"); } - if (flag == 0) { - await writeFunction("The Result before stopping node.\n") - } + let current = new Date(); + + let csvWriter = createCsvWriter({ + path: flag == 0 ? `./results/result-${current.getDate()}_${current.getMonth()+1}_${current.getFullYear()}-${current.getHours()}_${current.getMinutes()}-part1.csv` + : `./results/result-${current.getDate()}_${current.getMonth()+1}_${current.getFullYear()}-${current.getHours()}_${current.getMinutes()}-part2.csv`, + header: [ + { id: "node", title: "Node" }, + { id: "ip", title: "IP" }, + { id: "request", title: "Request" }, + { id: "response", title: "Response" }, + ], + }); - if(flag == 1) { - await writeFunction("\n\n\nThe Result after stopping node " + nodeFailure + "\n"); - } + let data = []; for(child = 1; child < nbNodes+1; child++) { if (flag == 1 && child == nodeFailure) continue; @@ -57,16 +52,25 @@ test_star_static = function () { await Minima_API.init(ip_addrs[child.toString()]); + let midData = {}; + midData["node"] = child; + midData["ip"] = ip_addrs[child.toString()]; + var status = await Minima_API.status(); - var data = "\n " + ip_addrs[child.toString()] + ":\nStatus: \n" + JSON.stringify(status) + "\n"; - await writeFunction(data); - + midData["request"] = "status"; + midData["response"] = JSON.stringify(status); + data.push(midData); + + let tempData = {} var network = await Minima_API.network(); - data = "network: \n" + JSON.stringify(network) + "\n"; - await writeFunction(data); + tempData["node"] = child; + tempData["ip"] = ip_addrs[child.toString()]; + tempData["request"] = "network"; + tempData["response"] = JSON.stringify(network); + data.push(tempData); } + await csvWriter.writeRecords(data) }) } - module.exports = test_star_static From f7cf4d9fada81958468142fa832c8362182fd50c Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Mon, 28 Jun 2021 12:03:08 +0100 Subject: [PATCH 35/55] Docker: re-enabled gradlew cache line to avoid gradle dl each time. --- Dockerfile | 2 +- Dockerfile.arm64v8 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4f46e6b88..066cc7096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ COPY gradle gradle COPY gradlew settings.gradle build.gradle ./ WORKDIR /usr/src/minima/ # Call gradlew before copying the source code to only download the gradle distribution once (layer will be cached) -#RUN ./gradlew --no-daemon -v +RUN ./gradlew --no-daemon -v COPY lib lib COPY src src COPY test test diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index 9cf5752f8..dfa8ee410 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -6,7 +6,7 @@ COPY gradle gradle COPY gradlew settings.gradle build.gradle ./ WORKDIR /usr/src/minima/ # Call gradlew before copying the source code to only download the gradle distribution once (layer will be cached) -#RUN ./gradlew --no-daemon -v +RUN ./gradlew --no-daemon -v COPY lib lib COPY src src COPY test test From ce4b5f3ff6531f003334879c9d0fc85a1a40a225 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Mon, 28 Jun 2021 12:03:43 +0100 Subject: [PATCH 36/55] p2p: save and re-load neighbours list V1. --- .../minima/system/network/base/P2PStart.java | 287 ++++++++++++++++-- 1 file changed, 262 insertions(+), 25 deletions(-) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 7917035e7..8b07386b2 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -1,36 +1,60 @@ package org.minima.system.network.base; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; +import java.io.PrintWriter; import java.net.InetSocketAddress; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.Vector; import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; import java.util.stream.Stream; +import static io.netty.buffer.Unpooled.*; + // Import log4j classes. import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.minima.system.network.NetworkHandler; +import org.minima.system.network.base.DiscoveryNetworkFactory.DiscoveryNetworkBuilder; import org.minima.system.network.base.LibP2PNetwork.PrivateKeyProvider; import org.minima.system.network.base.libp2p.PrivateKeyGenerator; +//import org.minima.system.network.base.NodeRecordConverter; import org.minima.system.network.base.peer.DiscoveryPeer; import org.minima.system.network.base.peer.LibP2PNodeId; import org.minima.system.network.base.peer.NodeId; import org.minima.system.network.base.peer.Peer; import org.minima.utils.messages.Message; import org.minima.utils.messages.MessageProcessor; +import org.minima.utils.messages.TimerMessage; + +//import org.ethereum.beacon.discovery.schema.NodeRecord; +import org.ethereum.beacon.discovery.schema.NodeRecord; +import org.ethereum.beacon.discovery.schema.NodeRecordFactory; import io.libp2p.core.PeerId; import io.libp2p.core.crypto.PrivKey; import io.libp2p.core.crypto.PubKey; +import io.libp2p.core.multiformats.Multihash; +import io.libp2p.crypto.keys.Secp256k1Kt; +import io.libp2p.etc.encode.Base58; +import io.netty.buffer.ByteBuf; import io.libp2p.core.crypto.KEY_TYPE; import io.libp2p.core.crypto.KeyKt; +//import io.libp2p.core.crypto.marshalPublicKey; public class P2PStart extends MessageProcessor { @@ -39,38 +63,59 @@ public class P2PStart extends MessageProcessor { // these messages only control LIBP2P+DISCV5 <> Minima comms at the moment public static final String P2P_START_SCAN = "P2P_START_SCAN"; public static final String P2P_STOP_SCAN = "P2P_STOP_SCAN"; - + public static final String P2P_SAVE_NEIGHBOURS = "P2P_SAVE_NEIGHBOURS"; + public static final int SAVE_NEIGHBOURS_DELAY = 10*1000; // save list every 10 seconds + public static final String COMMA_DELIMITER = ","; private String mConfFolder; private NetworkHandler mNetwork; private DiscoveryNetwork network; Set activeKnownNodes; + Set allDiscoveredNodes = new HashSet<>(); private NodeId nodeId; private PubKey pubKey; + private File mP2PBootnodesFile; class MinimaNodeInfo { final public InetSocketAddress socket; final public String nodeRecord; final public String nodeID; - public MinimaNodeInfo(String nodeID, String nodeRecord, InetSocketAddress socket) { + final public String discoMultiAddrTCP; + public MinimaNodeInfo(String nodeID, String nodeRecord, InetSocketAddress socket, String discoMultiAddrTCP) { this.nodeID = nodeID; this.nodeRecord = nodeRecord; this.socket = socket; + this.discoMultiAddrTCP = discoMultiAddrTCP; } } // staticPeers = list of static peers in multiaddr format: /ip4/127.0.0.1/tcp/10219/p2p/16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt // bootnodes = list of ENR: enr:-Iu4QGvbP4hn3cxao3aFyZfeGBG0Ygp-KPJsK9h7pM_0FfCGauk0P2haW7AEiLaMLEDxRngy4SjCx6GGfwlsRBf0BBwBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQMCButDl63KBqEEyxV2R3nCvnHb7sEIgOACbb6yt6oxqYN0Y3CCJ-uDdWRwgifr - public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPeers, String[] bootnodes) { + // ENR above in enr-viewer.com leads to: pubkey=0x030206eb4397adca06a104cb15764779c2be71dbeec10880e0026dbeb2b7aa31a9 + // ip=127.0.0.1, tcp=10219, udp=10219 + // if we protobuf the pubkey (crypto.proto) we get 08021221030206eb4397adca06a104cb15764779c2be71dbeec10880e0026dbeb2b7aa31a9 + // 08 means varint follows with value 02. which means SECP256K1 key + // 12 means String follows, starting with length. length is 0x21 or 33 dec + // now we multihash it. Because the key is small (<50) it is inlined: 002508021221030206eb4397adca06a104cb15764779c2be71dbeec10880e0026dbeb2b7aa31a9 + // 00 -> Identity, 0x25=length (protobuf pubkey SECP256K1, 0x21+0x04) + // and now we base58 encode it: 16Uiu2HAmCnuHVjxoQtZzqenqjRr6hAja1XWCuC1SiqcWcWcp4iSt + // So the hex prefix to the pubkey to build the peerid is always 002508021221 (in our use case). + public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPeers, String[] _bootnodes) { super(P2P_THREAD); this.mConfFolder = zConfFolder; mNetwork = minimaNet; - if(staticPeers == null || staticPeers.length == 0) { - logger.info("P2P layer - no static peer."); + Vector staticPeers = new Vector<>(); + Vector bootnodes = new Vector<>(); + if(_staticPeers == null || _staticPeers.length == 0) { + logger.info("P2P layer - no static peer provided by minima."); } - if(bootnodes == null || bootnodes.length == 0) { - logger.info("P2P layer - no bootnode."); + if(_bootnodes == null || _bootnodes.length == 0) { + logger.info("P2P layer - no bootnode provided by minima."); + } + if(_staticPeers != null && _bootnodes != null && _staticPeers.length == _bootnodes.length) { + logger.info("P2P layer - received bootnode and staticpeer info from minima, adding to config"); + Collections.addAll(staticPeers, _staticPeers); + Collections.addAll(bootnodes, _bootnodes); } - // check config file for SECP256K1 private key File mRoot = ensureFolder(new File(mConfFolder)); @@ -79,6 +124,8 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee //Current used TxPOW File mP2PDir = ensureFolder(new File(mRoot,"p2p")); File mP2PNodePrivKeyFile = new File(mP2PDir, "NodePrivKey.pkey"); + mP2PBootnodesFile = new File(mP2PDir, "bootnodes.csv"); + // two lines below just to get rid of temporary not initialized issue PrivateKeyProvider provider = PrivateKeyGenerator::generate; PrivKey privKey = provider.get(); @@ -91,6 +138,9 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee byte[] keyBuffer; keyBuffer = inputStream.readAllBytes(); privKey = KeyKt.unmarshalPrivateKey(keyBuffer); + PubKey pubKey = privKey.publicKey(); + System.out.println("Loaded private key from local dir: " + hex(privKey.bytes())); + System.out.println("Computed public key: " + hex(pubKey.bytes())); } catch (FileNotFoundException e) { System.out.println("Failed to read private key from disk - " + e.getMessage()); e.printStackTrace(); @@ -107,6 +157,9 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee try { FileOutputStream outputStream = new FileOutputStream(mP2PNodePrivKeyFile); outputStream.write(KeyKt.marshalPrivateKey(privKey)); + System.out.println("Generated new private key and saved to local dir: " + hex(privKey.bytes())); + System.out.println("Computed public key: " + hex(pubKey.bytes()) ); + outputStream.close(); } catch (Exception e) { System.out.println("Failed to save private key to disk - " + e.getMessage()); } @@ -120,17 +173,82 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee System.out.println("P2P layer - generated node private key: " + privKey.toString()); //System.out.println("P2P layer - generated node private key bytes: " + String. privKey.raw(); // generate ENR nodeid - not to be confused with libp2pnodeid which is session specific! - nodeId = new LibP2PNodeId(PeerId.fromPubKey(privKey.publicKey())); - System.out.println("P2P layer - nodeid: " + nodeId.toString()); + PeerId pid = PeerId.fromPubKey(privKey.publicKey()); + byte[] marshaledPubKey = KeyKt.marshalPublicKey(privKey.publicKey()); + nodeId = new LibP2PNodeId(pid); + // if pubkey is less than 42 bytes then peerid will be + // the multihash digest of identity (0) and the pubkey as a protobuf (keytype, data) + System.out.println("P2P layer - peerid - raw hex: " + pid.toHex()); + System.out.println("P2P layer - pubkey - raw hex: " + hex(privKey.publicKey().bytes())); + System.out.println("P2P layer - protobuf pubkey: " + P2PStart.hex(marshaledPubKey)); + System.out.println("P2P layer - protobuf pubkey length (dec): " + marshaledPubKey.length); + System.out.println("P2P layer - protobuf pubkey length (hex)): " + Integer.toHexString(marshaledPubKey.length)); + ByteBuf wrappedBuf = wrappedBuffer(marshaledPubKey); + System.out.println("P2P layer - wrapped buff protobuf pubkey (hex)): " + hex(wrappedBuf.array())); // Integer.toHexString(marshaledPubKey.length)); + System.out.println("P2P layer - wrapped buff protobuf pubkey capacity (dec): " + wrappedBuf.capacity()); + + // this line below does not work. it should just add 0x0025 in front of the protobuf (identity+protobuf pubkey length). + Multihash mhash = Multihash.digest(new Multihash.Descriptor(Multihash.Digest.Identity, null), wrappedBuf, (marshaledPubKey.length)*8); + //System.out.println("P2P layer - multihash Str value: " + mhash.toString()); + // last 8 bytes are zeros for some reason + // as we are fixed size we only copy the meaningful bytes before Base58 encoding. + byte[] mhashbytes = mhash.getBytes().copy(0, 39).array(); + System.out.println("P2P layer - multihash hex bytes value: " + hex(mhashbytes)); + System.out.println("P2P layer - multihash base58 value: " + Base58.INSTANCE.encode(mhashbytes)); + + System.out.println("P2P layer - peerid - base58 encoded: " + pid.toHex()); + System.out.println("P2P layer - nodeid (peerid base58): " + nodeId.toString()); System.out.println("P2P layer - nodeid base58: " + nodeId.toBase58()); pubKey = privKey.publicKey(); + + // bootnodes + if(mP2PBootnodesFile.exists()) { + // try loading bootnodes from CSV file + List> records = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(mP2PBootnodesFile))) { + String line; + while ((line = br.readLine()) != null) { + if(line.length() > 0) { + String[] values = line.split(P2PStart.COMMA_DELIMITER); + logger.debug("P2PStart:csv load: read and CSV split one line: " + line); + if(values.length == 2) { + // expect p2p multiaddr and ENR + // TODO: add fields validation + records.add(Arrays.asList(values)); + System.out.println("Loaded bootnode from CSV: " + records.get(records.size()-1).toString()); + } else { + logger.warn("Failed to parse line from CSV: " + line); + } + } else { + logger.debug("P2PStart: skipping empty line in CSV"); + } + } + } catch(FileNotFoundException e) { + + } catch(IOException e) { + + } + + for( List record: records) { + staticPeers.add(record.get(0)); + bootnodes.add(record.get(1)); + } + } + DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); try { - if(staticPeers != null && staticPeers.length > 0 && bootnodes != null && bootnodes.length > 0) { - System.out.println("Building p2p layer using provided params: staticpeer=" + staticPeers[0] + " and bootnode=" + bootnodes[0]); - network = factory.builder().setPrivKey(privKey).staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); - } else if(staticPeers == null || staticPeers.length == 0) { + if(staticPeers != null && staticPeers.size() > 0 && bootnodes != null && bootnodes.size() > 0) { + System.out.println("Building p2p layer using provided params: staticpeer=" + staticPeers.get(0) + " and bootnode=" + bootnodes.get(0)); + //TODO: refactor below line to loop over staticPeers and bootnode data + DiscoveryNetworkBuilder builder = factory.builder(); + builder = builder.setPrivKey(privKey); + for(int i = 0; i < staticPeers.size(); i++) { + builder = builder.staticPeer(staticPeers.get(i)).bootnode(bootnodes.get(i)); + } + network = builder.buildAndStart(); +// network = factory.builder().setPrivKey(privKey).staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); + } else if(staticPeers == null || staticPeers.size() == 0) { logger.info("P2P: starting in standalone mode"); System.out.println("P2P: starting in standalone mode"); network = factory.builder().setPrivKey(privKey).buildAndStart(); @@ -151,12 +269,49 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] staticPee activeKnownNodes = new HashSet<>(); // TODO: send to yourself a message 5 seconds in the future PostMessage(P2P_START_SCAN); // could also be a TimerMessage + PostMessage(P2P_SAVE_NEIGHBOURS); + // PostTimerMessage(new TimerMessage(SAVE_NEIGHBOURS_DELAY, P2P_SAVE_NEIGHBOURS)); } else { // initialization failed - what do we do? logger.error("Failed to start P2P network."); } } + + public static String hex(byte[] bytes) { + StringBuilder result = new StringBuilder(); + for (byte aByte : bytes) { + result.append(String.format("%02x", aByte)); + // upper case + // result.append(String.format("%02X", aByte)); + } + return result.toString(); + } + + public String convertToCSV(MinimaNodeInfo i) { + StringBuilder builder = new StringBuilder(); + builder.append(i.discoMultiAddrTCP); + builder.append(","); + builder.append(i.nodeRecord); + return builder.toString(); + } + + // from https://www.baeldung.com/java-csv + public String convertToCSV(String[] data) { + return Stream.of(data) + .map(this::escapeSpecialCharacters) + .collect(Collectors.joining(",")); + } + + public String escapeSpecialCharacters(String data) { + String escapedData = data.replaceAll("\\R", " "); + if (data.contains(",") || data.contains("\"") || data.contains("'")) { + data = data.replace("\"", "\"\""); + escapedData = "\"" + data + "\""; + } + return escapedData; + } + public NodeId getNodeId() { return nodeId; } @@ -192,8 +347,52 @@ protected void processMessage(Message zMessage) throws Exception { logger.warn("P2PStart received message: " + zMessage.toString()); if(zMessage.isMessageType(P2P_START_SCAN)) { - - if(network != null) { + p2pAddNewNodes(); + // p2pSaveNeighbours(); // for now save neighbours list after each scan + } else if(zMessage.isMessageType(P2P_SAVE_NEIGHBOURS)) { + // p2pSaveNeighbours(); + } + } + + private void p2pSaveNeighbours() { + logger.debug("p2pSaveNeighbours: trying to save neighbours list"); + System.out.println("p2pSaveNeighbours: trying to save neighbours list"); + if(allDiscoveredNodes.size() == 0) { + logger.debug("p2pSaveNeighbours: empty nodes list, nothing to save, exiting."); + return; + } + try { + //File csvOutputFile = new File(CSV_FILE_NAME); + File csvOutputFile = mP2PBootnodesFile; + FileWriter csvWriter = new FileWriter(csvOutputFile); + try (PrintWriter pw = new PrintWriter(csvWriter, true)) { + for(MinimaNodeInfo i: allDiscoveredNodes) { + logger.debug("saving node i: " + i.nodeRecord); + pw.println(convertToCSV(i)); + } +// allDiscoveredNodes.stream() +// // dataLines.stream() +// .map(this::convertToCSV) +// .forEach(pw::println); + } + csvWriter.flush(); + csvWriter.close(); + //csvOutputFile. + //assertTrue(csvOutputFile.exists()); + } catch(IOException e) { + logger.warn("Could not save neighbours list to file! msg=" + e.getMessage()); + } + logger.debug("p2pSaveNeighbours: end"); + // try { + // Thread.sleep(SAVE_NEIGHBOURS_DELAY); + // PostMessage(P2P_SAVE_NEIGHBOURS); + // } catch(Exception e) { + + // } + } + + private void p2pAddNewNodes() { + if(network != null) { Set activeKnownNodes = new HashSet<>(); while (true) { @@ -214,22 +413,31 @@ protected void processMessage(Message zMessage) throws Exception { Set newActiveNodes = new HashSet<>(); Set unconnectedNewNodes = new HashSet<>(); - //logger.debug("trying to stream discovery peers"); network.streamKnownDiscoveryPeers() .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr PeerId peerid = new PeerId(discoPeer.getNodeID().toArray()); + // nodeAddress: enr_ip:enr_port + // pubkey:enr_secp256k1 + // nodeid: derived(enr_secp256k1) logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey() + " peerid: " + peerid + " nodeid:" + discoPeer.getNodeID().toHexString() + " enr: " + discoPeer.getNodeRecord()); - //TODO: establish link between Bytes nodeID and libp2p nodeid / peerid - //TODO: verify values for nodeid and enr and filter existing nodes vs new based on nodeid + //TODO: establish link between Bytes nodeID and libp2p nodeid / peerid + //TODO: verify values for nodeid and enr and filter existing nodes vs new based on nodeid + //Optional tmpdiscopeer = NodeRecordConverter.convertToDiscoveryPeer(discoPeer.getNodeRecord()); + + //nodeRecord.getNodeId() newActiveNodes.add(discoPeer.getNodeAddress()); if(!knownNodeIDs.contains(discoPeer.getNodeID().toString())) { logger.debug("FOUND NEW NODE: nodeid:" + discoPeer.getNodeID().toString() + " " + discoPeer.getNodeRecord().toString()); - unconnectedNewNodes.add(new MinimaNodeInfo(discoPeer.getNodeID().toHexString(), - discoPeer.getNodeRecord().toString(), - discoPeer.getNodeAddress())); + MinimaNodeInfo aNewNodeInfo = new MinimaNodeInfo(discoPeer.getNodeID().toHexString(), + discoPeer.getNodeRecord().toString(), + discoPeer.getNodeAddress(), + getDiscoMultiAddrTCPFromENR(discoPeer.getNodeRecord(), discoPeer.getPublicKey().toArray()) + ); + unconnectedNewNodes.add(aNewNodeInfo); + allDiscoveredNodes.add(aNewNodeInfo); } else { logger.debug("SKIPPING an already connected node: nodeid:" + discoPeer.getNodeID().toHexString() + " " + discoPeer.getNodeRecord().toString()); } @@ -245,7 +453,7 @@ protected void processMessage(Message zMessage) throws Exception { if (i.nodeRecord.compareTo(network.getENR())==0) { logger.warn("IGNORING node ENR in list of new peers."); } else if(i.nodeRecord == null || i.nodeID == null) { - logger.warn("IGNORING empty ndoeRecord or nodeID."); + logger.warn("IGNORING empty nodeRecord or nodeID."); } else { logger.info("CONNECTING to new ENR " + i.nodeRecord); System.out.println("Starting MinimaClient: " + i.socket.toString().substring(1) + ":9001"); @@ -258,6 +466,8 @@ protected void processMessage(Message zMessage) throws Exception { // update known nodes activeKnownNodes = newActiveNodes; + // save list + p2pSaveNeighbours(); try { Thread.sleep(5000); PostMessage(P2P_START_SCAN); @@ -265,10 +475,37 @@ protected void processMessage(Message zMessage) throws Exception { } } - } } + } - + public String getDiscoMultiAddrTCPFromENR(String ENR, byte[] marshalledPubKey) { + NodeRecord nodeRecord = NodeRecordFactory.DEFAULT.fromEnr(ENR); + // extract multiaddress from nodeRecord + nodeRecord.getTcpAddress(); + //nodeRecord. + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append("/ip4/"); + strBuilder.append(nodeRecord.getTcpAddress().get().getAddress().getHostAddress()); + strBuilder.append("/tcp/"); + strBuilder.append(nodeRecord.getTcpAddress().get().getPort()); + strBuilder.append("/p2p/"); + // construct nodeid from public key + //discoPeer.getPublicKey() +// new PubKey(KeyType.Secp256k1); + //PubKey hostPubKey = PubKey.fromString(discoPeer.getPublicKey()); + //discoPeer.getPublicKey(); +// PubKey pubKey = Secp256k1Kt.unmarshalSecp256k1PublicKey(08021221 + discoPeer.getPublicKey().toArray()); + + PubKey pubKey = Secp256k1Kt.unmarshalSecp256k1PublicKey(marshalledPubKey); + PeerId pid = PeerId.fromPubKey(pubKey); + nodeId = new LibP2PNodeId(pid); + logger.debug("PubKey: " + hex(pubKey.bytes())); + logger.debug("pid: " + pid.toHex()); + logger.debug("Built nodeid of discovered peer: " + nodeId.toBase58()); + strBuilder.append(nodeId.toBase58()); + logger.debug("Discovery TCP address: " + strBuilder.toString()); + logger.debug("discover peer (enr fields): " + nodeRecord.toString()); + return strBuilder.toString(); } //TODO: refactor below code to use above object and constructor instead - if possible From 663e926c9291c0c0a320f286ae493da5f563216f Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Tue, 29 Jun 2021 16:39:36 +0100 Subject: [PATCH 37/55] p2p: always use the same port for P2P discovery (basePort + 5) and persists all nodes data in emulator. --- endtoend/api/package/index.js | 284 ++++++++++++++++++ endtoend/api/package/package.json | 15 + endtoend/src/staticTests.js | 11 +- .../network/base/DiscoveryNetworkFactory.java | 12 +- .../minima/system/network/base/P2PStart.java | 44 +-- 5 files changed, 341 insertions(+), 25 deletions(-) create mode 100644 endtoend/api/package/index.js create mode 100644 endtoend/api/package/package.json diff --git a/endtoend/api/package/index.js b/endtoend/api/package/index.js new file mode 100644 index 000000000..d535ba83f --- /dev/null +++ b/endtoend/api/package/index.js @@ -0,0 +1,284 @@ +const axios = require("axios"); + +const cfg = { + HTTP_TIMEOUT: 30000 +} + +var Minima_API = { + + rpchost : "http://127.0.0.1:9002", + + init: function(host) { + Minima_API.rpchost = `http://${host}:9002` + }, + + help: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/help", params) + }, + + tutorial: function() { + return get_minima_endpoint(Minima_API.rpchost, "/tutorial", "") + }, + + status: function() { + return get_minima_endpoint(Minima_API.rpchost, "/status", "") + }, + + topblock: function() { + return get_minima_endpoint(Minima_API.rpchost, "/topblock", "") + }, + + history: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/history", params) + }, + + backup: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/backup", params) + }, + + flushmempool: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/flushmempool", params) + }, + + check: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/check", params) + }, + + printdb: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/printdb", params) + }, + + printtree: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/printtree", params) + }, + + automine: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/automine", params) + }, + + trace: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/trace", params) + }, + + network: function() { + return get_minima_endpoint(Minima_API.rpchost, "/network", "") + }, + + connect: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/connect", params) + }, + + disconnect: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/disconnect", params) + }, + + reconnect: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/reconnect", params) + }, + + weblink: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/weblink", params) + }, + + gimme50: function() { + return get_minima_endpoint(Minima_API.rpchost, "/gimme50", "") + }, + + send: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/send", params) + }, + + sendpoll: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/sendpoll", params) + }, + + newaddress: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/newaddress", params) + }, + + balance: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/balance", params) + }, + + keys: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/keys", params) + }, + + exportkey: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/exportkey", params) + }, + + importkey: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/importkey", params) + }, + + coins: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/coins", params) + }, + + coinsimple: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/coinsimple", params) + }, + + keepcoin: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/keepcoin", params) + }, + + txpowsearch: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txpowsearch", params) + }, + + txpowinfo: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txpowinfo", params) + }, + + scripts: function() { + return get_minima_endpoint(Minima_API.rpchost, "/scripts", "") + }, + + newscript: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/newscript", params) + }, + + extrascript: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/extrascript", params) + }, + + cleanscript: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/cleanscript", params) + }, + + runscript: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/runscript", params) + }, + + tokens: function() { + return get_minima_endpoint(Minima_API.rpchost, "/tokens", "") + }, + + tokencreate: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/tokencreate", params) + }, + + tokenvalidate: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/tokenvalidate", params) + }, + + sign: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/sign", params) + }, + + verify: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/verify", params) + }, + + chainsha: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/chainsha", params) + }, + + random: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/random", params) + }, + + hash: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/hash", params) + }, + + maxima: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/maxima", params) + }, + + sshtunnel: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/sshtunnel", params) + }, + + minidapps: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/minidapps", params) + }, + + txnlist: function (params="") { + return get_minima_endpoint(Minima_API.rpchost, "/txnlist", params) + }, + + txncreate: function(params="") { + return get_minima_endpoint(Minima_API.rpchost, "/txncreate", params) + }, + + txndelete: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txndelete", params) + }, + + txnexport: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnexport", params) + }, + + txnimport: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnimport", params) + }, + + txninput: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txninput", params) + }, + + txnoutput: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnoutput", params) + }, + + txnreminput: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnreminput", params) + }, + + txnremoutput: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnremoutput", params) + }, + + txnstate: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnstate", params) + }, + + txnscript: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnscript", params) + }, + + txnsign: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnsign", params) + }, + + txnauto: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnauto", params) + }, + + txnsignauto: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnsignauto", params) + }, + + txnvalidate: function(params) { + return get_minima_endpoint(Minima_API.rpchost, "/txnvalidate", params) + }, + + quit: function() { + get_minima_endpoint(Minima_API.rpchost, "/quit", "") + } +} + +const get_minima_endpoint = async (host, endpoint, params="") => { + const url = host + endpoint + "+" + params.replace(/;/g, "+"); + try{ + const response = await axios.get(url, {timeout: cfg.HTTP_TIMEOUT}, {maxContentLength: 3000}, {responseType: 'plain'}) + + // handle success + if(response && response.status == 200) { + console.log(params, response.data); + if(response.data.status == true) { + return response.data.response; + } + } + } catch { error => { + // handle error + console.log(error); + } + } +} + +exports.Minima_API = Minima_API + diff --git a/endtoend/api/package/package.json b/endtoend/api/package/package.json new file mode 100644 index 000000000..82cc98a8e --- /dev/null +++ b/endtoend/api/package/package.json @@ -0,0 +1,15 @@ +{ + "name": "minima-api", + "version": "1.0.0", + "description": "node package for minima APIs", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.21.1" + } +} diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 9ad92e628..fb92f41c7 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -13,6 +13,10 @@ require('chai') require('chai').assert; // *** config *** +DOCKER_P2P_PATH = '/root/.minima/p2p' +DOCKER_HOST_CONFIG_ROOT = '/Users/jeromerousselot/src/minima/Minima/data-e2e' +DOCKER_HOST_CONFIG_DIR = '/p2p' + const cfg = { image: 'minima:latest', // docker image name to run -> can be customised docker_net: "minima-e2e-testnet", // docker private network name -> MUST BE CREATED MANUALLY @@ -23,12 +27,14 @@ const cfg = { hostConfig1: { AutoRemove: true, // comment this out to inspect stopped containers NetworkMode: "minima-e2e-testnet", - 'Binds': ['/Users/jeromerousselot/src/minima/Minima/node1/p2p:/root/.minima/p2p'], + 'Binds': [], + //'Binds': [ DOCKER_HOST_CONFIG_ROOT + '/node1' + DOCKER_HOST_CONFIG_DIR + ':' + DOCKER_P2P_PATH], CpuShares: 10, // node 1 in private mode uses auto-mining and aim for 100% CPU usage, so we throttle it }, hostConfig: { AutoRemove: true, // comment this out to inspect stopped containers NetworkMode: "minima-e2e-testnet", + 'Binds': [], CpuShares: 10, }, // unused - can be applied on a node to expose its RPC port on localhost - not needed for our tests @@ -62,6 +68,8 @@ function sleep(ms) { const start_docker_node_1 = async function (topology, nbNodes, tests_collection) { console.log("Creating container 1"); + // set node 1 config path + cfg.hostConfig1.Binds = [ DOCKER_HOST_CONFIG_ROOT + '/node' + '1' + DOCKER_HOST_CONFIG_DIR + ':' + DOCKER_P2P_PATH]; // Create the container. containers["1"] = await createMinimaContainer(cfg.node1_args, cfg.node_prefix + "1", cfg.hostConfig1); // Start the container. @@ -110,6 +118,7 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { for (let pos = 2; pos < nbNodes+1; pos++) { var node_args = get_node_args(cfg.TOPO_STAR, pos); console.log("topo star node " + pos + " args: " + node_args); + cfg.hostConfig.Binds = [ DOCKER_HOST_CONFIG_ROOT + '/node' + pos + DOCKER_HOST_CONFIG_DIR + ':' + DOCKER_P2P_PATH]; containers[pos] = await createMinimaContainer(node_args, cfg.node_prefix + pos, cfg.hostConfig); // Start the container. diff --git a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java index c24063c2e..c63c773f3 100644 --- a/src/org/minima/system/network/base/DiscoveryNetworkFactory.java +++ b/src/org/minima/system/network/base/DiscoveryNetworkFactory.java @@ -97,12 +97,16 @@ public DiscoveryNetworkBuilder setPrivKey(final PrivKey privKey) { this.privKey = privKey; return this; } - public DiscoveryNetwork buildAndStart() throws Exception { + public DiscoveryNetwork buildAndStart(int _port) throws Exception { int attempt = 1; while (true) { - - final Random random = new Random(); - final int port = MIN_PORT + random.nextInt(MAX_PORT - MIN_PORT); + final int port; + if(_port == 0) { + final Random random = new Random(); + port = MIN_PORT + random.nextInt(MAX_PORT - MIN_PORT); + } else { + port = _port; + } final DiscoveryConfig discoveryConfig = DiscoveryConfig.builder().staticPeers(staticPeers).bootnodes(bootnodes).build(); final NetworkConfig config = diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index 8b07386b2..c36584d74 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -15,6 +15,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Hashtable; import java.util.List; import java.util.Optional; import java.util.Set; @@ -71,10 +72,12 @@ public class P2PStart extends MessageProcessor { private DiscoveryNetwork network; Set activeKnownNodes; Set allDiscoveredNodes = new HashSet<>(); + Hashtable allDiscoveredNodes2 = new Hashtable<>(); private NodeId nodeId; private PubKey pubKey; private File mP2PBootnodesFile; - + private File mRoot; + class MinimaNodeInfo { final public InetSocketAddress socket; final public String nodeRecord; @@ -118,7 +121,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe } // check config file for SECP256K1 private key - File mRoot = ensureFolder(new File(mConfFolder)); + mRoot = ensureFolder(new File(mConfFolder)); String mRootPath = mRoot.getAbsolutePath(); //Current used TxPOW @@ -208,6 +211,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe List> records = new ArrayList<>(); try (BufferedReader br = new BufferedReader(new FileReader(mP2PBootnodesFile))) { String line; + logger.debug("starting to read p2p bootnodes CSV file: " + mP2PBootnodesFile.getAbsolutePath()); while ((line = br.readLine()) != null) { if(line.length() > 0) { String[] values = line.split(P2PStart.COMMA_DELIMITER); @@ -246,12 +250,12 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe for(int i = 0; i < staticPeers.size(); i++) { builder = builder.staticPeer(staticPeers.get(i)).bootnode(bootnodes.get(i)); } - network = builder.buildAndStart(); + network = builder.buildAndStart(mNetwork.getBasePort() + 5); // network = factory.builder().setPrivKey(privKey).staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); } else if(staticPeers == null || staticPeers.size() == 0) { logger.info("P2P: starting in standalone mode"); System.out.println("P2P: starting in standalone mode"); - network = factory.builder().setPrivKey(privKey).buildAndStart(); + network = factory.builder().setPrivKey(privKey).buildAndStart(mNetwork.getBasePort() + 5); } } catch (Exception e) { logger.error("P2P failed to start through DiscoveyrNetworkFactory."); @@ -357,30 +361,30 @@ protected void processMessage(Message zMessage) throws Exception { private void p2pSaveNeighbours() { logger.debug("p2pSaveNeighbours: trying to save neighbours list"); System.out.println("p2pSaveNeighbours: trying to save neighbours list"); - if(allDiscoveredNodes.size() == 0) { + if(allDiscoveredNodes2.size() == 0) { logger.debug("p2pSaveNeighbours: empty nodes list, nothing to save, exiting."); return; } try { + File mP2PDir = ensureFolder(new File(mRoot,"p2p")); //File csvOutputFile = new File(CSV_FILE_NAME); File csvOutputFile = mP2PBootnodesFile; FileWriter csvWriter = new FileWriter(csvOutputFile); try (PrintWriter pw = new PrintWriter(csvWriter, true)) { - for(MinimaNodeInfo i: allDiscoveredNodes) { - logger.debug("saving node i: " + i.nodeRecord); + for(MinimaNodeInfo i: allDiscoveredNodes2.values()) { + logger.debug("saving node i: " + i.nodeRecord + "line=" + convertToCSV(i)); pw.println(convertToCSV(i)); } -// allDiscoveredNodes.stream() -// // dataLines.stream() -// .map(this::convertToCSV) -// .forEach(pw::println); + } catch(Exception e) { + logger.warn("Could not write lines to neighbours list file! " + "msg=" + e.getMessage()); + e.printStackTrace(); +// logger.warn("Stack trace: " + e.getStackTrace().toString()); } csvWriter.flush(); - csvWriter.close(); - //csvOutputFile. - //assertTrue(csvOutputFile.exists()); } catch(IOException e) { - logger.warn("Could not save neighbours list to file! msg=" + e.getMessage()); + logger.warn("Could not flush and close neighbours list file! " + "msg=" + e.getMessage()); + e.printStackTrace(); +// logger.warn("Stack trace: " + e.getStackTrace().toString()); } logger.debug("p2pSaveNeighbours: end"); // try { @@ -437,7 +441,7 @@ private void p2pAddNewNodes() { getDiscoMultiAddrTCPFromENR(discoPeer.getNodeRecord(), discoPeer.getPublicKey().toArray()) ); unconnectedNewNodes.add(aNewNodeInfo); - allDiscoveredNodes.add(aNewNodeInfo); + allDiscoveredNodes2.put(aNewNodeInfo.nodeID, aNewNodeInfo); } else { logger.debug("SKIPPING an already connected node: nodeid:" + discoPeer.getNodeID().toHexString() + " " + discoPeer.getNodeRecord().toString()); } @@ -525,7 +529,7 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc node1_addr_fields = args[0].split("/"); node1_id = node1_addr_fields[node1_addr_fields.length - 1]; String bootnode_1 = args[1]; - network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(); + network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(0); LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR @@ -538,10 +542,10 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc System.out.println("Found node1_id: " + node1_id); // Multiaddr address = Multiaddr.fromString(args[0]); - network = factory.builder().staticPeer(args[0]).buildAndStart(); + network = factory.builder().staticPeer(args[0]).buildAndStart(0); } else { // server mode only - no peer to connect to - network = factory.builder().buildAndStart(); + network = factory.builder().buildAndStart(0); //id = network.getNodeId(); } @@ -579,7 +583,7 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc } } } catch (Exception e) { - // TODO Auto-generated catch block + // TODO Auto-generated catch blocks e.printStackTrace(); } From fe2d600f6de00c1e3678ff9b76aa15d73851bdce Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 30 Jun 2021 15:58:16 +0100 Subject: [PATCH 38/55] p2p: cleaned up code and added host pub key to neighbours CSV list. --- .../minima/system/network/base/P2PStart.java | 245 ++++++++++-------- 1 file changed, 131 insertions(+), 114 deletions(-) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index c36584d74..e53a0a196 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.Vector; import java.util.concurrent.ExecutionException; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -132,111 +133,50 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe // two lines below just to get rid of temporary not initialized issue PrivateKeyProvider provider = PrivateKeyGenerator::generate; PrivKey privKey = provider.get(); - if(mP2PNodePrivKeyFile.exists()) { - // try loading file from private key - // if error, bailout? - FileInputStream inputStream; - try { - inputStream = new FileInputStream(mP2PNodePrivKeyFile); - byte[] keyBuffer; - keyBuffer = inputStream.readAllBytes(); - privKey = KeyKt.unmarshalPrivateKey(keyBuffer); - PubKey pubKey = privKey.publicKey(); - System.out.println("Loaded private key from local dir: " + hex(privKey.bytes())); - System.out.println("Computed public key: " + hex(pubKey.bytes())); - } catch (FileNotFoundException e) { - System.out.println("Failed to read private key from disk - " + e.getMessage()); - e.printStackTrace(); - } catch (IOException e) { - System.out.println("Failed to read private key from disk - " + e.getMessage()); - e.printStackTrace(); - } - } else { - // generate a SECP256K1 private key - //PrivateKeyProvider keyProvider = PrivateKeyGenerator::generate; - privKey = KeyKt.generateKeyPair(KEY_TYPE.SECP256K1).component1(); - // privKey = keyProvider.get(); - // save priv key to file - try { - FileOutputStream outputStream = new FileOutputStream(mP2PNodePrivKeyFile); - outputStream.write(KeyKt.marshalPrivateKey(privKey)); - System.out.println("Generated new private key and saved to local dir: " + hex(privKey.bytes())); - System.out.println("Computed public key: " + hex(pubKey.bytes()) ); - outputStream.close(); - } catch (Exception e) { - System.out.println("Failed to save private key to disk - " + e.getMessage()); - } + privKey = loadNodePrivateKey(mP2PNodePrivKeyFile); + if(privKey == null) { + privKey = generateNodePrivateKey(mP2PNodePrivKeyFile); } + if(privKey == null) { System.out.println("P2P Error - priv key uninitialized!"); return; } - // privKey.toString(); - logger.warn("P2P layer - generated node private key: " + privKey.toString()); - System.out.println("P2P layer - generated node private key: " + privKey.toString()); - //System.out.println("P2P layer - generated node private key bytes: " + String. privKey.raw(); - // generate ENR nodeid - not to be confused with libp2pnodeid which is session specific! + PeerId pid = PeerId.fromPubKey(privKey.publicKey()); - byte[] marshaledPubKey = KeyKt.marshalPublicKey(privKey.publicKey()); nodeId = new LibP2PNodeId(pid); + pubKey = privKey.publicKey(); + + //byte[] marshaledPubKey = KeyKt.marshalPublicKey(privKey.publicKey()); // if pubkey is less than 42 bytes then peerid will be // the multihash digest of identity (0) and the pubkey as a protobuf (keytype, data) - System.out.println("P2P layer - peerid - raw hex: " + pid.toHex()); - System.out.println("P2P layer - pubkey - raw hex: " + hex(privKey.publicKey().bytes())); - System.out.println("P2P layer - protobuf pubkey: " + P2PStart.hex(marshaledPubKey)); - System.out.println("P2P layer - protobuf pubkey length (dec): " + marshaledPubKey.length); - System.out.println("P2P layer - protobuf pubkey length (hex)): " + Integer.toHexString(marshaledPubKey.length)); - ByteBuf wrappedBuf = wrappedBuffer(marshaledPubKey); - System.out.println("P2P layer - wrapped buff protobuf pubkey (hex)): " + hex(wrappedBuf.array())); // Integer.toHexString(marshaledPubKey.length)); - System.out.println("P2P layer - wrapped buff protobuf pubkey capacity (dec): " + wrappedBuf.capacity()); + // logger.debug("P2P layer - peerid - raw hex: " + pid.toHex()); + // logger.debug("P2P layer - pubkey - raw hex: " + hex(privKey.publicKey().bytes())); + // logger.debug("P2P layer - protobuf pubkey: " + P2PStart.hex(marshaledPubKey)); + // logger.debug("P2P layer - protobuf pubkey length (dec): " + marshaledPubKey.length); + // logger.debug("P2P layer - protobuf pubkey length (hex)): " + Integer.toHexString(marshaledPubKey.length)); + //ByteBuf wrappedBuf = wrappedBuffer(marshaledPubKey); + // logger.debug("P2P layer - wrapped buff protobuf pubkey (hex)): " + hex(wrappedBuf.array())); // Integer.toHexString(marshaledPubKey.length)); + // logger.debug("P2P layer - wrapped buff protobuf pubkey capacity (dec): " + wrappedBuf.capacity()); // this line below does not work. it should just add 0x0025 in front of the protobuf (identity+protobuf pubkey length). - Multihash mhash = Multihash.digest(new Multihash.Descriptor(Multihash.Digest.Identity, null), wrappedBuf, (marshaledPubKey.length)*8); + //Multihash mhash = Multihash.digest(new Multihash.Descriptor(Multihash.Digest.Identity, null), wrappedBuf, (marshaledPubKey.length)*8); //System.out.println("P2P layer - multihash Str value: " + mhash.toString()); // last 8 bytes are zeros for some reason // as we are fixed size we only copy the meaningful bytes before Base58 encoding. - byte[] mhashbytes = mhash.getBytes().copy(0, 39).array(); - System.out.println("P2P layer - multihash hex bytes value: " + hex(mhashbytes)); - System.out.println("P2P layer - multihash base58 value: " + Base58.INSTANCE.encode(mhashbytes)); - - System.out.println("P2P layer - peerid - base58 encoded: " + pid.toHex()); - System.out.println("P2P layer - nodeid (peerid base58): " + nodeId.toString()); - System.out.println("P2P layer - nodeid base58: " + nodeId.toBase58()); - pubKey = privKey.publicKey(); - - - // bootnodes - if(mP2PBootnodesFile.exists()) { - // try loading bootnodes from CSV file - List> records = new ArrayList<>(); - try (BufferedReader br = new BufferedReader(new FileReader(mP2PBootnodesFile))) { - String line; - logger.debug("starting to read p2p bootnodes CSV file: " + mP2PBootnodesFile.getAbsolutePath()); - while ((line = br.readLine()) != null) { - if(line.length() > 0) { - String[] values = line.split(P2PStart.COMMA_DELIMITER); - logger.debug("P2PStart:csv load: read and CSV split one line: " + line); - if(values.length == 2) { - // expect p2p multiaddr and ENR - // TODO: add fields validation - records.add(Arrays.asList(values)); - System.out.println("Loaded bootnode from CSV: " + records.get(records.size()-1).toString()); - } else { - logger.warn("Failed to parse line from CSV: " + line); - } - } else { - logger.debug("P2PStart: skipping empty line in CSV"); - } - } - } catch(FileNotFoundException e) { - - } catch(IOException e) { - - } - - for( List record: records) { - staticPeers.add(record.get(0)); - bootnodes.add(record.get(1)); + //byte[] mhashbytes = mhash.getBytes().copy(0, 39).array(); + //logger.debug("P2P layer - multihash hex bytes value: " + hex(mhashbytes)); + //logger.debug("P2P layer - multihash base58 value: " + Base58.INSTANCE.encode(mhashbytes)); // same as nodeId.toBase58() + //logger.debug("P2P layer - peerid - base58 encoded: " + pid.toHex()); + //logger.debug("P2P layer - nodeid (peerid base58): " + nodeId.toString()); // same as .toBase58() + logger.debug("P2P layer - nodeid base58: " + nodeId.toBase58()); + + // bootnodes CSV + List> records = loadP2PNeighbours(mP2PBootnodesFile); + if (records != null) { + for (List record : records) { + staticPeers.add(record.get(1)); + bootnodes.add(record.get(2)); } } @@ -244,14 +184,12 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe try { if(staticPeers != null && staticPeers.size() > 0 && bootnodes != null && bootnodes.size() > 0) { System.out.println("Building p2p layer using provided params: staticpeer=" + staticPeers.get(0) + " and bootnode=" + bootnodes.get(0)); - //TODO: refactor below line to loop over staticPeers and bootnode data DiscoveryNetworkBuilder builder = factory.builder(); builder = builder.setPrivKey(privKey); for(int i = 0; i < staticPeers.size(); i++) { builder = builder.staticPeer(staticPeers.get(i)).bootnode(bootnodes.get(i)); } - network = builder.buildAndStart(mNetwork.getBasePort() + 5); -// network = factory.builder().setPrivKey(privKey).staticPeer(staticPeers[0]).bootnode(bootnodes[0]).buildAndStart(); + network = builder.buildAndStart(mNetwork.getBasePort() + 5); } else if(staticPeers == null || staticPeers.size() == 0) { logger.info("P2P: starting in standalone mode"); System.out.println("P2P: starting in standalone mode"); @@ -273,7 +211,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe activeKnownNodes = new HashSet<>(); // TODO: send to yourself a message 5 seconds in the future PostMessage(P2P_START_SCAN); // could also be a TimerMessage - PostMessage(P2P_SAVE_NEIGHBOURS); + PostMessage(P2P_SAVE_NEIGHBOURS); // for now we save neighbours at the end of each scan, this timer is not used // PostTimerMessage(new TimerMessage(SAVE_NEIGHBOURS_DELAY, P2P_SAVE_NEIGHBOURS)); } else { // initialization failed - what do we do? @@ -281,7 +219,6 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe } } - public static String hex(byte[] bytes) { StringBuilder result = new StringBuilder(); for (byte aByte : bytes) { @@ -292,8 +229,10 @@ public static String hex(byte[] bytes) { return result.toString(); } - public String convertToCSV(MinimaNodeInfo i) { + public String convertToCSV(String pubKey, MinimaNodeInfo i) { StringBuilder builder = new StringBuilder(); + builder.append(pubKey); + builder.append(","); builder.append(i.discoMultiAddrTCP); builder.append(","); builder.append(i.nodeRecord); @@ -352,46 +291,125 @@ protected void processMessage(Message zMessage) throws Exception { if(zMessage.isMessageType(P2P_START_SCAN)) { p2pAddNewNodes(); - // p2pSaveNeighbours(); // for now save neighbours list after each scan + // saveP2PNeighbours(); // for now save neighbours list after each scan } else if(zMessage.isMessageType(P2P_SAVE_NEIGHBOURS)) { - // p2pSaveNeighbours(); + // saveP2PNeighbours(); } } - private void p2pSaveNeighbours() { - logger.debug("p2pSaveNeighbours: trying to save neighbours list"); - System.out.println("p2pSaveNeighbours: trying to save neighbours list"); + private PrivKey loadNodePrivateKey(File mP2PNodePrivKeyFile) { + if (!mP2PNodePrivKeyFile.exists()) { + return null; + } + PrivKey privKey = null; + // try loading file from private key + // if error, bailout? + FileInputStream inputStream; + try { + inputStream = new FileInputStream(mP2PNodePrivKeyFile); + byte[] keyBuffer; + keyBuffer = inputStream.readAllBytes(); + privKey = KeyKt.unmarshalPrivateKey(keyBuffer); + PubKey pubKey = privKey.publicKey(); + System.out.println("Loaded private key from local dir: " + hex(privKey.bytes())); + System.out.println("Computed public key: " + hex(pubKey.bytes())); + } catch (FileNotFoundException e) { + System.out.println("Failed to read private key from disk - " + e.getMessage()); + e.printStackTrace(); + } catch (IOException e) { + System.out.println("Failed to read private key from disk - " + e.getMessage()); + e.printStackTrace(); + } + return privKey; + } + + private PrivKey generateNodePrivateKey(File mP2PNodePrivKeyFile) { + // generate a SECP256K1 private key + // PrivateKeyProvider keyProvider = PrivateKeyGenerator::generate; + PrivKey privKey = KeyKt.generateKeyPair(KEY_TYPE.SECP256K1).component1(); + // privKey = keyProvider.get(); + // save priv key to file + try { + FileOutputStream outputStream = new FileOutputStream(mP2PNodePrivKeyFile); + outputStream.write(KeyKt.marshalPrivateKey(privKey)); + System.out.println("Generated new private key and saved to local dir: " + hex(privKey.bytes())); + System.out.println("Computed public key: " + hex(pubKey.bytes())); + outputStream.close(); + } catch (Exception e) { + System.out.println("Failed to save private key to disk - " + e.getMessage()); + } + return privKey; + } + + private List> loadP2PNeighbours(File mP2PBootnodesFile) { + if(!mP2PBootnodesFile.exists()) { + return null; + } + + // try loading bootnodes from CSV file + List> records = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(mP2PBootnodesFile))) { + String line; + logger.debug("starting to read p2p bootnodes CSV file: " + mP2PBootnodesFile.getAbsolutePath()); + while ((line = br.readLine()) != null) { + if (line.length() > 0) { + String[] values = line.split(P2PStart.COMMA_DELIMITER); + logger.debug("P2PStart:csv load: read and CSV split one line: " + line); + if (values.length == 2) { + // expect public key, p2p multiaddr and ENR + // TODO: add fields validation + records.add(Arrays.asList(values)); + System.out.println("Loaded bootnode from CSV: " + records.get(records.size() - 1).toString()); + } else { + logger.warn("Failed to parse line from CSV: " + line); + } + } else { + logger.debug("P2PStart: skipping empty line in CSV"); + } + } + } catch (FileNotFoundException e) { + + } catch (IOException e) { + + } + return records; + } + + private void saveP2PNeighbours() { + logger.debug("saveP2PNeighbours: trying to save neighbours list"); + System.out.println("saveP2PNeighbours: trying to save neighbours list"); if(allDiscoveredNodes2.size() == 0) { - logger.debug("p2pSaveNeighbours: empty nodes list, nothing to save, exiting."); + logger.debug("savep2pNeighbours: empty nodes list, nothing to save, exiting."); return; } try { - File mP2PDir = ensureFolder(new File(mRoot,"p2p")); //File csvOutputFile = new File(CSV_FILE_NAME); File csvOutputFile = mP2PBootnodesFile; FileWriter csvWriter = new FileWriter(csvOutputFile); - try (PrintWriter pw = new PrintWriter(csvWriter, true)) { - for(MinimaNodeInfo i: allDiscoveredNodes2.values()) { - logger.debug("saving node i: " + i.nodeRecord + "line=" + convertToCSV(i)); - pw.println(convertToCSV(i)); - } + try (PrintWriter pw = new PrintWriter(csvWriter, true)) { + allDiscoveredNodes2.forEach(new BiConsumer() { + @Override + public void accept(String pubKey, MinimaNodeInfo i) { + logger.debug("saving node i: pubkey=" + pubKey + " enr=" + i.nodeRecord + " line=" + convertToCSV(pubKey, i)); + pw.println(convertToCSV(pubKey, i)); + } + }); } catch(Exception e) { logger.warn("Could not write lines to neighbours list file! " + "msg=" + e.getMessage()); e.printStackTrace(); // logger.warn("Stack trace: " + e.getStackTrace().toString()); } - csvWriter.flush(); + //csvWriter.flush(); } catch(IOException e) { logger.warn("Could not flush and close neighbours list file! " + "msg=" + e.getMessage()); e.printStackTrace(); // logger.warn("Stack trace: " + e.getStackTrace().toString()); } - logger.debug("p2pSaveNeighbours: end"); + logger.debug("saveP2PNeighbours: end"); // try { // Thread.sleep(SAVE_NEIGHBOURS_DELAY); // PostMessage(P2P_SAVE_NEIGHBOURS); // } catch(Exception e) { - // } } @@ -471,7 +489,7 @@ private void p2pAddNewNodes() { activeKnownNodes = newActiveNodes; // save list - p2pSaveNeighbours(); + saveP2PNeighbours(); try { Thread.sleep(5000); PostMessage(P2P_START_SCAN); @@ -588,7 +606,6 @@ public static void main(String[] args) throws ExecutionException, InterruptedExc } } - private static File ensureFolder(File zFolder) { if(!zFolder.exists()) { From 97b137f1c35c87aac7afeb2bd9c5787634a70cf9 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Wed, 30 Jun 2021 16:12:29 +0100 Subject: [PATCH 39/55] p2p: removed debug statements. --- .../minima/system/network/base/P2PStart.java | 100 ++---------------- 1 file changed, 10 insertions(+), 90 deletions(-) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index e53a0a196..d760f3a69 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -72,7 +72,6 @@ public class P2PStart extends MessageProcessor { private NetworkHandler mNetwork; private DiscoveryNetwork network; Set activeKnownNodes; - Set allDiscoveredNodes = new HashSet<>(); Hashtable allDiscoveredNodes2 = new Hashtable<>(); private NodeId nodeId; private PubKey pubKey; @@ -140,6 +139,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe if(privKey == null) { System.out.println("P2P Error - priv key uninitialized!"); + logger.error("P2P Error - priv key uninitialized!"); return; } @@ -158,8 +158,7 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe //ByteBuf wrappedBuf = wrappedBuffer(marshaledPubKey); // logger.debug("P2P layer - wrapped buff protobuf pubkey (hex)): " + hex(wrappedBuf.array())); // Integer.toHexString(marshaledPubKey.length)); // logger.debug("P2P layer - wrapped buff protobuf pubkey capacity (dec): " + wrappedBuf.capacity()); - - // this line below does not work. it should just add 0x0025 in front of the protobuf (identity+protobuf pubkey length). + // The line below does not work. It should just add 0x0025 in front of the protobuf (identity+protobuf pubkey length). //Multihash mhash = Multihash.digest(new Multihash.Descriptor(Multihash.Digest.Identity, null), wrappedBuf, (marshaledPubKey.length)*8); //System.out.println("P2P layer - multihash Str value: " + mhash.toString()); // last 8 bytes are zeros for some reason @@ -209,7 +208,6 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe + " , discovery address: " + discAddr.get()); System.out.println("Starting discovery loop info"); activeKnownNodes = new HashSet<>(); - // TODO: send to yourself a message 5 seconds in the future PostMessage(P2P_START_SCAN); // could also be a TimerMessage PostMessage(P2P_SAVE_NEIGHBOURS); // for now we save neighbours at the end of each scan, this timer is not used // PostTimerMessage(new TimerMessage(SAVE_NEIGHBOURS_DELAY, P2P_SAVE_NEIGHBOURS)); @@ -332,11 +330,12 @@ private PrivKey generateNodePrivateKey(File mP2PNodePrivKeyFile) { try { FileOutputStream outputStream = new FileOutputStream(mP2PNodePrivKeyFile); outputStream.write(KeyKt.marshalPrivateKey(privKey)); - System.out.println("Generated new private key and saved to local dir: " + hex(privKey.bytes())); - System.out.println("Computed public key: " + hex(pubKey.bytes())); + //System.out.println("Generated new node private key and saved to local dir: " + hex(privKey.bytes())); + System.out.println("Computed node public key: " + hex(pubKey.bytes())); outputStream.close(); } catch (Exception e) { - System.out.println("Failed to save private key to disk - " + e.getMessage()); + System.out.println("Failed to save node private key to disk - " + e.getMessage()); + logger.error("Failed to save node private key to disk - " + e.getMessage()); } return privKey; } @@ -359,9 +358,11 @@ private List> loadP2PNeighbours(File mP2PBootnodesFile) { // expect public key, p2p multiaddr and ENR // TODO: add fields validation records.add(Arrays.asList(values)); - System.out.println("Loaded bootnode from CSV: " + records.get(records.size() - 1).toString()); + //System.out.println("Loaded bootnode from CSV: " + records.get(records.size() - 1).toString()); + logger.info("Loaded bootnode from CSV: " + records.get(records.size() - 1).toString()); } else { - logger.warn("Failed to parse line from CSV: " + line); + logger.error("Failed to parse line from CSV: " + line); + System.out.println("P2P - error parsing line from CSV: " + line); } } else { logger.debug("P2PStart: skipping empty line in CSV"); @@ -377,7 +378,6 @@ private List> loadP2PNeighbours(File mP2PBootnodesFile) { private void saveP2PNeighbours() { logger.debug("saveP2PNeighbours: trying to save neighbours list"); - System.out.println("saveP2PNeighbours: trying to save neighbours list"); if(allDiscoveredNodes2.size() == 0) { logger.debug("savep2pNeighbours: empty nodes list, nothing to save, exiting."); return; @@ -397,27 +397,19 @@ public void accept(String pubKey, MinimaNodeInfo i) { } catch(Exception e) { logger.warn("Could not write lines to neighbours list file! " + "msg=" + e.getMessage()); e.printStackTrace(); -// logger.warn("Stack trace: " + e.getStackTrace().toString()); } //csvWriter.flush(); } catch(IOException e) { logger.warn("Could not flush and close neighbours list file! " + "msg=" + e.getMessage()); e.printStackTrace(); -// logger.warn("Stack trace: " + e.getStackTrace().toString()); } logger.debug("saveP2PNeighbours: end"); - // try { - // Thread.sleep(SAVE_NEIGHBOURS_DELAY); - // PostMessage(P2P_SAVE_NEIGHBOURS); - // } catch(Exception e) { - // } } private void p2pAddNewNodes() { if(network != null) { Set activeKnownNodes = new HashSet<>(); while (true) { - // we dont really care about this list... network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true @@ -533,78 +525,6 @@ public String getDiscoMultiAddrTCPFromENR(String ENR, byte[] marshalledPubKey) //TODO: refactor below code to use above object and constructor instead - if possible public static void main(String[] args) throws ExecutionException, InterruptedException { System.out.println("Hello world!"); - // attempt 1: start with DiscoveryNetworkFactory - DiscoveryNetworkFactory factory = new DiscoveryNetworkFactory(); - final DiscoveryNetwork network; - try { - - String node1_id; - String[] node1_addr_fields; - - if (args.length == 2) { // first is p2p addr, second is enr - System.out.println("Found addr node 1: " + args[0]); - System.out.println("Found boot node 1: " + args[1]); - node1_addr_fields = args[0].split("/"); - node1_id = node1_addr_fields[node1_addr_fields.length - 1]; - String bootnode_1 = args[1]; - network = factory.builder().staticPeer(args[0]).bootnode(bootnode_1).buildAndStart(0); - LibP2PNodeId id_1 = new LibP2PNodeId(PeerId.fromBase58(node1_id)); - - } else if (args.length == 1) { // first is p2p addr - deprecated - should start with p2p addr and ENR - // (application level address) - // DiscV5 cant start without at least one boot ndoe - logger.warn("Careful! This mode is deprecated, either start with 0 or 2 args, not 1."); - System.out.println("Found addr: " + args[0]); - node1_addr_fields = args[0].split("/"); - node1_id = node1_addr_fields[node1_addr_fields.length - 1]; - - System.out.println("Found node1_id: " + node1_id); - // Multiaddr address = Multiaddr.fromString(args[0]); - network = factory.builder().staticPeer(args[0]).buildAndStart(0); - } else { - // server mode only - no peer to connect to - network = factory.builder().buildAndStart(0); - //id = network.getNodeId(); - } - - Optional discAddr = network.getDiscoveryAddress(); - logger.warn("LOGGER nodeid: " + network.getNodeId() + " , nodeAddress: " + network.getNodeAddress() - + " , discovery address: " + discAddr.get()); - System.out.println("Starting discovery loop info"); - - if (network != null) { - Set activeKnownNodes = new HashSet<>(); - while (true) { - network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { - logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true - }); - - Set newActiveNodes = new HashSet<>(); - //logger.debug("trying to stream discovery peers"); - network.streamKnownDiscoveryPeers() - .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr - // logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey()); - newActiveNodes.add(discoPeer.getNodeAddress()); - }); - - Set delta = new HashSet(newActiveNodes); - delta.removeAll(activeKnownNodes); //now contains only new sockets - - for(InetSocketAddress i: delta) { - logger.info("New peer address: " + i.toString()); - } - - // update known nodes - activeKnownNodes = newActiveNodes; - - Thread.sleep(5000); - } - } - } catch (Exception e) { - // TODO Auto-generated catch blocks - e.printStackTrace(); - } - } private static File ensureFolder(File zFolder) { From 280eb3d80e58344e8d695381229a852adb91b805 Mon Sep 17 00:00:00 2001 From: panda Date: Tue, 13 Jul 2021 05:43:07 -0400 Subject: [PATCH 40/55] added cluster topology to e2e test --- endtoend/src/staticTests.js | 62 ++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 9ad92e628..23bedf7b7 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -35,7 +35,8 @@ const cfg = { hostCfgExpose: { NetworkMode: "minima-e2e-testnet", PortBindings: {"9002/tcp": [ { "HostPort": "9002"} ] } }, host_port: 9002, TOPO_STAR: "star", - TOPO_LINE: "line" + TOPO_LINE: "line", + TOPO_CLUSTER: "cluster" } var containers = {}; @@ -87,21 +88,26 @@ const runContainerInspect = (topology, nbNodes, tests_collection) => { } get_node_args = function(topology, pos) { - var parent = 0; - if(topology === cfg.TOPO_STAR) { - parent = ip_addrs["1"]; - } else if (topology === cfg.TOPO_LINE) { - parent = ip_addrs['' + (pos - 1)]; - } - p2p = true; - var node_args; + p2p = false; + var node_args = []; if(p2p) { // these two fields must be retrieved programmatically from node1 // node_args = ["-p2p-static","/ip4/172.18.0.2/udp/11522/p2p/16Uiu2HAkvSYiDo3G4Cw7XVicPK5BMjgm8vMHKYtGNCWQvskV3RdQ","-p2p-bootnode","enr:-Iu4QDirGhMYfgvNha7PVhMshqn1INf8ZjV2As0YkMgszLR1OlglWz68HjTLNxUml_BHbNGmq1C9zM3OyQiJzjX6YJYBgmlkgnY0gmlwhKwSAAKJc2VjcDI1NmsxoQIPFQyakHo15u_GazoWP_L3Qboxkjgpv2gK-Des9SMZj4N0Y3CCLQKDdWRwgi0C"]; node_args = ["-p2p-static", p2pdiscoveryaddr, "-p2p-bootnode", p2penr]; - } else { - node_args = ["-connect", parent, "9001"]; + } else if (topology === cfg.TOPO_STAR) { + node_args = ["-connect", ip_addrs["1"], "9001"]; + } else if (topology === cfg.TOPO_LINE) { + node_args = ["-connect", ip_addrs['' + (pos - 1)], "9001"]; + } else if (topology === cfg.TOPO_CLUSTER) { + if (pos < 5 + 1) { + for(let i = 1; i < pos; i++) { + node_args.push("-connect", ip_addrs['' + i], "9001"); + } + } else { + var rn = Math.ceil(Math.random() * 5); + node_args.push("-connect", ip_addrs['' + rn], '9001'); + } } return node_args; } @@ -166,11 +172,43 @@ const lineContainerInspect = (pos, nbNodes, tests_collection) => { }) } +start_other_nodes_cluster = async function (nbNodes, pos, tests_collection) { + if (pos < 2 || pos > nbNodes) { + return; + } + var node_args = get_node_args(cfg.TOPO_CLUSTER, pos); + console.log("topo cluster node " + pos + " args: " + node_args); + containers[pos] = await createMinimaContainer(node_args, cfg.node_prefix + pos, cfg.hostConfig); + + // Start the container. + await containers[pos].start(); + await sleep(10000); + + await clusterContainerInspect(pos, nbNodes, tests_collection); +} + +const clusterContainerInspect = (pos, nbNodes, tests_collection) => { + return new Promise((resolve) => { + containers[''+pos].inspect(async function(err, data) { + console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); + ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; + if(pos == nbNodes) { + await sleep(5000); + resolve(tests_collection(0, ip_addrs)) + } else { + resolve(start_other_nodes_cluster(nbNodes, pos+1, tests_collection)) + } + }) + }) +} + start_other_nodes = async function (topology, nbNodes, tests_collection) { if(topology === cfg.TOPO_STAR) { await start_other_nodes_star(nbNodes, tests_collection); } else if(topology === cfg.TOPO_LINE) { await start_other_nodes_line(nbNodes, 2, tests_collection); + } else if(topology === cfg.TOPO_CLUSTER) { + await start_other_nodes_cluster(nbNodes, 2, tests_collection); } else { console.log("Unsupported topology! This error should be caught earlier."); console.log(" topology=" + topology); @@ -238,7 +276,7 @@ stop_docker_nodes = async function() { // setup a network of nbNodes minima nodes in star topology and runs tests_collection on it with argument ip_addrs[node_prefix+ "01"] . start_static_network_tests = async function (topology, nbNodes, nodeFailure, tests_collection) { - if(!(topology === cfg.TOPO_STAR || topology === cfg.TOPO_LINE)) { + if(!(topology === cfg.TOPO_STAR || topology === cfg.TOPO_LINE || topology === cfg.TOPO_CLUSTER)) { console.log("Error! Unsupported topology: " + topology); return; } From c6b9c3248465090654059c18f7fc33d29a63e031 Mon Sep 17 00:00:00 2001 From: panda Date: Tue, 13 Jul 2021 12:41:58 -0400 Subject: [PATCH 41/55] updated e2e test --- endtoend/src/staticTests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index f6cdaec4f..685d6d254 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -108,12 +108,12 @@ get_node_args = function(topology, pos) { } else if (topology === cfg.TOPO_LINE) { node_args = ["-connect", ip_addrs['' + (pos - 1)], "9001"]; } else if (topology === cfg.TOPO_CLUSTER) { - if (pos < 5 + 1) { + if (pos < 3 + 1) { for(let i = 1; i < pos; i++) { node_args.push("-connect", ip_addrs['' + i], "9001"); } } else { - var rn = Math.ceil(Math.random() * 5); + var rn = Math.ceil(Math.random() * 3); node_args.push("-connect", ip_addrs['' + rn], '9001'); } } From b2df4d011330c0f7f46d06b5f0ccd4435d76eaa0 Mon Sep 17 00:00:00 2001 From: panda Date: Wed, 14 Jul 2021 02:35:41 -0400 Subject: [PATCH 42/55] updated e2e test --- endtoend/package-lock.json | 34 ++++++++++++++++++++++++++++++---- endtoend/package.json | 2 ++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/endtoend/package-lock.json b/endtoend/package-lock.json index 3595c06a0..d9bae740e 100644 --- a/endtoend/package-lock.json +++ b/endtoend/package-lock.json @@ -10,6 +10,8 @@ "license": "ISC", "dependencies": { "axios": "^0.21.1", + "bn.js": "^4.11.9", + "bn.js-types": "^1.0.1", "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", @@ -19,6 +21,11 @@ "minima-api": "file:minima-api-1.0.0.tgz" } }, + "node_modules/@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, "node_modules/asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -83,8 +90,15 @@ "node_modules/bn.js": { "version": "4.11.9", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", - "peer": true + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "node_modules/bn.js-types": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bn.js-types/-/bn.js-types-1.0.1.tgz", + "integrity": "sha512-1kTB06ujmupJAPOCwno8pGYVwU1Ye33EoKT77ZwuFJbiBVlSx0TLhA8xlbrmY4DPaXS7fnpVeBVpGVimjMZEVQ==", + "dependencies": { + "@types/node": "^10.12.12" + } }, "node_modules/buffer": { "version": "5.7.1", @@ -462,6 +476,11 @@ } }, "dependencies": { + "@types/node": { + "version": "10.17.60", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", + "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -509,8 +528,15 @@ "bn.js": { "version": "4.11.9", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", - "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", - "peer": true + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==" + }, + "bn.js-types": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/bn.js-types/-/bn.js-types-1.0.1.tgz", + "integrity": "sha512-1kTB06ujmupJAPOCwno8pGYVwU1Ye33EoKT77ZwuFJbiBVlSx0TLhA8xlbrmY4DPaXS7fnpVeBVpGVimjMZEVQ==", + "requires": { + "@types/node": "^10.12.12" + } }, "buffer": { "version": "5.7.1", diff --git a/endtoend/package.json b/endtoend/package.json index e79cddd20..2443986f8 100644 --- a/endtoend/package.json +++ b/endtoend/package.json @@ -10,6 +10,8 @@ "license": "ISC", "dependencies": { "axios": "^0.21.1", + "bn.js": "^4.11.9", + "bn.js-types": "^1.0.1", "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", From 3c334d3bae3c41168b099939209ef50daf77214b Mon Sep 17 00:00:00 2001 From: panda Date: Wed, 14 Jul 2021 03:59:43 -0400 Subject: [PATCH 43/55] updated endtoend test --- endtoend/src/staticTests.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 685d6d254..a63d0c1c2 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -130,7 +130,7 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { await containers[pos].start(); - await sleep(10000); + await sleep(20000); console.log("node " + pos); await starContainerInspect(pos, nbNodes, tests_collection); } @@ -161,7 +161,7 @@ start_other_nodes_line = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(10000); + await sleep(20000); await lineContainerInspect(pos, nbNodes, tests_collection); } @@ -191,7 +191,7 @@ start_other_nodes_cluster = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(10000); + await sleep(20000); await clusterContainerInspect(pos, nbNodes, tests_collection); } From 0e8e40c3c3cf14d28a561a1045234a2c143ce19a Mon Sep 17 00:00:00 2001 From: spartacusrex Date: Thu, 15 Jul 2021 12:39:54 +0100 Subject: [PATCH 44/55] Add a variable to specify if this is an incoming or outgoing connection and the local port as well as port --- .../system/network/base/MinimaClient.java | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/org/minima/system/network/base/MinimaClient.java b/src/org/minima/system/network/base/MinimaClient.java index dbe9d0a17..cb8a61f46 100644 --- a/src/org/minima/system/network/base/MinimaClient.java +++ b/src/org/minima/system/network/base/MinimaClient.java @@ -78,6 +78,7 @@ public class MinimaClient extends MessageProcessor { //The Host and Port String mHost; int mPort; + int mLocalPort; //Ping each other to know you are still up and running.. every 10 mins.. public static final int PING_INTERVAL = 1000 * 60 * 10; @@ -89,7 +90,12 @@ public class MinimaClient extends MessageProcessor { boolean mReconnect = false; int mReconnectAttempts = 0; - /** + /** + * Incoming or Outgoing + */ + boolean mIncoming; + + /** * Constructor * * @param zSock @@ -117,6 +123,7 @@ public MinimaClient(String zHost, int zPort, NetworkHandler zNetwork) { //Store mHost = zHost; mPort = zPort; + mLocalPort = zPort; //We will attempt to reconnect if this connection breaks.. mReconnect = true; @@ -127,6 +134,9 @@ public MinimaClient(String zHost, int zPort, NetworkHandler zNetwork) { //Create a UID mUID = ""+Math.abs(new Random().nextInt()); + //Outgoing connection + mIncoming = false; + //Start the connection PostMessage(NETCLIENT_INITCONNECT); } @@ -136,13 +146,14 @@ public MinimaClient(Socket zSock, NetworkHandler zNetwork) { //This is an incoming connection.. no reconnect attempt mReconnect = false; - + //Store mSocket = zSock; //Store mHost = mSocket.getInetAddress().getHostAddress(); mPort = mSocket.getPort(); + mLocalPort = mSocket.getLocalPort(); //Main network Handler mNetworkMain = zNetwork; @@ -150,6 +161,9 @@ public MinimaClient(Socket zSock, NetworkHandler zNetwork) { //Create a UID mUID = ""+Math.abs(new Random().nextInt()); + //Incoming connection + mIncoming = true; + //Start the system.. PostMessage(NETCLIENT_STARTUP); } @@ -178,6 +192,14 @@ public String getUID() { return mUID; } + public boolean isIncoming() { + return mIncoming; + } + + public int getLocalPort() { + return mLocalPort; + } + public String getNodeID() { return nodeID; } @@ -196,6 +218,8 @@ public JSONObject toJSON() { ret.put("uid", mUID); ret.put("host", getHost()); ret.put("port", getPort()); + ret.put("localport", getLocalPort()); + ret.put("incoming", mIncoming); return ret; } From e035d480c7a1a2caaeac4f6a7048e60d29c758c0 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 15 Jul 2021 15:50:17 +0100 Subject: [PATCH 45/55] Dockerfile: added comments to document the Dockerfiles. --- Dockerfile | 5 ++++- Dockerfile.arm64v8 | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 066cc7096..49d1f285c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -#FROM openjdk:11 +#FROM openjdk:11-slim FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.9_11-slim as build-stage WORKDIR /usr/src/minima/ COPY gradle gradle @@ -18,6 +18,9 @@ RUN ls -l build/libs/* RUN stat build/libs/minima-all.jar #RUN tar -cf minimajar.tar build/libs/minima-all.jar +#FROM openjdk:11-slim as production-stage +#RUN apt-get update +#RUN apt-get install -y curl FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.9_11-slim as production-stage RUN apk --no-cache add curl COPY --from=build-stage /usr/src/minima/build/libs/minima-all.jar /opt/minima/minima.jar diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 index dfa8ee410..0da55353e 100644 --- a/Dockerfile.arm64v8 +++ b/Dockerfile.arm64v8 @@ -1,4 +1,4 @@ -#FROM openjdk:11 +#FROM openjdk:11 as build-stage #FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.9_11-slim as build-stage FROM arm64v8/adoptopenjdk:11-jdk-hotspot-focal as build-stage WORKDIR /usr/src/minima/ @@ -19,6 +19,8 @@ RUN ls -l build/libs/* RUN stat build/libs/minima-all.jar #RUN tar -cf minimajar.tar build/libs/minima-all.jar +# no alpine multi arch images yet, so we build with 11-jdk-hotspot-focal to build a 500 MB image (:11 leads to 700 MB image) +#FROM openjdk:11 as production-stage #FROM adoptopenjdk/openjdk11:x86_64-alpine-jdk-11.0.9_11-slim as production-stage FROM arm64v8/adoptopenjdk:11-jdk-hotspot-focal as production-stage #COPY --from=build-stage /usr/src/minima/build/resources/main/log4j2.xml /opt/minima/log4j2.xml From 0d0c29dac60d20009e2d7a95b04dd1544f54f392 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 15 Jul 2021 18:13:51 +0100 Subject: [PATCH 46/55] e2e: re-enabled p2p mode by default. --- endtoend/src/staticTests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index a63d0c1c2..16f5873ec 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -96,7 +96,7 @@ const runContainerInspect = (topology, nbNodes, tests_collection) => { } get_node_args = function(topology, pos) { - p2p = false; + p2p = true; var node_args = []; if(p2p) { From e09e4ec42a8e457b6e252edcec53bab1f4a33456 Mon Sep 17 00:00:00 2001 From: Jerome Rousselot Date: Thu, 15 Jul 2021 18:14:37 +0100 Subject: [PATCH 47/55] p2p: refactored timer to use timerMessage and uses CONNECT Minima API when new nodes are found. --- .../minima/system/network/base/P2PStart.java | 164 +++++++++--------- 1 file changed, 86 insertions(+), 78 deletions(-) diff --git a/src/org/minima/system/network/base/P2PStart.java b/src/org/minima/system/network/base/P2PStart.java index d760f3a69..0f8e2737e 100644 --- a/src/org/minima/system/network/base/P2PStart.java +++ b/src/org/minima/system/network/base/P2PStart.java @@ -66,6 +66,7 @@ public class P2PStart extends MessageProcessor { public static final String P2P_START_SCAN = "P2P_START_SCAN"; public static final String P2P_STOP_SCAN = "P2P_STOP_SCAN"; public static final String P2P_SAVE_NEIGHBOURS = "P2P_SAVE_NEIGHBOURS"; + public static final int P2P_SCAN_INTERVAL = 10*1000; // sync neighbours table from p2p layer every N ms. public static final int SAVE_NEIGHBOURS_DELAY = 10*1000; // save list every 10 seconds public static final String COMMA_DELIMITER = ","; private String mConfFolder; @@ -208,8 +209,8 @@ public P2PStart(String zConfFolder, NetworkHandler minimaNet, String[] _staticPe + " , discovery address: " + discAddr.get()); System.out.println("Starting discovery loop info"); activeKnownNodes = new HashSet<>(); - PostMessage(P2P_START_SCAN); // could also be a TimerMessage - PostMessage(P2P_SAVE_NEIGHBOURS); // for now we save neighbours at the end of each scan, this timer is not used + PostTimerMessage(new TimerMessage(P2P_SCAN_INTERVAL, P2P_START_SCAN)); // could also be a TimerMessage + // PostMessage(P2P_SAVE_NEIGHBOURS); // for now we save neighbours at the end of each scan, this timer is not used // PostTimerMessage(new TimerMessage(SAVE_NEIGHBOURS_DELAY, P2P_SAVE_NEIGHBOURS)); } else { // initialization failed - what do we do? @@ -289,10 +290,14 @@ protected void processMessage(Message zMessage) throws Exception { if(zMessage.isMessageType(P2P_START_SCAN)) { p2pAddNewNodes(); - // saveP2PNeighbours(); // for now save neighbours list after each scan + // timer must be re-sent each time + PostTimerMessage(new TimerMessage(P2P_SCAN_INTERVAL, P2P_START_SCAN)); } else if(zMessage.isMessageType(P2P_SAVE_NEIGHBOURS)) { // saveP2PNeighbours(); - } + } else if(zMessage.isMessageType(P2P_STOP_SCAN)) { + // TODO: stop scanning + // also stop P2P layer? + } } private PrivKey loadNodePrivateKey(File mP2PNodePrivKeyFile) { @@ -407,89 +412,92 @@ public void accept(String pubKey, MinimaNodeInfo i) { } private void p2pAddNewNodes() { - if(network != null) { + logger.debug("p2pAddNewNodes: start"); + if (network != null) { Set activeKnownNodes = new HashSet<>(); - while (true) { - // we dont really care about this list... - network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { - logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true - - }); - - ArrayList mClients = mNetwork.getNetClients(); - - Set knownNodeIDs = new HashSet<>(); - for(MinimaClient mClient: mClients) { - logger.debug(" mclient nodeid=" + mClient.getNodeID() + ", nodeRecord=" + mClient.getNodeRecord()); - knownNodeIDs.add(mClient.getNodeID()); - } + // // we dont really care about this list... + // network.streamPeers().filter(peer -> peer.getId() != null).forEach(peer -> { + // logger.debug("peer: id=" + peer.getId()); // peer address == peer id and " isConnected=" true + // }); - Set newActiveNodes = new HashSet<>(); - Set unconnectedNewNodes = new HashSet<>(); - network.streamKnownDiscoveryPeers() - .forEach(discoPeer -> { // disc peer node address should be inetsocketaddr - PeerId peerid = new PeerId(discoPeer.getNodeID().toArray()); - // nodeAddress: enr_ip:enr_port - // pubkey:enr_secp256k1 - // nodeid: derived(enr_secp256k1) - logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey() - + " peerid: " + peerid - + " nodeid:" + discoPeer.getNodeID().toHexString() + " enr: " + discoPeer.getNodeRecord()); - //TODO: establish link between Bytes nodeID and libp2p nodeid / peerid - //TODO: verify values for nodeid and enr and filter existing nodes vs new based on nodeid - //Optional tmpdiscopeer = NodeRecordConverter.convertToDiscoveryPeer(discoPeer.getNodeRecord()); - - //nodeRecord.getNodeId() - newActiveNodes.add(discoPeer.getNodeAddress()); - - if(!knownNodeIDs.contains(discoPeer.getNodeID().toString())) { - logger.debug("FOUND NEW NODE: nodeid:" + discoPeer.getNodeID().toString() + " " + discoPeer.getNodeRecord().toString()); - MinimaNodeInfo aNewNodeInfo = new MinimaNodeInfo(discoPeer.getNodeID().toHexString(), - discoPeer.getNodeRecord().toString(), - discoPeer.getNodeAddress(), - getDiscoMultiAddrTCPFromENR(discoPeer.getNodeRecord(), discoPeer.getPublicKey().toArray()) - ); - unconnectedNewNodes.add(aNewNodeInfo); - allDiscoveredNodes2.put(aNewNodeInfo.nodeID, aNewNodeInfo); - } else { - logger.debug("SKIPPING an already connected node: nodeid:" + discoPeer.getNodeID().toHexString() + " " + discoPeer.getNodeRecord().toString()); - } - }); - - - Set delta = new HashSet(newActiveNodes); - delta.removeAll(activeKnownNodes); //now contains only new sockets - - for(MinimaNodeInfo i: unconnectedNewNodes) { - logger.info("New peer address: " + i.socket.toString().substring(1)); - // TODO: replace ENR with nodeID, but P2PStart.nodeID is not the correct value (16... and not the bytes) - if (i.nodeRecord.compareTo(network.getENR())==0) { - logger.warn("IGNORING node ENR in list of new peers."); - } else if(i.nodeRecord == null || i.nodeID == null) { - logger.warn("IGNORING empty nodeRecord or nodeID."); - } else { - logger.info("CONNECTING to new ENR " + i.nodeRecord); - System.out.println("Starting MinimaClient: " + i.socket.toString().substring(1) + ":9001"); - String nodeRecord = i.nodeRecord, nodeID = i.nodeID; - MinimaClient mclient = new MinimaClient(i.socket.getAddress().toString().substring(1), 9001, mNetwork, nodeID, nodeRecord); // hardcode port for now - mNetwork.PostMessage(new Message(NetworkHandler.NETWORK_NEWCLIENT).addObject("client", mclient)); - } - } + ArrayList mClients = mNetwork.getNetClients(); - // update known nodes - activeKnownNodes = newActiveNodes; + Set knownNodeIDs = new HashSet<>(); - // save list - saveP2PNeighbours(); - try { - Thread.sleep(5000); - PostMessage(P2P_START_SCAN); - } catch(Exception e) { + for (MinimaClient mClient : mClients) { + logger.debug(" mclient nodeid=" + mClient.getNodeID() + ", nodeRecord=" + mClient.getNodeRecord()); + knownNodeIDs.add(mClient.getNodeID()); + } + Set newActiveNodes = new HashSet<>(); + Set unconnectedNewNodes = new HashSet<>(); + network.streamKnownDiscoveryPeers().forEach(discoPeer -> { // disc peer node address should be + // inetsocketaddr + PeerId peerid = new PeerId(discoPeer.getNodeID().toArray()); + // nodeAddress: enr_ip:enr_port + // pubkey:enr_secp256k1 + // nodeid: derived(enr_secp256k1) + logger.debug("discovery peer: " + discoPeer.getNodeAddress() + " pubkey=" + discoPeer.getPublicKey() + + " peerid: " + peerid + " nodeid:" + discoPeer.getNodeID().toHexString() + " enr: " + + discoPeer.getNodeRecord()); + // TODO: establish link between Bytes nodeID and libp2p nodeid / peerid + // TODO: verify values for nodeid and enr and filter existing nodes vs new based + // on nodeid + // Optional tmpdiscopeer = + // NodeRecordConverter.convertToDiscoveryPeer(discoPeer.getNodeRecord()); + + // nodeRecord.getNodeId() + newActiveNodes.add(discoPeer.getNodeAddress()); + + if (!knownNodeIDs.contains(discoPeer.getNodeID().toString())) { + logger.debug("FOUND NEW NODE: nodeid:" + discoPeer.getNodeID().toString() + " " + + discoPeer.getNodeRecord().toString()); + MinimaNodeInfo aNewNodeInfo = new MinimaNodeInfo(discoPeer.getNodeID().toHexString(), + discoPeer.getNodeRecord().toString(), discoPeer.getNodeAddress(), + getDiscoMultiAddrTCPFromENR(discoPeer.getNodeRecord(), discoPeer.getPublicKey().toArray())); + unconnectedNewNodes.add(aNewNodeInfo); + allDiscoveredNodes2.put(aNewNodeInfo.nodeID, aNewNodeInfo); + } else { + logger.debug("SKIPPING an already connected node: nodeid:" + discoPeer.getNodeID().toHexString() + + " " + discoPeer.getNodeRecord().toString()); + } + }); + + Set delta = new HashSet(newActiveNodes); + delta.removeAll(activeKnownNodes); // now contains only new sockets + + for (MinimaNodeInfo i : unconnectedNewNodes) { + logger.info("New peer address: " + i.socket.toString().substring(1)); + // TODO: replace ENR with nodeID, but P2PStart.nodeID is not the correct value + // (16... and not the bytes) + if (i.nodeRecord.compareTo(network.getENR()) == 0) { + logger.warn("IGNORING node ENR in list of new peers."); + } else if (i.nodeRecord == null || i.nodeID == null) { + logger.warn("IGNORING empty nodeRecord or nodeID."); + } else { + logger.info("CONNECTING to new ENR " + i.nodeRecord); + System.out.println("Starting MinimaClient: " + i.socket.toString().substring(1) + ":9001"); + String nodeRecord = i.nodeRecord, nodeID = i.nodeID; + MinimaClient mclient = new MinimaClient(i.socket.getAddress().toString().substring(1), 9001, + mNetwork, nodeID, nodeRecord); // hardcode port for now + mNetwork.PostMessage(new Message(NetworkHandler.NETWORK_NEWCLIENT).addObject("client", mclient)); } } + + // update known nodes + activeKnownNodes = newActiveNodes; + + // save list + saveP2PNeighbours(); + // try { + // Thread.sleep(5000); + // PostMessage(P2P_START_SCAN); + // } catch(Exception e) { + + // } } + logger.debug("p2pAddNewNodes: end"); } public String getDiscoMultiAddrTCPFromENR(String ENR, byte[] marshalledPubKey) { From 2645d6306e1160e70804055583e22a364f3e3654 Mon Sep 17 00:00:00 2001 From: panda Date: Thu, 22 Jul 2021 12:39:03 -0400 Subject: [PATCH 48/55] generate chart for p2p test --- endtoend/Dockerfile | 13 +- endtoend/package-lock.json | 2307 +++++++++++++++++++++++++++++++++++ endtoend/package.json | 2 + endtoend/src/gen_graph.js | 171 +++ endtoend/src/index.js | 15 +- endtoend/src/staticTests.js | 2 +- 6 files changed, 2506 insertions(+), 4 deletions(-) create mode 100644 endtoend/src/gen_graph.js diff --git a/endtoend/Dockerfile b/endtoend/Dockerfile index d94694840..5319cc694 100644 --- a/endtoend/Dockerfile +++ b/endtoend/Dockerfile @@ -8,7 +8,15 @@ WORKDIR /app/ # install some packages on Alpine RUN apk --no-cache add \ python2 \ - bash + bash \ + build-base \ + g++ \ + cairo-dev \ + jpeg-dev \ + pango-dev \ + giflib-dev + +RUN apk add --update --repository http://dl-3.alpinelinux.org/alpine/edge/testing libmount ttf-dejavu ttf-droid ttf-freefont ttf-liberation ttf-ubuntu-font-family fontconfig # Copy nodejs package files COPY package-lock.json package.json minima-api-1.0.0.tgz /app/ @@ -29,11 +37,14 @@ CMD ["node", "src/index.js"] ARG topology=star ARG nbNodes=3 ARG nodeFailure=2 +ARG graph=false ENV topology=$topology ENV nbNodes=$nbNodes ENV nodeFailure=$nodeFailure +ENV graph=$graph RUN echo $topology RUN echo $nbNodes RUN echo $nodeFailure +RUN echo $graph diff --git a/endtoend/package-lock.json b/endtoend/package-lock.json index d9bae740e..9e24a9e85 100644 --- a/endtoend/package-lock.json +++ b/endtoend/package-lock.json @@ -15,17 +15,204 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "chartjs-node-canvas": "^3.2.0", "csv-writer": "^1.6.0", "dockerode": "^3.2.1", + "exceljs": "^4.2.1", "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" } }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.17.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.5.tgz", + "integrity": "sha512-bjqH2cX/O33jXT/UmReo2pM7DIJREPMnarixbQ57DOOzzFaI6D2+IcwaJQaJpv0M1E9TIhPCYVxrkcityLjlqA==" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.17.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.5.tgz", + "integrity": "sha512-bjqH2cX/O33jXT/UmReo2pM7DIJREPMnarixbQ57DOOzzFaI6D2+IcwaJQaJpv0M1E9TIhPCYVxrkcityLjlqA==" + }, + "node_modules/@mapbox/node-pre-gyp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz", + "integrity": "sha512-4srsKPXWlIxp5Vbqz5uLfBN+du2fJChBoYn/f2h991WLdk7jUvcSk/McVLSv/X+xQIPI8eGD5GjrnygdyHnhPA==", + "dependencies": { + "detect-libc": "^1.0.3", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.1", + "nopt": "^5.0.0", + "npmlog": "^4.1.2", + "rimraf": "^3.0.2", + "semver": "^7.3.4", + "tar": "^6.1.0" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, "node_modules/@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" }, + "node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "node_modules/archiver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz", + "integrity": "sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg==", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.0", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "node_modules/are-we-there-yet/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/are-we-there-yet/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/are-we-there-yet/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -42,6 +229,11 @@ "node": "*" } }, + "node_modules/async": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" + }, "node_modules/axios": { "version": "0.21.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", @@ -50,6 +242,11 @@ "follow-redirects": "^1.10.0" } }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -77,6 +274,26 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/big-integer": { + "version": "1.6.48", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", + "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -87,6 +304,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" + }, "node_modules/bn.js": { "version": "4.11.9", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", @@ -100,6 +322,15 @@ "@types/node": "^10.12.12" } }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "node_modules/buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -123,6 +354,44 @@ "ieee754": "^1.1.13" } }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=", + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/canvas": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.8.0.tgz", + "integrity": "sha512-gLTi17X8WY9Cf5GZ2Yns8T5lfBOcGgFehDFb+JQwDqdOoBOcECS9ZWMEAqMSVcMYwXD659J8NyzjRY/2aE+C2Q==", + "hasInstallScript": true, + "dependencies": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.14.0", + "simple-get": "^3.0.3" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.0.tgz", @@ -159,6 +428,58 @@ "chai": "^4.0.0" } }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/chart.js": { + "version": "2.9.4", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz", + "integrity": "sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A==", + "peer": true, + "dependencies": { + "chartjs-color": "^2.1.0", + "moment": "^2.10.2" + } + }, + "node_modules/chartjs-color": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz", + "integrity": "sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==", + "peer": true, + "dependencies": { + "chartjs-color-string": "^0.6.0", + "color-convert": "^1.9.3" + } + }, + "node_modules/chartjs-color-string": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz", + "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==", + "peer": true, + "dependencies": { + "color-name": "^1.0.0" + } + }, + "node_modules/chartjs-node-canvas": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/chartjs-node-canvas/-/chartjs-node-canvas-3.2.0.tgz", + "integrity": "sha512-MT0K28VG8BXwCdh5BdRXNw4nzJWKzcN3t/xgTr8zJ8M6uOXl7hRdtIW8rEUcylkLED8LGsnJmSkcnqlhPy841g==", + "dependencies": { + "canvas": "^2.6.1", + "tslib": "^1.14.1" + }, + "peerDependencies": { + "chart.js": "^2.7.3" + } + }, "node_modules/check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -172,11 +493,101 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "peer": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-convert/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "peer": true + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "peer": true + }, + "node_modules/compress-commons": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.1.tgz", + "integrity": "sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "node_modules/crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "dependencies": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + }, + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.2.tgz", + "integrity": "sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/csv-writer": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" }, + "node_modules/dayjs": { + "version": "1.10.6", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.6.tgz", + "integrity": "sha512-AztC/IOW4L1Q41A86phW5Thhcrco3xuAA+YX/BLpLWWjRcTj5TOt/QImBLmCKlrF7u7k47arTnOyL6GnbG8Hvw==" + }, "node_modules/debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -193,6 +604,17 @@ } } }, + "node_modules/decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "dependencies": { + "mimic-response": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -204,6 +626,22 @@ "node": ">=0.12" } }, + "node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, + "node_modules/detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "bin": { + "detect-libc": "bin/detect-libc.js" + }, + "engines": { + "node": ">=0.10" + } + }, "node_modules/docker-modem": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-2.1.4.tgz", @@ -230,6 +668,41 @@ "node": ">= 8.0" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -238,6 +711,45 @@ "once": "^1.4.0" } }, + "node_modules/exceljs": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.2.1.tgz", + "integrity": "sha512-EogoTdXH1X1PxqD9sV8caYd1RIfXN3PVlCV+mA/87CgdO2h4X5xAEbr7CaiP8tffz7L4aBFwsdMbjfMXi29NjA==", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.5.0", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/follow-redirects": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.2.tgz", @@ -267,6 +779,73 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, + "node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/fstream/node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/fstream/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "dependencies": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, "node_modules/get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", @@ -275,6 +854,47 @@ "node": "*" } }, + "node_modules/glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==" + }, + "node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, + "node_modules/https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -294,11 +914,239 @@ } ] }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jszip": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E=" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha1-SeKM1VkBNFjIFMVHnTxmOiG/qmw=" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha1-I+89lTVWUgOmbO/VuDD4SJEa+0g=" + }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minima-api": { "version": "1.0.0", "resolved": "file:minima-api-1.0.0.tgz", @@ -308,16 +1156,137 @@ "axios": "^0.21.1" } }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "node_modules/minipass": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz", + "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "dependencies": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "node_modules/moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "peer": true, + "engines": { + "node": "*" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "dependencies": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -326,6 +1295,19 @@ "wrappy": "1" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/pathval": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", @@ -334,6 +1316,22 @@ "node": "*" } }, + "node_modules/printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==", + "bin": { + "printj": "bin/printj.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "node_modules/pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -356,6 +1354,28 @@ "node": ">= 6" } }, + "node_modules/readdir-glob": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz", + "integrity": "sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==", + "dependencies": { + "minimatch": "^3.0.4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -380,6 +1400,83 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "node_modules/set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, + "node_modules/signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "dependencies": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -425,6 +1522,46 @@ "safe-buffer": "~5.2.0" } }, + "node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", + "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "dependencies": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/tar-fs": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", @@ -451,6 +1588,38 @@ "node": ">=6" } }, + "node_modules/tar/node_modules/chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "dependencies": { + "rimraf": "^3.0.0" + }, + "engines": { + "node": ">=8.17.0" + } + }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", + "engines": { + "node": "*" + } + }, + "node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", @@ -464,23 +1633,284 @@ "node": ">=4" } }, + "node_modules/unzipper": { + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", + "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "dependencies": { + "string-width": "^1.0.2 || 2" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/zip-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.0.tgz", + "integrity": "sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==", + "dependencies": { + "archiver-utils": "^2.1.0", + "compress-commons": "^4.1.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } } }, "dependencies": { + "@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + }, + "dependencies": { + "@types/node": { + "version": "14.17.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.5.tgz", + "integrity": "sha512-bjqH2cX/O33jXT/UmReo2pM7DIJREPMnarixbQ57DOOzzFaI6D2+IcwaJQaJpv0M1E9TIhPCYVxrkcityLjlqA==" + } + } + }, + "@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "requires": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + }, + "dependencies": { + "@types/node": { + "version": "14.17.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.5.tgz", + "integrity": "sha512-bjqH2cX/O33jXT/UmReo2pM7DIJREPMnarixbQ57DOOzzFaI6D2+IcwaJQaJpv0M1E9TIhPCYVxrkcityLjlqA==" + } + } + }, + "@mapbox/node-pre-gyp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.5.tgz", + "integrity": "sha512-4srsKPXWlIxp5Vbqz5uLfBN+du2fJChBoYn/f2h991WLdk7jUvcSk/McVLSv/X+xQIPI8eGD5GjrnygdyHnhPA==", + "requires": { + "detect-libc": "^1.0.3", + "https-proxy-agent": "^5.0.0", + "make-dir": "^3.1.0", + "node-fetch": "^2.6.1", + "nopt": "^5.0.0", + "npmlog": "^4.1.2", + "rimraf": "^3.0.2", + "semver": "^7.3.4", + "tar": "^6.1.0" + } + }, "@types/node": { "version": "10.17.60", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" + }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } + }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, + "aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + }, + "archiver": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.0.tgz", + "integrity": "sha512-iUw+oDwK0fgNpvveEsdQ0Ase6IIKztBJU2U0E9MzszMfmVVUyv1QJhS2ITW9ZCqx8dktAxVAjWWkKehuZE8OPg==", + "requires": { + "archiver-utils": "^2.1.0", + "async": "^3.2.0", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + } + }, + "archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "requires": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "requires": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "asn1": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", @@ -494,6 +1924,11 @@ "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, + "async": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" + }, "axios": { "version": "0.21.1", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", @@ -502,6 +1937,11 @@ "follow-redirects": "^1.10.0" } }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -515,6 +1955,20 @@ "tweetnacl": "^0.14.3" } }, + "big-integer": { + "version": "1.6.48", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", + "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" + }, + "binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", + "requires": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + } + }, "bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", @@ -525,6 +1979,11 @@ "readable-stream": "^3.4.0" } }, + "bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=" + }, "bn.js": { "version": "4.11.9", "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", @@ -538,6 +1997,15 @@ "@types/node": "^10.12.12" } }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, "buffer": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", @@ -547,6 +2015,31 @@ "ieee754": "^1.1.13" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, + "buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==" + }, + "buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=" + }, + "canvas": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.8.0.tgz", + "integrity": "sha512-gLTi17X8WY9Cf5GZ2Yns8T5lfBOcGgFehDFb+JQwDqdOoBOcECS9ZWMEAqMSVcMYwXD659J8NyzjRY/2aE+C2Q==", + "requires": { + "@mapbox/node-pre-gyp": "^1.0.0", + "nan": "^2.14.0", + "simple-get": "^3.0.3" + } + }, "chai": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.0.tgz", @@ -574,6 +2067,52 @@ "integrity": "sha512-01jt2gSXAw7UYFPT5K8d7HYjdXj2vyeIuE+0T/34FWzlNcVbs1JkPxRu7rYMfQnJhrHT8Nr6qjSf5ZwwLU2EYg==", "requires": {} }, + "chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", + "requires": { + "traverse": ">=0.3.0 <0.4" + } + }, + "chart.js": { + "version": "2.9.4", + "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz", + "integrity": "sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A==", + "peer": true, + "requires": { + "chartjs-color": "^2.1.0", + "moment": "^2.10.2" + } + }, + "chartjs-color": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz", + "integrity": "sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==", + "peer": true, + "requires": { + "chartjs-color-string": "^0.6.0", + "color-convert": "^1.9.3" + } + }, + "chartjs-color-string": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz", + "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==", + "peer": true, + "requires": { + "color-name": "^1.0.0" + } + }, + "chartjs-node-canvas": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/chartjs-node-canvas/-/chartjs-node-canvas-3.2.0.tgz", + "integrity": "sha512-MT0K28VG8BXwCdh5BdRXNw4nzJWKzcN3t/xgTr8zJ8M6uOXl7hRdtIW8rEUcylkLED8LGsnJmSkcnqlhPy841g==", + "requires": { + "canvas": "^2.6.1", + "tslib": "^1.14.1" + } + }, "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", @@ -584,11 +2123,88 @@ "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" }, + "code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "peer": true, + "requires": { + "color-name": "1.1.3" + }, + "dependencies": { + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "peer": true + } + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "peer": true + }, + "compress-commons": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.1.tgz", + "integrity": "sha512-QLdDLCKNV2dtoTorqgxngQCMA+gWXkM/Nwu7FpeBhk/RdkzimqC3jueb/FDmaZeXh+uby1jkBqE3xArsLBE5wQ==", + "requires": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + }, + "core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, + "crc32-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.2.tgz", + "integrity": "sha512-DxFZ/Hk473b/muq1VJ///PMNLj0ZMnzye9thBpmjpJKCc5eMgB95aK8zCGrGfQ90cWo561Te6HK9D+j4KPdM6w==", + "requires": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + } + }, "csv-writer": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==" }, + "dayjs": { + "version": "1.10.6", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.10.6.tgz", + "integrity": "sha512-AztC/IOW4L1Q41A86phW5Thhcrco3xuAA+YX/BLpLWWjRcTj5TOt/QImBLmCKlrF7u7k47arTnOyL6GnbG8Hvw==" + }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -597,6 +2213,14 @@ "ms": "2.1.2" } }, + "decompress-response": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", + "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "requires": { + "mimic-response": "^2.0.0" + } + }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -605,6 +2229,16 @@ "type-detect": "^4.0.0" } }, + "delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=" + }, + "detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + }, "docker-modem": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-2.1.4.tgz", @@ -625,6 +2259,43 @@ "tar-fs": "~2.0.1" } }, + "duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "requires": { + "readable-stream": "^2.0.2" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -633,6 +2304,36 @@ "once": "^1.4.0" } }, + "exceljs": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.2.1.tgz", + "integrity": "sha512-EogoTdXH1X1PxqD9sV8caYd1RIfXN3PVlCV+mA/87CgdO2h4X5xAEbr7CaiP8tffz7L4aBFwsdMbjfMXi29NjA==", + "requires": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.5.0", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + } + }, + "exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==" + }, + "fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "requires": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + } + }, "follow-redirects": { "version": "1.13.2", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.2.tgz", @@ -648,21 +2349,320 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, + "fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "requires": { + "minipass": "^3.0.0" + } + }, + "fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + }, + "fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "requires": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + }, + "rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "requires": { + "glob": "^7.1.3" + } + } + } + }, + "gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "requires": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=" }, + "glob": { + "version": "7.1.7", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", + "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==" + }, + "has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, + "immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "requires": { + "number-is-nan": "^1.0.0" + } + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "jszip": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.6.0.tgz", + "integrity": "sha512-jgnQoG9LKnWO3mnVNBnfhkh0QknICd1FGSrXcgrl67zioyJ4wgx25o9ZqwNtrROSflGBCGYnJfjrIyRIby1OoQ==", + "requires": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "set-immediate-shim": "~1.0.1" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, + "lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "requires": { + "immediate": "~3.0.5" + } + }, + "listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=" + }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" + }, + "lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E=" + }, + "lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=" + }, + "lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha1-QVxEePK8wwEgwizhDtMib30+GOA=" + }, + "lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==" + }, + "lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha1-SeKM1VkBNFjIFMVHnTxmOiG/qmw=" + }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, + "lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha1-I+89lTVWUgOmbO/VuDD4SJEa+0g=" + }, + "lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" + }, + "lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M=" + }, + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "requires": { + "semver": "^6.0.0" + }, + "dependencies": { + "semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==" + } + } + }, + "mimic-response": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + }, "minima-api": { "version": "file:minima-api-1.0.0.tgz", "integrity": "sha512-3fSeecdPWE3J/a7F1XJhwZAwwgpz9OONmebOtWVic/a8kArYXUt93j0+PR034x67RbSmOrzM3ogzHxJav4matA==", @@ -670,16 +2670,101 @@ "axios": "^0.21.1" } }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "minipass": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.1.3.tgz", + "integrity": "sha512-Mgd2GdMVzY+x3IJ+oHnVM+KG3lA5c8tnabyJKmHSaG2kAGpudxuOf8ToDkhumF7UzME7DecbQE9uOZhNm7PuJg==", + "requires": { + "yallist": "^4.0.0" + } + }, + "minizlib": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", + "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", + "requires": { + "minipass": "^3.0.0", + "yallist": "^4.0.0" + } + }, + "mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==" + }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" }, + "moment": { + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", + "peer": true + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "nan": { + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.2.tgz", + "integrity": "sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==" + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, + "nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "requires": { + "abbrev": "1" + } + }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, + "npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "requires": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" + }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -688,11 +2773,31 @@ "wrappy": "1" } }, + "pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + }, "pathval": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==" }, + "printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==" + }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -712,6 +2817,22 @@ "util-deprecate": "^1.0.1" } }, + "readdir-glob": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz", + "integrity": "sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==", + "requires": { + "minimatch": "^3.0.4" + } + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -722,6 +2843,57 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "requires": { + "xmlchars": "^2.2.0" + } + }, + "semver": { + "version": "7.3.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.5.tgz", + "integrity": "sha512-PoeGJYh8HK4BTO/a9Tf6ZG3veo/A7ZVsYrSA6J8ny9nb3B1VrpkuN+z9OE5wfE5p6H4LchYZsegiQgbJD94ZFQ==", + "requires": { + "lru-cache": "^6.0.0" + } + }, + "set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=" + }, + "set-immediate-shim": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz", + "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=" + }, + "setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + }, + "signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==" + }, + "simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==" + }, + "simple-get": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", + "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "requires": { + "decompress-response": "^4.2.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -758,6 +2930,44 @@ "safe-buffer": "~5.2.0" } }, + "string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "requires": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "tar": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.0.tgz", + "integrity": "sha512-DUCttfhsnLCjwoDoFcI+B2iJgYa93vBnDUATYEeRx6sntCTdN01VnqsIuTlALXla/LWooNg0yEGeB+Y8WdFxGA==", + "requires": { + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "minipass": "^3.0.0", + "minizlib": "^2.1.1", + "mkdirp": "^1.0.3", + "yallist": "^4.0.0" + }, + "dependencies": { + "chownr": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", + "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==" + } + } + }, "tar-fs": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", @@ -781,6 +2991,24 @@ "readable-stream": "^3.1.1" } }, + "tmp": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", + "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "requires": { + "rimraf": "^3.0.0" + } + }, + "traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=" + }, + "tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + }, "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", @@ -791,15 +3019,94 @@ "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" }, + "unzipper": { + "version": "0.10.11", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", + "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", + "requires": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, + "wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "requires": { + "string-width": "^1.0.2 || 2" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + }, + "xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "zip-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.0.tgz", + "integrity": "sha512-zshzwQW7gG7hjpBlgeQP9RuyPGNxvJdzR8SUM3QhxCnLjWN2E7j3dOvpeDcQoETfHx0urRS7EtmVToql7YpU4A==", + "requires": { + "archiver-utils": "^2.1.0", + "compress-commons": "^4.1.0", + "readable-stream": "^3.6.0" + } } } } diff --git a/endtoend/package.json b/endtoend/package.json index 2443986f8..e25503898 100644 --- a/endtoend/package.json +++ b/endtoend/package.json @@ -15,8 +15,10 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "chartjs-node-canvas": "^3.2.0", "csv-writer": "^1.6.0", "dockerode": "^3.2.1", + "exceljs": "^4.2.1", "fs": "^0.0.1-security", "minima-api": "file:minima-api-1.0.0.tgz" } diff --git a/endtoend/src/gen_graph.js b/endtoend/src/gen_graph.js new file mode 100644 index 000000000..e70a29dc5 --- /dev/null +++ b/endtoend/src/gen_graph.js @@ -0,0 +1,171 @@ +const { CanvasRenderService } = require("chartjs-node-canvas"); +// const chartjs = require('chart.js'); +const fs = require("fs"); +var Excel = require("exceljs"); +var workbook = new Excel.Workbook(); + +var configuration = { + type: "line", + data: { + labels: [1, 2, 3, 4, 5], + datasets: [ + { + label: "Part1", + fill: false, + data: [2478, 5267, 734, 784, 433], + backgroundColor: "rgba(255, 99, 132, 1)", + borderColor: "rgba(255,99,132,1)", + }, + { + label: "Part2", + fill: false, + data: [1233, 3467, 7634, 384, 4533], + backgroundColor: "rgba(162, 99, 132, 1)", + borderColor: "rgba(162,99,132,1)", + }, + ], + }, + options: { + title: { + display: true, + text: 'Chart for p2p Test', + fontColor: "#07BDA7" + }, + scales: { + xAxes: [{ + scaleLabel: { + display: true, + labelString: "number of Minima nodes", + fontColor: "#028B7B" + }, + ticks: { + beginAtZero: true + } + }], + yAxes: [{ + scaleLabel: { + display: true, + labelString: "number of p2p neighbours", + fontColor: "#028B7B" + }, + ticks: { + beginAtZero: true + } + }] + } + } +}; + +const mkChart = async (params) => { + const canvasRenderService = new CanvasRenderService(400, 400); + return await canvasRenderService.renderToBuffer(configuration); +}; + +const test_graph_gen = async () => { + await readFiles("./results/"); + var image = await mkChart("test"); + + fs.writeFile("./results/graph.png", image, "base64", function (err) { + console.log(err); + }); +}; + +const readFiles = (dirname) => { + let label1 = [], + label2 = [], + set1 = [], + set2 = []; + return new Promise((resolve, reject) => { + fs.readdir(dirname, async function (err, filenames) { + if (err) { + console.log(err); + return; + } + for (let key in filenames) { + var nodes = [], + data = [], + sign = false; + var filename = filenames[key]; + var arr = filename.split("."); + if(arr[arr.length-1] != "csv") continue; + var worksheet = await workbook.csv.readFile(dirname + filename); + var partName = filename.split("-")[3].split(".")[0]; + + var column1 = worksheet.getColumn(1); + column1.eachCell(function (cell, rowNumber) { + if (cell.value != null) { + var temp = cell.value; + nodes.push(temp); + } + }); + nodes = nodes.slice(1); + let nodeCnt = nodes.length / 2; + if (partName === "part1") { + if (label1.indexOf(nodeCnt) > -1) { + sign = true; + } else { + label1.push(nodeCnt); + } + } else if (partName === "part2") { + if (label2.indexOf(nodeCnt) > -1) sign = true; + else label2.push(nodeCnt); + } + + var column2 = worksheet.getColumn(4); + column2.eachCell(function (cell, rowNumber) { + if (cell.value != null) { + var temp = cell.value; + data.push(temp); + } + }); + data = data.slice(1); + const reg = /\"p2pPeercount\"\:([0-9]+)\}/; + const count = parseInt(data[0].match(reg)[1]); + + if (!sign) { + if (partName === "part1") set1.push(count); + else if (partName === "part2") set2.push(count); + } + } + + //sorting values of x Axe + let temp = []; + for (let key in label1) { + let pair = {}; + pair.x = label1[key]; + pair.y = set1[key]; + temp.push(pair); + } + temp.sort((a,b) => { + return a.x - b.x + }); + label1 = temp.map(item => item.x); + set1 = temp.map(item => item.y); + + temp = []; + for (let key in label2) { + let pair = {}; + pair.x = label2[key]; + pair.y = set2[key]; + temp.push(pair); + } + temp.sort((a,b) => { + return a.x - b.x + }); + label2 = temp.map(item => item.x); + set2 = temp.map(item => item.y); + console.log("label1: ", label1); + console.log("label2: ", label2); + console.log("set1: ", set1); + console.log("set2: ", set2); + configuration.data.labels = label1; + configuration.data.datasets[0].data = set1; + configuration.data.datasets[1].data = set2; + resolve(); + }); + }); +}; + +module.exports = test_graph_gen + +// test_graph_gen() \ No newline at end of file diff --git a/endtoend/src/index.js b/endtoend/src/index.js index 61e7371b4..cc601e3ce 100644 --- a/endtoend/src/index.js +++ b/endtoend/src/index.js @@ -1,3 +1,14 @@ -var test_star_static = require('./test_star_static.js'); +const graph = process.env.graph; + +if (graph === "true") { + var test_graph_gen = require('./gen_graph.js'); + test_graph_gen(); +} else if(graph === "false") { + var test_star_static = require('./test_star_static.js'); + test_star_static(); +} + + + + -test_star_static(); diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index a63d0c1c2..16f5873ec 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -96,7 +96,7 @@ const runContainerInspect = (topology, nbNodes, tests_collection) => { } get_node_args = function(topology, pos) { - p2p = false; + p2p = true; var node_args = []; if(p2p) { From 73cdd113f9796456f5d9eb6f7441d8c8dd39267e Mon Sep 17 00:00:00 2001 From: panda Date: Fri, 23 Jul 2021 04:08:20 -0400 Subject: [PATCH 49/55] added npm package to e2e --- endtoend/package-lock.json | 25 ++++++------------------- endtoend/package.json | 1 + 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/endtoend/package-lock.json b/endtoend/package-lock.json index 9e24a9e85..bf3c3ee8f 100644 --- a/endtoend/package-lock.json +++ b/endtoend/package-lock.json @@ -15,6 +15,7 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "chart.js": "^2.9.4", "chartjs-node-canvas": "^3.2.0", "csv-writer": "^1.6.0", "dockerode": "^3.2.1", @@ -443,7 +444,6 @@ "version": "2.9.4", "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz", "integrity": "sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A==", - "peer": true, "dependencies": { "chartjs-color": "^2.1.0", "moment": "^2.10.2" @@ -453,7 +453,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz", "integrity": "sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==", - "peer": true, "dependencies": { "chartjs-color-string": "^0.6.0", "color-convert": "^1.9.3" @@ -463,7 +462,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz", "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==", - "peer": true, "dependencies": { "color-name": "^1.0.0" } @@ -505,7 +503,6 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "peer": true, "dependencies": { "color-name": "1.1.3" } @@ -513,14 +510,12 @@ "node_modules/color-convert/node_modules/color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "peer": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "peer": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/compress-commons": { "version": "4.1.1", @@ -1215,7 +1210,6 @@ "version": "2.29.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", - "peer": true, "engines": { "node": "*" } @@ -2079,7 +2073,6 @@ "version": "2.9.4", "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-2.9.4.tgz", "integrity": "sha512-B07aAzxcrikjAPyV+01j7BmOpxtQETxTSlQ26BEYJ+3iUkbNKaOJ/nDbT6JjyqYxseM0ON12COHYdU2cTIjC7A==", - "peer": true, "requires": { "chartjs-color": "^2.1.0", "moment": "^2.10.2" @@ -2089,7 +2082,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/chartjs-color/-/chartjs-color-2.4.1.tgz", "integrity": "sha512-haqOg1+Yebys/Ts/9bLo/BqUcONQOdr/hoEr2LLTRl6C5LXctUdHxsCYfvQVg5JIxITrfCNUDr4ntqmQk9+/0w==", - "peer": true, "requires": { "chartjs-color-string": "^0.6.0", "color-convert": "^1.9.3" @@ -2099,7 +2091,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/chartjs-color-string/-/chartjs-color-string-0.6.0.tgz", "integrity": "sha512-TIB5OKn1hPJvO7JcteW4WY/63v6KwEdt6udfnDE9iCAZgy+V4SrbSxoIbTw/xkUIapjEI4ExGtD0+6D3KyFd7A==", - "peer": true, "requires": { "color-name": "^1.0.0" } @@ -2132,7 +2123,6 @@ "version": "1.9.3", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "peer": true, "requires": { "color-name": "1.1.3" }, @@ -2140,16 +2130,14 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", - "peer": true + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" } } }, "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "peer": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "compress-commons": { "version": "4.1.1", @@ -2713,8 +2701,7 @@ "moment": { "version": "2.29.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.29.1.tgz", - "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==", - "peer": true + "integrity": "sha512-kHmoybcPV8Sqy59DwNDY3Jefr64lK/by/da0ViFcuA4DH0vQg5Q6Ze5VimxkfQNSC+Mls/Kx53s7TjP1RhFEDQ==" }, "ms": { "version": "2.1.2", diff --git a/endtoend/package.json b/endtoend/package.json index e25503898..ef5775be2 100644 --- a/endtoend/package.json +++ b/endtoend/package.json @@ -15,6 +15,7 @@ "chai": "^4.3.0", "chai-as-promised": "^7.1.1", "chai-bn": "^0.2.1", + "chart.js": "^2.9.4", "chartjs-node-canvas": "^3.2.0", "csv-writer": "^1.6.0", "dockerode": "^3.2.1", From 20e9825f4e1738c108c197b7f6b81ba376379904 Mon Sep 17 00:00:00 2001 From: panda Date: Sat, 24 Jul 2021 23:45:20 -0400 Subject: [PATCH 50/55] updated endtoend --- endtoend/src/gen_graph.js | 7 ++++--- endtoend/src/staticTests.js | 17 +++++------------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/endtoend/src/gen_graph.js b/endtoend/src/gen_graph.js index e70a29dc5..556ed6f7b 100644 --- a/endtoend/src/gen_graph.js +++ b/endtoend/src/gen_graph.js @@ -120,7 +120,8 @@ const readFiles = (dirname) => { }); data = data.slice(1); const reg = /\"p2pPeercount\"\:([0-9]+)\}/; - const count = parseInt(data[0].match(reg)[1]); + let counts = data.map((item, index) => index % 2 === 0 && parseInt(item.match(reg)[1])); + var count = counts.reduce(function(a, b) { return Math.max(a, b); }); if (!sign) { if (partName === "part1") set1.push(count); @@ -166,6 +167,6 @@ const readFiles = (dirname) => { }); }; -module.exports = test_graph_gen +// module.exports = test_graph_gen -// test_graph_gen() \ No newline at end of file +test_graph_gen() \ No newline at end of file diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 16f5873ec..6a9231ef5 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -76,10 +76,8 @@ const start_docker_node_1 = async function (topology, nbNodes, tests_collection) // Start the container. await containers["1"].start(); process.stdout.write("Trying to sleep for 5 seconds..."); - await sleep(10000); -// process.stdout.write("Did I sleep 5 seconds?"); + await sleep(5000); - console.log("Hello1"); await runContainerInspect(topology, nbNodes, tests_collection); } @@ -89,7 +87,7 @@ const runContainerInspect = (topology, nbNodes, tests_collection) => { ip_addrs["1"] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; console.log("Started node 1," + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); get_node_p2p_params(ip_addrs["1"], function() { - resolve(start_other_nodes(topology, nbNodes, tests_collection)); + resolve(start_other_nodes(topology, nbNodes, tests_collection)); }); }) }) @@ -130,8 +128,7 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { await containers[pos].start(); - await sleep(20000); - console.log("node " + pos); + await sleep(5000); await starContainerInspect(pos, nbNodes, tests_collection); } } @@ -142,7 +139,6 @@ const starContainerInspect = (pos, nbNodes, tests_collection) => { console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; if(pos == nbNodes) { - await sleep(5000); resolve(tests_collection(0, ip_addrs)) } else { resolve(null) @@ -161,7 +157,7 @@ start_other_nodes_line = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(20000); + await sleep(5000); await lineContainerInspect(pos, nbNodes, tests_collection); } @@ -172,7 +168,6 @@ const lineContainerInspect = (pos, nbNodes, tests_collection) => { console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; if(pos == nbNodes) { - await sleep(5000); resolve(tests_collection(0, ip_addrs)) } else { resolve(start_other_nodes_line(nbNodes, pos+1, tests_collection)) @@ -191,7 +186,7 @@ start_other_nodes_cluster = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(20000); + await sleep(5000); await clusterContainerInspect(pos, nbNodes, tests_collection); } @@ -202,7 +197,6 @@ const clusterContainerInspect = (pos, nbNodes, tests_collection) => { console.log("Started node " + pos + " IP: " + JSON.stringify(data.NetworkSettings.Networks[cfg.docker_net].IPAddress)); ip_addrs[pos] = data.NetworkSettings.Networks[cfg.docker_net].IPAddress; if(pos == nbNodes) { - await sleep(5000); resolve(tests_collection(0, ip_addrs)) } else { resolve(start_other_nodes_cluster(nbNodes, pos+1, tests_collection)) @@ -308,7 +302,6 @@ start_static_network_tests = async function (topology, nbNodes, nodeFailure, tes //stop one node and run tests await containers[''+nodeFailure].stop(); console.log("node " + nodeFailure + " stopping..."); - await sleep(10000); await tests_collection(1, ip_addrs); } From 39c8ba2581c81fa9fa26d9600c4292925e99f9ba Mon Sep 17 00:00:00 2001 From: panda Date: Sun, 25 Jul 2021 00:10:28 -0400 Subject: [PATCH 51/55] updated generating graph --- endtoend/src/gen_graph.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/endtoend/src/gen_graph.js b/endtoend/src/gen_graph.js index 556ed6f7b..dd6cb9d9e 100644 --- a/endtoend/src/gen_graph.js +++ b/endtoend/src/gen_graph.js @@ -167,6 +167,6 @@ const readFiles = (dirname) => { }); }; -// module.exports = test_graph_gen +module.exports = test_graph_gen -test_graph_gen() \ No newline at end of file +// test_graph_gen() \ No newline at end of file From 4633f4856a8c0b7c312a91c8dabc5bb73a766d45 Mon Sep 17 00:00:00 2001 From: panda Date: Sun, 25 Jul 2021 02:15:44 -0400 Subject: [PATCH 52/55] updated endtoend --- endtoend/src/test_star_static.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index fdfa940a5..3b50f4581 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -25,7 +25,7 @@ test_star_static = function () { console.log("tests collection"); //wait for processing(should depend on nbNodes but also system performance) - await sleep(60000); + await sleep(100000); if (!fs.existsSync("./results")){ fs.mkdirSync("./results"); From 77490932dda708a4fbaa6e8d6f3d22237f43bd60 Mon Sep 17 00:00:00 2001 From: panda Date: Sun, 25 Jul 2021 13:07:23 -0400 Subject: [PATCH 53/55] updated endtoend --- endtoend/src/staticTests.js | 12 ++++++------ endtoend/src/test_star_static.js | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 6a9231ef5..7d0470ff4 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -76,7 +76,7 @@ const start_docker_node_1 = async function (topology, nbNodes, tests_collection) // Start the container. await containers["1"].start(); process.stdout.write("Trying to sleep for 5 seconds..."); - await sleep(5000); + await sleep(10000); await runContainerInspect(topology, nbNodes, tests_collection); } @@ -128,7 +128,7 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { await containers[pos].start(); - await sleep(5000); + await sleep(10000); await starContainerInspect(pos, nbNodes, tests_collection); } } @@ -157,7 +157,7 @@ start_other_nodes_line = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(5000); + await sleep(10000); await lineContainerInspect(pos, nbNodes, tests_collection); } @@ -186,7 +186,7 @@ start_other_nodes_cluster = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(5000); + await sleep(10000); await clusterContainerInspect(pos, nbNodes, tests_collection); } @@ -296,8 +296,8 @@ start_static_network_tests = async function (topology, nbNodes, nodeFailure, tes } await stop_docker_nodes(); // give 5 seconds to stop all docker nodes (should depend on nbNodes but also system performance) - await sleep(5000); - await start_docker_node_1(topology, nbNodes, tests_collection); + await sleep(10000); + await start_docker_node_1(topology, nbNodes, tests_collection); //stop one node and run tests await containers[''+nodeFailure].stop(); diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 3b50f4581..51444311a 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -25,7 +25,7 @@ test_star_static = function () { console.log("tests collection"); //wait for processing(should depend on nbNodes but also system performance) - await sleep(100000); + await sleep(60000); if (!fs.existsSync("./results")){ fs.mkdirSync("./results"); @@ -57,6 +57,7 @@ test_star_static = function () { midData["ip"] = ip_addrs[child.toString()]; var status = await Minima_API.status(); + console.log("============test============") midData["request"] = "status"; midData["response"] = JSON.stringify(status); data.push(midData); From 3fad67711d0aabb757a85f0dea47d3769e73cb48 Mon Sep 17 00:00:00 2001 From: panda Date: Sun, 25 Jul 2021 14:03:45 -0400 Subject: [PATCH 54/55] worked in endtoend --- endtoend/src/staticTests.js | 10 +++++----- endtoend/src/test_star_static.js | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/endtoend/src/staticTests.js b/endtoend/src/staticTests.js index 7d0470ff4..5be0f73d5 100644 --- a/endtoend/src/staticTests.js +++ b/endtoend/src/staticTests.js @@ -76,7 +76,7 @@ const start_docker_node_1 = async function (topology, nbNodes, tests_collection) // Start the container. await containers["1"].start(); process.stdout.write("Trying to sleep for 5 seconds..."); - await sleep(10000); + await sleep(5000); await runContainerInspect(topology, nbNodes, tests_collection); } @@ -128,7 +128,7 @@ start_other_nodes_star = async function(nbNodes, tests_collection) { await containers[pos].start(); - await sleep(10000); + await sleep(5000); await starContainerInspect(pos, nbNodes, tests_collection); } } @@ -157,7 +157,7 @@ start_other_nodes_line = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(10000); + await sleep(5000); await lineContainerInspect(pos, nbNodes, tests_collection); } @@ -186,7 +186,7 @@ start_other_nodes_cluster = async function (nbNodes, pos, tests_collection) { // Start the container. await containers[pos].start(); - await sleep(10000); + await sleep(5000); await clusterContainerInspect(pos, nbNodes, tests_collection); } @@ -296,7 +296,7 @@ start_static_network_tests = async function (topology, nbNodes, nodeFailure, tes } await stop_docker_nodes(); // give 5 seconds to stop all docker nodes (should depend on nbNodes but also system performance) - await sleep(10000); + await sleep(5000); await start_docker_node_1(topology, nbNodes, tests_collection); //stop one node and run tests diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index 51444311a..bab98d73a 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -25,7 +25,7 @@ test_star_static = function () { console.log("tests collection"); //wait for processing(should depend on nbNodes but also system performance) - await sleep(60000); + await sleep(30000); if (!fs.existsSync("./results")){ fs.mkdirSync("./results"); @@ -57,6 +57,7 @@ test_star_static = function () { midData["ip"] = ip_addrs[child.toString()]; var status = await Minima_API.status(); + await sleep(5000) console.log("============test============") midData["request"] = "status"; midData["response"] = JSON.stringify(status); @@ -64,6 +65,7 @@ test_star_static = function () { let tempData = {} var network = await Minima_API.network(); + await sleep(5000) tempData["node"] = child; tempData["ip"] = ip_addrs[child.toString()]; tempData["request"] = "network"; From c895bbb1a547d47e4ecd7f09d06bfc1394864d38 Mon Sep 17 00:00:00 2001 From: panda Date: Sun, 25 Jul 2021 16:00:30 -0400 Subject: [PATCH 55/55] updated e2e --- endtoend/src/test_star_static.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/endtoend/src/test_star_static.js b/endtoend/src/test_star_static.js index bab98d73a..fdfa940a5 100644 --- a/endtoend/src/test_star_static.js +++ b/endtoend/src/test_star_static.js @@ -25,7 +25,7 @@ test_star_static = function () { console.log("tests collection"); //wait for processing(should depend on nbNodes but also system performance) - await sleep(30000); + await sleep(60000); if (!fs.existsSync("./results")){ fs.mkdirSync("./results"); @@ -57,15 +57,12 @@ test_star_static = function () { midData["ip"] = ip_addrs[child.toString()]; var status = await Minima_API.status(); - await sleep(5000) - console.log("============test============") midData["request"] = "status"; midData["response"] = JSON.stringify(status); data.push(midData); let tempData = {} var network = await Minima_API.network(); - await sleep(5000) tempData["node"] = child; tempData["ip"] = ip_addrs[child.toString()]; tempData["request"] = "network";