[NAT] Pivot to being a test runner (#171)

* Renamed to op-acceptor
* Focused on being a test runner

Author: scharissis

.circleci/config.yml

@@ -22,7 +22,7 @@ workflows:
             op-ufm/.* run-build-op-ufm true
             proxyd/.* run-build-proxyd true
             cci-stats/.* run-build-cci-stats true
-            op-nat/.* run-build-op-nat true
+            op-acceptor/.* run-build-op-acceptor true
             .circleci/.* run-all true
             .github/.* run-all true
           filters:

.circleci/continue_config.yml

@@ -25,7 +25,7 @@ parameters:
   run-build-cci-stats:
     type: boolean
     default: false
-  run-build-op-nat:
+  run-build-op-acceptor:
     type: boolean
     default: false
   run-all:
@@ -96,7 +96,7 @@ jobs:
             echo "run-build-pms: << pipeline.parameters.run-build-pms >>"
             echo "run-build-op-ufm: << pipeline.parameters.run-build-op-ufm >>"
             echo "run-build-proxyd: << pipeline.parameters.run-build-proxyd >>"
-            echo "run-build-op-nat: << pipeline.parameters.run-build-op-nat >>"
+            echo "run-build-op-acceptor: << pipeline.parameters.run-build-op-acceptor >>"
             echo "run-all: << pipeline.parameters.run-all >>"
             echo ""
             echo "Pipeline Trigger Information:"
@@ -143,10 +143,10 @@ jobs:
               echo "op-ufm tag regex match: false"
             fi
 
-            if [[ $CURRENT_TAG =~ ^op-nat/v.* ]]; then
-              echo "op-nat tag regex match: true"
+            if [[ $CURRENT_TAG =~ ^op-acceptor/v.* ]]; then
+              echo "op-acceptor tag regex match: true"
             else
-              echo "op-nat tag regex match: false"
+              echo "op-acceptor tag regex match: false"
             fi
   
   docker-build:
@@ -519,35 +519,35 @@ workflows:
           docker_name: cci-stats
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
           docker_context: .
-  op-nat:
+  op-acceptor:
     when:
-      or: [<< pipeline.parameters.run-build-op-nat >>, << pipeline.parameters.run-all >>]
+      or: [<< pipeline.parameters.run-build-op-acceptor >>, << pipeline.parameters.run-all >>]
     jobs:
       - go-lint:
-          name: op-nat-lint
-          module: op-nat
+          name: op-acceptor-lint
+          module: op-acceptor
       - go-test:
-          name: op-nat-tests
-          module: op-nat
+          name: op-acceptor-tests
+          module: op-acceptor
       - docker-build:
-          name: op-nat-docker-build
-          docker_file: op-nat/Dockerfile
-          docker_name: op-nat
+          name: op-acceptor-docker-build
+          docker_file: op-acceptor/Dockerfile
+          docker_name: op-acceptor
           docker_tags: <<pipeline.git.revision>>,<<pipeline.git.branch>>
           docker_context: .
   release:
     jobs:
       - log-config-results:
           filters:
             tags:
-              only: /^(op-nat|cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
+              only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
             branches:
               ignore: /.*/
       - hold:
           type: approval
           filters:
             tags:
-              only: /^(op-nat|cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
+              only: /^(cci-stats|peer-mgmt-service|proxyd|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/
             branches:
               ignore: /.*/
       - docker-build:
@@ -561,7 +561,7 @@ workflows:
                 - op-conductor-mon
                 - peer-mgmt-service
                 - cci-stats
-                - op-nat
+                - op-acceptor
           name: <<matrix.docker_name>>-docker-build
           filters:
             tags:
@@ -584,7 +584,7 @@ workflows:
                 - op-conductor-mon
                 - peer-mgmt-service
                 - cci-stats
-                - op-nat
+                - op-acceptor
           name: <<matrix.docker_name>>-docker-publish
           filters:
             tags:
@@ -605,7 +605,7 @@ workflows:
                 - op-conductor-mon
                 - peer-mgmt-service
                 - cci-stats
-                - op-nat
+                - op-acceptor
           name: <<matrix.docker_name>>-docker-tag
           filters:
             tags:
@@ -627,7 +627,7 @@ workflows:
                 - op-conductor-mon
                 - peer-mgmt-service
                 - cci-stats
-                - op-nat
+                - op-acceptor
           name: <<matrix.module>>-go-release
           filters:
             tags:

README.md

@@ -3,8 +3,8 @@
 This repository is an extension of the [Optimism monorepo](https://github.com/ethereum-optimism/optimism) and contains the infrastructure that supports the Optimism ecosystem.
 
 ## Components
+- op-acceptor: Network Acceptance Tester: A tool that tests the network acceptance of devnets. (See [op-acceptor/README.md](./acceptor/README.md) or [devdocs/pm](https://devdocs.optimism.io/pm/acceptance-testing.html) for more information)
 - op-conductor-mon: Monitors multiple op-conductor instances and provides a unified interface for reporting metrics.
-- op-nat: Network Acceptance Tester: A tool that tests the network acceptance of devnets. (See [op-nat/README.md](./op-nat/README.md) or [devdocs/pm](https://devdocs.optimism.io/pm/acceptance-testing.html) for more information)
 - op-signer: Thin gateway that supports various RPC endpoints to sign payloads from op-stack components using private key stored in KMS.
 - op-ufm: User facing monitoring creates transactions at regular intervals and observe transaction propagation across different RPC providers.
 

op-nat/.dockerignore op-acceptor/.dockerignore

@@ -3,7 +3,7 @@
 
 version: 2
 
-project_name: op-nat
+project_name: op-acceptor
 
 before:
   hooks:
@@ -13,7 +13,7 @@ before:
 builds:
   - id: main
     main: ./cmd/main
-    binary: bin/op-nat
+    binary: bin/op-acceptor
     env:
       - CGO_ENABLED=0
     goos:
@@ -54,5 +54,5 @@ release:
   make_latest: false
 
 monorepo:
-  tag_prefix: op-nat/
-  dir: op-nat
+  tag_prefix: op-acceptor/
+  dir: op-acceptor

op-nat/.gitignore op-acceptor/.gitignore

@@ -5,7 +5,7 @@ WORKDIR /app
 # Install dependencies first for better caching
 RUN apk add --no-cache just
 
-COPY op-nat/ .
+COPY op-acceptor/ .
 
 RUN just build
 
@@ -16,8 +16,9 @@ RUN addgroup -S app && adduser -S app -G app && \
     mkdir -p /devnets && \
     chown -R app:app /devnets
 
-COPY --from=builder /app/bin/op-nat /app/
+COPY --from=builder /app/bin/op-acceptor /app/
+COPY op-acceptor/example-validators.yaml /app/
 USER app
 
 VOLUME /devnets
-ENTRYPOINT ["/app/op-nat"]
+ENTRYPOINT ["/app/op-acceptor"]

op-nat/.goreleaser.yaml op-acceptor/.goreleaser.yaml

@@ -10,10 +10,10 @@ LDFLAGS := -ldflags "$(LDFLAGSSTRING)"
 all: build
 
 build:
-	go build -v $(LDFLAGS) -o ./bin/op-nat ./cmd
+	go build -v $(LDFLAGS) -o ./bin/op-acceptor ./cmd
 
 clean:
-	rm ./bin/op-nat
+	rm ./bin/op-acceptor
 
 test:
 	go test -v ./...

op-nat/Dockerfile op-acceptor/Dockerfile

@@ -0,0 +1,106 @@
+# Network Acceptance Tester (op-acceptor)
+op-acceptor is a tool to run checks against a network to determine if it is ready for production. It helps ensure your network meets all necessary requirements before deployment.
+The checks it runs are standard Go tests.
+
+## Concepts
+
+### Test Discovery
+op-acceptor discovers tests. You simply need to provide a path to a directory containing your tests. op-acceptor will discover all tests within that directory and also those within any subdirectories.
+
+### Test Grouping
+1. **Test** - An individual check that validates a specific aspect of your network
+2. **Suite** - A logical grouping of related tests
+3. **Gate** - A high-level collection of suites and/or tests that represent a complete validation scenario
+
+
+## Contributing
+
+Please note that this project is under active development and the API may evolve. We welcome all contributions and appreciate your interest in improving op-acceptor!
+
+### Adding a new test/suite/gate
+All tests, suites, and gates are defined in a `validators.yaml` file. The filename is not important.
+For an example `validators.yaml`, see [example-validators.yaml](./example-validators.yaml).
+
+**Gate**
+A gate is a collection of suites and/or tests that represent a complete validation scenario. A gate can inherit from one or more other gates.
+
+```yaml
+# Example gate definition
+- id: alphanet
+  description: "Alphanet validation gate"
+  inherits: ["localnet"]
+  suites:
+    - id: interop
+      description: "Interop suite"
+      tests:
+        - id: TestInteropSystem
+```
+
+**Suite**
+A suite is a collection of tests that validate a specific aspect of your network. It's a convenience mechanism for grouping tests together.
+
+```yaml
+# Example suite definition
+- id: interop
+  description: "Interop suite"
+  tests:
+    - id: TestInteropSystem
+```
+
+**Test**
+A test is a single check that validates a specific aspect of your network.
+A description is optional.
+A package is optional. If not provided, the test will be searched for. This lengthens the runtime, so we recommend providing the package.
+If a package is provided and no name is provided, op-acceptor runs all tests in the package.
+
+```yaml
+# Run test 'TestInteropSystem' in package 'github.com/ethereum-optimism/optimism/kurtosis-devnet/tests/interop'
+- name: TestInteropSystem
+  description: "Test the interop system"
+  package: github.com/ethereum-optimism/optimism/kurtosis-devnet/tests/interop
+```
+
+```yaml
+# Run all tests in package 'github.com/ethereum-optimism/optimism/kurtosis-devnet/tests/interop'
+- package: github.com/ethereum-optimism/optimism/kurtosis-devnet/tests/interop
+```
+
+## Development
+
+### Prerequisites
+* [Go](https://go.dev/dl/) 1.22+
+* [Just](https://just.systems/)
+
+### Getting Started
+Build the binary:
+```bash
+just build
+```
+
+Run tests:
+```bash
+just test
+```
+
+Run op-acceptor:
+```bash
+DEVNET_ENV_URL=devnets/alpaca-devnet.json # path to the devnet manifest
+go run cmd/main.go \
+  --gate betanet \                  # The gate to run
+  --testdir ../../optimism/ \       # Path to the directory containing your tests
+  --validators validators.yaml \    # Path to the validator definitions
+```
+
+Want to monitor your validation runs? Start our local monitoring stack:
+```bash
+just start-monitoring  # Launches Prometheus and Grafana alongside op-acceptor
+```
+
+### Future Development
+We track our public roadmap and issues on [Github](https://github.com/ethereum-optimism/infra/issues). Feel free to:
+* Submit bug reports
+* Propose new features
+* Contribute improvements
+* Join discussions
+
+Your feedback and contributions help make op-acceptor better for everyone!

op-nat/Makefile op-acceptor/Makefile

@@ -8,28 +8,27 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/urfave/cli/v2"
 
-	nat "github.com/ethereum-optimism/infra/op-nat"
-	"github.com/ethereum-optimism/infra/op-nat/flags"
-	"github.com/ethereum-optimism/infra/op-nat/service"
-	"github.com/ethereum-optimism/infra/op-nat/validators/gates"
+	nat "github.com/ethereum-optimism/infra/op-acceptor"
+	"github.com/ethereum-optimism/infra/op-acceptor/flags"
+	"github.com/ethereum-optimism/infra/op-acceptor/service"
 	"github.com/ethereum-optimism/optimism/op-service/cliapp"
 	"github.com/ethereum-optimism/optimism/op-service/ctxinterrupt"
 	oplog "github.com/ethereum-optimism/optimism/op-service/log"
 )
 
 var (
-	Version   = "v0.0.1"
+	Version   = "v0.1.0"
 	GitCommit = ""
 	GitDate   = ""
 )
 
 func main() {
 	app := cli.NewApp()
-	app.Flags = cliapp.ProtectFlags(flags.Flags)
 	app.Version = fmt.Sprintf("%s-%s-%s", Version, GitCommit, GitDate)
-	app.Name = "op-nat"
+	app.Name = "op-acceptor"
 	app.Usage = "Optimism Network Acceptance Tester Service"
-	app.Description = "op-nat tests networks"
+	app.Description = "op-acceptor tests networks"
+	app.Flags = cliapp.ProtectFlags(flags.Flags)
 	app.Action = cliapp.LifecycleCmd(run)
 
 	// Start server
@@ -51,22 +50,24 @@ func run(ctx *cli.Context, closeApp context.CancelCauseFunc) (cliapp.Lifecycle,
 	oplog.SetGlobalLogHandler(log.Handler())
 	oplog.SetupDefaults()
 
-	// TODO: map validators from flags
-	var validators = []nat.Validator{
-		gates.Alphanet,
-	}
-
-	cfg, err := nat.NewConfig(ctx, log, validators)
+	// Initialize the service with both paths
+	cfg, err := nat.NewConfig(
+		ctx,
+		log,
+		ctx.String(flags.TestDir.Name),
+		ctx.String(flags.ValidatorConfig.Name),
+		ctx.String(flags.Gate.Name),
+	)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create config: %w", err)
 	}
 
 	cfg.Log.Debug("Config", "config", cfg)
 
-	c, err := nat.New(ctx.Context, cfg, Version)
+	natService, err := nat.New(ctx.Context, cfg, Version)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create nat: %w", err)
 	}
 
-	return c, nil
+	return natService, nil
 }