From bdeafd6a196c7683b8aa2cbbd13431ea7d3ee9d5 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Mon, 6 May 2019 13:52:11 -0400 Subject: [PATCH 1/4] Add multistage build --- Dockerfile.build | 18 ++++++++++++++++++ cmd/mc-router/main.go | 2 +- {mcproto => pkg/mcproto}/types.go | 0 {server => pkg/server}/api_server.go | 0 {server => pkg/server}/connector.go | 2 +- {server => pkg/server}/k8s.go | 0 {server => pkg/server}/routes.go | 0 {server => pkg/server}/routes_test.go | 0 8 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 Dockerfile.build rename {mcproto => pkg/mcproto}/types.go (100%) rename {server => pkg/server}/api_server.go (100%) rename {server => pkg/server}/connector.go (98%) rename {server => pkg/server}/k8s.go (100%) rename {server => pkg/server}/routes.go (100%) rename {server => pkg/server}/routes_test.go (100%) diff --git a/Dockerfile.build b/Dockerfile.build new file mode 100644 index 0000000..ea14e4b --- /dev/null +++ b/Dockerfile.build @@ -0,0 +1,18 @@ +FROM golang:1.12 AS builder + +ENV CGO_ENABLED=0 + +WORKDIR /build + +COPY go.mod /build/go.mod +RUN go mod download + +COPY cmd /build/cmd +COPY pkg /build/pkg +RUN go build -o mc-router cmd/mc-router/main.go + +FROM scratch + +COPY --from=builder /build/mc-router /bin/mc-router + +ENTRYPOINT ["/bin/mc-router"] diff --git a/cmd/mc-router/main.go b/cmd/mc-router/main.go index edf0454..e37e80c 100644 --- a/cmd/mc-router/main.go +++ b/cmd/mc-router/main.go @@ -4,7 +4,7 @@ import ( "context" "flag" "fmt" - "github.com/itzg/mc-router/server" + "github.com/itzg/mc-router/pkg/server" "github.com/sirupsen/logrus" "net" "os" diff --git a/mcproto/types.go b/pkg/mcproto/types.go similarity index 100% rename from mcproto/types.go rename to pkg/mcproto/types.go diff --git a/server/api_server.go b/pkg/server/api_server.go similarity index 100% rename from server/api_server.go rename to pkg/server/api_server.go diff --git a/server/connector.go b/pkg/server/connector.go similarity index 98% rename from server/connector.go rename to pkg/server/connector.go index f8130e9..f36b8dc 100644 --- a/server/connector.go +++ b/pkg/server/connector.go @@ -3,7 +3,7 @@ package server import ( "net" "github.com/sirupsen/logrus" - "github.com/itzg/mc-router/mcproto" + "github.com/itzg/mc-router/pkg/mcproto" "context" "io" "bytes" diff --git a/server/k8s.go b/pkg/server/k8s.go similarity index 100% rename from server/k8s.go rename to pkg/server/k8s.go diff --git a/server/routes.go b/pkg/server/routes.go similarity index 100% rename from server/routes.go rename to pkg/server/routes.go diff --git a/server/routes_test.go b/pkg/server/routes_test.go similarity index 100% rename from server/routes_test.go rename to pkg/server/routes_test.go From 913162fdd467aa4fd196c75f2a410cd7cd04b96e Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Mon, 6 May 2019 14:03:58 -0400 Subject: [PATCH 2/4] add molecule tests --- molecule/default/asserts.yml | 44 +++++++++++++++++ molecule/default/molecule.yml | 54 ++++++++++++++++++++ molecule/default/playbook.yml | 81 ++++++++++++++++++++++++++++++ molecule/default/prepare.yml | 93 +++++++++++++++++++++++++++++++++++ molecule/default/verify.yml | 48 ++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 molecule/default/asserts.yml create mode 100644 molecule/default/molecule.yml create mode 100644 molecule/default/playbook.yml create mode 100644 molecule/default/prepare.yml create mode 100644 molecule/default/verify.yml diff --git a/molecule/default/asserts.yml b/molecule/default/asserts.yml new file mode 100644 index 0000000..6f70e04 --- /dev/null +++ b/molecule/default/asserts.yml @@ -0,0 +1,44 @@ +--- + +- name: Verify + hosts: localhost + connection: local + vars: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" + tasks: + - block: + - name: Assert that an address has been set on the custom resource + assert: + that: custom_resource.0.status.address is defined + vars: + custom_resource: "{{ q('k8s', + api_version='games.fabianism.us/v1alpha1', + kind='Minecraft', + resource_name='example-minecraft', + namespace=namespace + )}}" + + - name: Store address + set_fact: + address: "{{ q('k8s', + api_version='games.fabianism.us/v1alpha1', + kind='Minecraft', + resource_name='example-minecraft', + namespace=namespace + ).0.status.address}}" + rescue: + - name: Output CR + debug: var=custom_resource + vars: + custom_resource: "{{ q('k8s', + api_version='games.fabianism.us/v1alpha1', + kind='Minecraft', + resource_name='example-minecraft', + namespace=namespace + )}}" + + - fail: + msg: "Failed in asserts.yml" + +- import_playbook: '{{ playbook_dir }}/../default/asserts.yml' diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml new file mode 100644 index 0000000..08721d9 --- /dev/null +++ b/molecule/default/molecule.yml @@ -0,0 +1,54 @@ +--- +dependency: + name: galaxy +driver: + name: docker +lint: + name: yamllint + enabled: False +platforms: +- name: kind-test-local + groups: + - k8s + image: bsycorp/kind:latest-1.12 + privileged: True + override_command: no + exposed_ports: + - 8443/tcp + - 10080/tcp + published_ports: + - 0.0.0.0:${TEST_CLUSTER_PORT:-10443}:8443/tcp + pre_build_image: yes + volumes: + - ${MOLECULE_PROJECT_DIRECTORY}:/src/mc-router:Z +provisioner: + name: ansible + log: True + lint: + name: ansible-lint + enabled: False + inventory: + group_vars: + all: + namespace: ${TEST_NAMESPACE:-osdk-test} + env: + K8S_AUTH_KUBECONFIG: /tmp/molecule/kind-test-local/kubeconfig + KUBECONFIG: /tmp/molecule/kind-test-local/kubeconfig + KIND_PORT: '${TEST_CLUSTER_PORT:-10443}' +scenario: + name: default + test_sequence: + - lint + - destroy + - dependency + - syntax + - create + - prepare + - converge + - side_effect + - verify + - destroy +verifier: + name: testinfra + lint: + name: flake8 diff --git a/molecule/default/playbook.yml b/molecule/default/playbook.yml new file mode 100644 index 0000000..15f78e5 --- /dev/null +++ b/molecule/default/playbook.yml @@ -0,0 +1,81 @@ +--- + +- name: Build Controller in Kubernetes docker container + hosts: k8s + vars: + image_name: games.fabianism.us/mc-router:testing + tasks: + # using command so we don't need to install any dependencies + - name: Get existing image hash + command: docker images -q {{image_name}} + register: prev_hash + changed_when: false + + - name: Build Controller Image + command: docker build -f /src/mc-router/Dockerfile.build -t {{ image_name }} /src/mc-router + register: build_cmd + changed_when: not prev_hash.stdout or (prev_hash.stdout and prev_hash.stdout not in ''.join(build_cmd.stdout_lines[-2:])) + +- name: Converge + hosts: localhost + connection: local + vars: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + image_name: games.fabianism.us/mc-router:testing + tasks: + - block: + - name: Delete the Operator Deployment + k8s: + state: absent + namespace: '{{ namespace }}' + name: mc-router + api_version: apps/v1 + kind: Deployment + register: delete_deployment + when: hostvars[groups.k8s.0].build_cmd.changed + + - name: Wait 30s for Operator Deployment to terminate + k8s_facts: + api_version: 'apps/v1' + kind: 'Deployment' + namespace: '{{ namespace }}' + name: 'mc-router' + register: deployment + until: not deployment.resources + delay: 3 + retries: 10 + when: delete_deployment.changed + + - name: Create the Operator Deployment + k8s: + namespace: '{{ namespace }}' + definition: + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + run: mc-router + name: mc-router + spec: + selector: + matchLabels: + run: mc-router + strategy: + type: Recreate + template: + metadata: + labels: + run: mc-router + spec: + serviceAccountName: mc-router + containers: + - image: '{{ image_name }}' + name: mc-router + args: ["--api-binding", ":8080", "--in-kube-cluster"] + ports: + - name: proxy + containerPort: 25565 + - name: web + containerPort: 8080 + +- import_playbook: verify.yml diff --git a/molecule/default/prepare.yml b/molecule/default/prepare.yml new file mode 100644 index 0000000..7bdc392 --- /dev/null +++ b/molecule/default/prepare.yml @@ -0,0 +1,93 @@ +--- + +- name: Prepare + hosts: k8s + gather_facts: no + vars: + kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" + tasks: + - name: delete the kubeconfig if present + file: + path: '{{ kubeconfig }}' + state: absent + delegate_to: localhost + + - name: Fetch the kubeconfig + fetch: + dest: '{{ kubeconfig }}' + flat: yes + src: /root/.kube/config + + - name: Change the kubeconfig port to the proper value + replace: + regexp: 8443 + replace: "{{ lookup('env', 'KIND_PORT') }}" + path: '{{ kubeconfig }}' + delegate_to: localhost + + - name: Wait for the Kubernetes API to become available (this could take a minute) + uri: + url: "https://localhost:8443/apis" + status_code: 200 + validate_certs: no + register: result + until: (result.status|default(-1)) == 200 + retries: 60 + delay: 5 + +- name: Prepare operator resources + hosts: localhost + connection: local + vars: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + deploy_dir: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') }}/deploy" + tasks: + - name: Ensure specified namespace is present + k8s: + api_version: v1 + kind: Namespace + name: '{{ namespace }}' + + - name: Create RBAC resources and Service + k8s: + namespace: '{{ namespace }}' + definition: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: mc-router + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: services-watcher + rules: + - apiGroups: [""] + resources: ["services"] + verbs: ["watch","list"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: mc-router-services-watcher + subjects: + - kind: ServiceAccount + name: mc-router + namespace: '{{ namespace }}' + roleRef: + kind: ClusterRole + name: services-watcher + apiGroup: rbac.authorization.k8s.io + - apiVersion: v1 + kind: Service + metadata: + name: mc-router + spec: + type: NodePort + ports: + - targetPort: web + name: web + port: 8080 + - targetPort: proxy + name: proxy + port: 25565 + selector: + run: mc-router diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml new file mode 100644 index 0000000..ef97b2e --- /dev/null +++ b/molecule/default/verify.yml @@ -0,0 +1,48 @@ +--- + +- hosts: localhost + connection: local + gather_facts: no + vars: + ansible_python_interpreter: '{{ ansible_playbook_python }}' + tasks: + - name: Wait for minecraft deployment to report ready + k8s_facts: + api_version: 'apps/v1' + kind: 'Deployment' + namespace: '{{ namespace }}' + label_selectors: + - 'app=example-minecraft' + register: deployment + until: "'MinimumReplicasAvailable' in (deployment | json_query('resources[].status.conditions[].reason'))" + delay: 10 + retries: 60 + + - set_fact: + address: 172.17.0.2:{{ node_port }} + vars: + node_port: "{{ q('k8s', + api_version='v1', + kind='Service', + namespace=namespace, + resource_name='mc-router' + ).0.spec.ports.1.nodePort }}" + + - debug: var=address + + - name: Wait two minutes to confirm server is reachable + command: 'mcstatus {{ address }} ping' + changed_when: false + retries: 24 + delay: 5 + register: result + until: not (result is failed) + + - name: Fetch status of server + command: 'mcstatus {{ address }} status' + changed_when: false + register: mcstatus + + - name: Output status of server + debug: + var: mcstatus.stdout_lines From be75e8ed7a44dca1df58398dcaf4b24c2b3a012a Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Mon, 6 May 2019 14:04:06 -0400 Subject: [PATCH 3/4] Try to sniff every packet --- pkg/mcproto/types.go | 9 +++++++++ pkg/server/connector.go | 38 ++++++++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/pkg/mcproto/types.go b/pkg/mcproto/types.go index e8d5d16..9679ef2 100644 --- a/pkg/mcproto/types.go +++ b/pkg/mcproto/types.go @@ -5,6 +5,8 @@ import ( "errors" "io" "strings" + + "github.com/sirupsen/logrus" ) type Frame struct { @@ -38,6 +40,12 @@ func ReadVarInt(reader io.Reader) (int, error) { for numRead <= 5 { n, err := reader.Read(b) if err != nil { + logrus.WithError(err).WithFields(logrus.Fields{ + "b": b, + "numRead": numRead, + "result": result, + "n": n, + }).Infof("") return 0, err } if n == 0 { @@ -131,6 +139,7 @@ func ReadPacket(reader io.Reader) (*Packet, error) { packet.PacketID, err = ReadVarInt(remainder) if err != nil { + logrus.WithField("remainder", remainder.String()).Info("Failed to find PacketID") return nil, err } diff --git a/pkg/server/connector.go b/pkg/server/connector.go index f36b8dc..5c9c607 100644 --- a/pkg/server/connector.go +++ b/pkg/server/connector.go @@ -1,12 +1,13 @@ package server import ( - "net" - "github.com/sirupsen/logrus" - "github.com/itzg/mc-router/pkg/mcproto" + "bytes" "context" "io" - "bytes" + "net" + + "github.com/itzg/mc-router/pkg/mcproto" + "github.com/sirupsen/logrus" ) type IConnector interface { @@ -136,9 +137,30 @@ func pumpConnections(ctx context.Context, frontendConn, backendConn net.Conn) { } func pumpFrames(incoming io.Reader, outgoing io.Writer, errors chan<- error, from, to string) { - amount, err := io.Copy(outgoing, incoming) - if err != nil { - errors <- err + for { + inspectionBuffer := new(bytes.Buffer) + + inspectionReader := io.TeeReader(incoming, inspectionBuffer) + + packet, err := mcproto.ReadPacket(inspectionReader) + if err != nil { + logrus.WithError(err).Error("Failed to read packet") + errors <- err + continue + } + amount, err := io.Copy(outgoing, inspectionBuffer) + if err != nil { + errors <- err + continue + } + logrus.WithFields(logrus.Fields{ + "PacketID": packet.PacketID, + "PacketLength": packet.Length, + "from": from, + "to": to, + "amount": amount, + }).Info("Proxied packet") } - logrus.WithField("amount", amount).Infof("Finished relay %s->%s", from, to) + + logrus.Infof("Finished relay %s->%s", from, to) } From 83c15fbf5d8c53e7778fd64278dff6183ac7bb05 Mon Sep 17 00:00:00 2001 From: Fabian von Feilitzsch Date: Tue, 7 May 2019 11:06:42 -0400 Subject: [PATCH 4/4] only end when EOF is reached --- pkg/server/connector.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/server/connector.go b/pkg/server/connector.go index 5c9c607..3ceb658 100644 --- a/pkg/server/connector.go +++ b/pkg/server/connector.go @@ -124,11 +124,10 @@ func pumpConnections(ctx context.Context, frontendConn, backendConn net.Conn) { for { select { case err := <-errors: - if err != io.EOF { - logrus.WithError(err).Error("Error observed on connection relay") + if err == io.EOF { + return } - - return + logrus.WithError(err).Error("Error observed on connection relay") case <-ctx.Done(): return