Generate gogo and protobuf protos; give them distinct canonical packa…

…ge names (#27)

* All-In on Gogo means using (gogoproto.nullable) = false and (gogoproto.stdtime) for time fields

* And gogoproto.stdduration

* Update .gitignore

* Add placeholder to workaround golang/go#8279

* Working on test

* Use a vendor dir so the test can compile as a submodule

* Rename placeholder

* Remove TODOs

* Update .gitignore

* Add dependencies for the new test

* Cleanup protobuf dependency as much as possible

* aNeed this for imports

* Use a make rule

* Get the test to run

* Use new Makefile rules

* Include fakes

* Again...

* Use GOPATH-rooted includes

* Remap the import paths for LS protos

* Links working

* Add -f

* Rewrite .protos instead of link them

* Store translated copies of the .protos

* Remove stdduration feature

* Use int64 for micros consistently

* Get the Makefile from the docker iamge

* Revert collector.proto changes

* Restore lightstep.proto

* Regenerate protos

* Revert grpc version

* Fix

* Add the generated file

* Support multi-part GOPATH

* Fix

* Debug

* Test

* Uhhhhgggg

* Run dep

* Let's see

* Or this

* Try again

* Add removed files for benefit of non-Go builds

.gitignore

@@ -1,4 +1,3 @@
 .idea
 vendor
-collectorpb
-lightsteppb
+tmpgen

Gopkg.toml

@@ -0,0 +1,19 @@
+[[constraint]]
+  name = "github.com/gogo/protobuf"
+  version = "0.5.0"
+
+[[constraint]]
+  branch = "master"
+  name = "github.com/golang/protobuf"
+
+[[constraint]]
+  branch = "master"
+  name = "golang.org/x/net"
+
+[[constraint]]
+  branch = "master"
+  name = "google.golang.org/genproto"
+
+[[constraint]]
+  name = "google.golang.org/grpc"
+  version = "1.4.3"

Makefile

@@ -1,26 +1,64 @@
-.PHONY: default build test
-
 default: build
 
-build: clean-proto proto
+genproto.mk:
+	@docker pull lightstep/gogoprotoc:latest
+	@-docker rm -v lightstep-get-genproto-mk
+	@docker create --name lightstep-get-genproto-mk lightstep/gogoprotoc:latest
+	@docker cp lightstep-get-genproto-mk:/root/genproto.mk genproto.mk
+	@docker rm -v lightstep-get-genproto-mk
+
+include genproto.mk
+
+PKG_PREFIX = github.com/lightstep/lightstep-tracer-common
+
+PROTO_SOURCES = \
+	collector.proto \
+	lightstep.proto
+
+TEST_SOURCES = \
+	$(GOLANG)/gogo_test.go \
+	$(GOLANG)/protobuf_test.go
+
+GOGO_GENTGTS = $(call protos_to_gogo_targets,$(PROTO_SOURCES))
+PBUF_GENTGTS = $(call protos_to_protobuf_targets,$(PROTO_SOURCES))
+
+GOGO_LINKS  = $(call protoc_targets_to_link_targets,$(GOGO_GENTGTS))
+PBUF_LINKS  = $(call protoc_targets_to_link_targets,$(PBUF_GENTGTS))
+
+FAKES = \
+	golang/gogo/collectorpb/collectorpbfakes/fake_collector_service_client.go \
+	golang/protobuf/collectorpb/collectorpbfakes/fake_collector_service_client.go
+
+.PHONY: default build test clean proto-links proto
+.PHONY: $(GOGO_GENTGTS) $(PBUF_GENTGTS) $(GOGO_LINKS) $(PBUF_LINKS)
+
+build: test
+
+proto: $(GOGO_GENTGTS) $(PBUF_GENTGTS) $(FAKES) 
+
+test: $(TEST_SOURCES)
+	$(GOPATH)/bin/dep ensure && $(GOPATH)/bin/dep prune
+	go test -v ./golang
+
+clean: 
+	$(call clean_protoc_targets,$(GOGO_GENTGTS) $(PBUF_GENTGTS))
 
-test: build
+proto-links: $(GOGO_LINKS) $(PBUF_LINKS)
 
-PROTO_GEN = lightsteppb/lightstep_carrier.pb.go collectorpb/collector.pb.go
+$(GOGO_LINKS): $(GOLANG)-$(GOGO)-%-link: %.proto
+	$(call gen_protoc_link,$<,$@,$(GOGO))
 
-.PHONY: proto clean-proto
+$(PBUF_LINKS): $(GOLANG)-$(PBUF)-%-link: %.proto
+	$(call gen_protoc_link,$<,$@,$(PBUF))
 
-clean-proto:
-	@rm -f $(PROTO_GEN)
+$(GOGO_GENTGTS): $(GOLANG)-$(GOGO)-%: %.proto proto-links
+	$(call gen_gogo_target,$<)
 
-proto: $(PROTO_GEN)
+$(PBUF_GENTGTS): $(GOLANG)-$(PBUF)-%: %.proto proto-links
+	$(call gen_protobuf_target,$<)
 
-collectorpb/collector.pb.go: collector.proto
-	docker run --rm -v $(shell pwd):/input:ro -v $(shell pwd)/collectorpb:/output \
-	  lightstep/grpc-gateway:latest \
-		protoc -I/root/go/src/tmp/vendor/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis --go_out=plugins=grpc:/output --proto_path=/input /input/collector.proto
+golang/gogo/collectorpb/collectorpbfakes/fake_collector_service_client.go: golang/gogo/collectorpb/collector.pb.go
+	$(call generate_fake,$@,$<,CollectorServiceClient)
 
-lightsteppb/lightstep_carrier.pb.go: lightstep_carrier.proto
-	docker run --rm -v $(shell pwd):/input:ro -v $(shell pwd)/lightsteppb:/output \
-	  lightstep/protoc:latest \
-	  protoc --go_out=plugins=grpc:/output --proto_path=/input /input/lightstep_carrier.proto
+golang/protobuf/collectorpb/collectorpbfakes/fake_collector_service_client.go: golang/protobuf/collectorpb/collector.pb.go
+	$(call generate_fake,$@,$<,CollectorServiceClient)

circle.yml

@@ -1,6 +1,28 @@
 machine:
   services:
     - docker
+  environment:
+    GOPATH: ${HOME}/go
+    BUILD_DIR: ${GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}
+    GOOGLE_APPLICATION_CREDENTIALS: ${HOME}/gcloud-service-key.json
+    HELM_INSTALL_DIR: ${HOME}/bin
+
+dependencies:
+  pre:
+
+dependencies:
+  pre:
+    # Move the build target into its proper Go place
+    - mkdir -p ${GOPATH}/src/github.com/${CIRCLE_PROJECT_USERNAME}
+    - mv ${HOME}/${CIRCLE_PROJECT_REPONAME} ${BUILD_DIR}
+    # Symlink back to make CircleCI tools happier
+    - ln -s ${BUILD_DIR} ${HOME}/${CIRCLE_PROJECT_REPONAME}
+    - go get github.com/golang/dep/...
+    - go install github.com/golang/dep/...
+  override:
+    - make -C ${BUILD_DIR} build
+  cache_directories:
+    - ~/bin
 
 test:
   override:

genproto.mk

@@ -0,0 +1,98 @@
+# To include this protoc/make snippet, copy this to the top of your Makefile:
+#
+# default: build
+#
+# genproto.mk:
+# 	@docker pull lightstep/gogoprotoc:latest
+# 	@-docker rm -v lightstep-get-genproto-mk
+# 	@docker create --name lightstep-get-genproto-mk lightstep/gogoprotoc:latest
+# 	@docker cp lightstep-get-genproto-mk:/genproto.mk genproto.mk
+# 	@docker rm -v lightstep-get-genproto-mk
+#
+# include genproto.mk
+
+GOLANG = golang
+PBUF   = protobuf
+GOGO   = gogo
+
+# Use the final GOPATH element, since that's where circleci puts the code (lame!)
+ROOT = $(shell echo ${GOPATH} | tr : \\n | tail -1)
+
+PWD     = $(shell pwd)
+TMPNAME = tmpgen
+TMPDIR  = $(PWD)/$(TMPNAME)
+
+# List of standard protoc options
+PROTOC_OPTS = plugins=grpc
+
+# These flags manage mapping the google-standard protobuf types (e.g., Timestamp)
+# into the annotated versions supplied with Gogo.  The trailing `,` matters.
+GOGO_OPTS = Mgoogle/protobuf/timestamp.proto=github.com/gogo/protobuf/types,
+
+define protos_to_gogo_targets
+$(foreach proto,$(1),$(GOLANG)-$(GOGO)-$(basename $(proto)))
+endef
+
+define protos_to_protobuf_targets
+$(foreach proto,$(1),$(GOLANG)-$(PBUF)-$(basename $(proto)))
+endef
+
+define gen_gogo_target
+$(call gen_protoc_target,$(1),$(GOLANG)/$(GOGO)/$(basename $(1))pb/$(basename $(1)).pb.go,$(GOGO),--gogofaster_out=$(GOGO_OPTS)$(PROTOC_OPTS))
+endef
+
+define gen_protobuf_target
+$(call gen_protoc_target,$(1),$(GOLANG)/$(PBUF)/$(basename $(1))pb/$(basename $(1)).pb.go,$(PBUF),--go_out=$(PROTOC_OPTS))
+endef
+
+define protoc_targets_to_link_targets
+$(foreach target,$(1),$(target)-link)
+endef
+
+define gen_protoc_link
+@mkdir -p "$(subst -,/,$(subst -link,,$(2)))pb"
+@echo "// DO NOT EDIT; THIS FILE IS AUTOGENERATED FROM ../../../$(1)" > $(subst -,/,$(subst -link,,$(2)))pb/$(1)
+@sed -E 's@import "github.com/lightstep/([^/]+)/(.*).proto"@import "github.com/lightstep/\1/$(GOLANG)/$(3)/\2pb/\2.proto"@g' < $(1) >> $(subst -,/,$(subst -link,,$(2)))pb/$(1)
+endef
+
+# $(1) = .proto input
+# $(2) = .pb.go output
+# $(3) = gogo or protobuf
+# $(4) = protoc-output spec
+#
+# Note: the --proto_path include "." below references the
+# docker image's $(ROOT)/src.  /input is mapped to the
+# host's $(ROOT)/src.
+define gen_protoc_target
+  @echo compiling $(1) [$(3)]
+  @mkdir -p $(TMPDIR)
+  @sed -E 's@import "github.com/lightstep/([^/]+)/(.*).proto"@import "github.com/lightstep/\1/$(GOLANG)/$(3)/\2pb/\2.proto"@g' < $(1) > $(TMPDIR)/$(1)
+  @docker run --rm \
+    -v $(ROOT)/src:/input:ro \
+    -v $(TMPDIR):/output \
+    lightstep/gogoprotoc:latest \
+    protoc \
+    -I./github.com/google/googleapis \
+    $(4):/output \
+    --proto_path=/input:. \
+    /input/$(PKG_PREFIX)/$(TMPNAME)/$(1)
+  @mkdir -p $(GOLANG)/$(3)/$(basename $(1))pb/$(basename $(1))pbfakes
+  @sed 's@package $(basename $(1))pb@package $(basename $(1))pb // import "$(PKG_PREFIX)/golang/$(3)/$(basename $(1))pb"@' < $(TMPDIR)/$(PKG_PREFIX)/$(TMPNAME)/$(basename $(1)).pb.go > $(GOLANG)/$(3)/$(basename $(1))pb/$(basename $(1)).pb.go
+  @rm $(TMPDIR)/$(PKG_PREFIX)/$(TMPNAME)/$(basename $(1)).pb.go
+  @rm $(TMPDIR)/$(1)
+endef
+
+define clean_protoc_targets
+  @rm -rf $(foreach target,$(1),$(subst -,/,$(target)pb))
+endef
+
+# generate_fake: runs counterfeiter in docker container to generate fake classes
+# $(1) output file path
+# $(2) input file path
+# $(3) class name
+define generate_fake
+  @docker run --rm \
+	-v $(ROOT):/usergo \
+	lightstep/gobuild:latest \
+	/bin/bash -c "cd /usergo/src/$(PKG_PREFIX) && counterfeiter -o $(1) $(2) $(3)"
+endef