Merge branch 'main' into release/v0.3.0

.github/workflows/website.yml

@@ -26,7 +26,6 @@ jobs:
       with:
         # ./tmp is cached for htmltest
         path: |
-          ./site/bin
           ./site/tmp
         key: ${{ runner.os }}-${{ hashFiles('./Makefile') }}
 

.gitignore

@@ -35,5 +35,4 @@ external/*
 e2e/log
 .idea
 # helm charts
-manifests/charts/**/*.tgz
-.idea
+manifests/charts/**/*.tgz

.licenserc.yaml

@@ -51,3 +51,5 @@ dependency:
   excludes:
     - name: github.com/dgryski/trifles # the author adds MIT License later
     - name: github.com/rcrowley/go-metrics # this repo uses BSD-2-Clause-Views, which is not recognized in skywalking-eye 0.6
+    - name: github.com/kr/logfmt # the author adds MIT License later
+    - name: github.com/pascaldekloe/goe # use CC0-1.0 License

Makefile

@@ -178,8 +178,9 @@ fix-cjk: dev-tools
 .PHONY: lint-website
 lint-website: $(LOCALBIN)
 	test -x $(LOCALBIN)/htmltest || GOBIN=$(LOCALBIN) go install github.com/wjdp/[email protected]
-	$(LOCALBIN)/htmltest --conf ./.htmltest.yml ./public | grep  -E '(target does not exist|Non-OK status: 404)' \
-		&& exit 1 || true
+	cd ./site && $(LOCALBIN)/htmltest --conf ./.htmltest.yml ./public > /tmp/htmltest.log || true
+	@# ignore 'lookup htnn.mosn.io: no such host' error for now
+	test -f /tmp/htmltest.log && (grep -E '(target does not exist|Non-OK status: 404)' /tmp/htmltest.log && exit 1 || true)
 
 .PHONY: lint-remain
 lint-remain:

controller/go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/envoyproxy/go-control-plane v0.12.1-0.20240326194405-485b2263e153
 	github.com/go-logr/logr v1.4.1
 	github.com/go-logr/zapr v1.3.0
+	github.com/hashicorp/consul/api v1.29.2
 	github.com/nacos-group/nacos-sdk-go v1.1.4
 	github.com/onsi/ginkgo/v2 v2.17.2
 	github.com/onsi/gomega v1.33.0
@@ -53,6 +54,7 @@ require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.18 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -62,6 +64,7 @@ require (
 	github.com/envoyproxy/envoy v1.29.4 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
 	github.com/evanphx/json-patch/v5 v5.8.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
@@ -81,13 +84,24 @@ require (
 	github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

controller/registries/consul/config.go

@@ -0,0 +1,180 @@
+// Copyright The HTNN Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import (
+	"fmt"
+	"net/url"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	consulapi "github.com/hashicorp/consul/api"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"mosn.io/htnn/controller/pkg/registry"
+	"mosn.io/htnn/controller/pkg/registry/log"
+	registrytype "mosn.io/htnn/types/pkg/registry"
+	"mosn.io/htnn/types/registries/consul"
+)
+
+func init() {
+	registry.AddRegistryFactory(consul.Name, func(store registry.ServiceEntryStore, om metav1.ObjectMeta) (registry.Registry, error) {
+		reg := &Consul{
+			logger: log.NewLogger(&log.RegistryLoggerOptions{
+				Name: om.Name,
+			}),
+			store:               store,
+			name:                om.Name,
+			softDeletedServices: map[consulService]bool{},
+			done:                make(chan struct{}),
+		}
+		return reg, nil
+	})
+}
+
+const (
+	defaultToken = ""
+)
+
+type Consul struct {
+	consul.RegistryType
+	logger log.RegistryLogger
+	store  registry.ServiceEntryStore
+	name   string
+	client *Client
+
+	lock                sync.RWMutex
+	watchingServices    map[consulService]bool
+	softDeletedServices map[consulService]bool
+
+	done    chan struct{}
+	stopped atomic.Bool
+}
+
+type Client struct {
+	consulClient  *consulapi.Client
+	consulCatalog *consulapi.Catalog
+
+	DataCenter string
+	NameSpace  string
+}
+
+type consulService struct {
+	DataCenter  string
+	ServiceName string
+}
+
+func (reg *Consul) NewClient(config *consul.Config) (*Client, error) {
+	uri, err := url.Parse(config.ServerUrl)
+	if err != nil {
+		return nil, fmt.Errorf("invalid server url: %s", config.ServerUrl)
+	}
+	clientConfig := consulapi.DefaultConfig()
+	clientConfig.Address = uri.Host
+	clientConfig.Scheme = uri.Scheme
+	clientConfig.Token = defaultToken
+	clientConfig.Datacenter = config.DataCenter
+
+	client, err := consulapi.NewClient(clientConfig)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create Consul client, err: %v", err)
+	}
+
+	return &Client{
+		consulClient:  client,
+		consulCatalog: client.Catalog(),
+		DataCenter:    config.DataCenter,
+	}, nil
+}
+
+func (reg *Consul) Start(c registrytype.RegistryConfig) error {
+	config := c.(*consul.Config)
+
+	client, err := reg.NewClient(config)
+	if err != nil {
+		return err
+	}
+
+	services, err := reg.fetchAllServices(client)
+	if err != nil {
+		return fmt.Errorf("fetch all services error: %v", err)
+	}
+	reg.client = client
+
+	reg.watchingServices = services
+
+	dur := 30 * time.Second
+	if config.ServiceRefreshInterval != nil {
+		dur = config.ServiceRefreshInterval.AsDuration()
+	}
+	go func() {
+		reg.logger.Infof("start refreshing services")
+		ticker := time.NewTicker(dur)
+		//q := consulapi.QueryOptions{
+		//    WaitTime: dur,
+		//}
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				err := reg.refresh()
+				if err != nil {
+					reg.logger.Errorf("failed to refresh services, err: %v", err)
+				}
+			case <-reg.done:
+				reg.logger.Infof("stop refreshing services")
+				return
+			}
+		}
+	}()
+
+	return nil
+}
+
+func (reg *Consul) Stop() error {
+	close(reg.done)
+	reg.stopped.Store(true)
+	reg.logger.Infof("stopped Consul registry")
+
+	reg.lock.Lock()
+	defer reg.lock.Unlock()
+
+	return nil
+}
+
+func (reg *Consul) Reload(c registrytype.RegistryConfig) error {
+	fmt.Println(c)
+	return nil
+}
+
+func (reg *Consul) refresh() error {
+	return nil
+}
+
+func (reg *Consul) fetchAllServices(client *Client) (map[consulService]bool, error) {
+	fmt.Println(client)
+	return nil, nil
+}
+
+func (reg *Consul) subscribe(serviceName string) error {
+	fmt.Println(serviceName)
+	return nil
+}
+
+func (reg *Consul) unsubscribe(serviceName string) error {
+	fmt.Println(serviceName)
+	return nil
+}

controller/registries/consul/config_test.go

@@ -0,0 +1,85 @@
+// Copyright The HTNN Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"mosn.io/htnn/controller/pkg/registry/log"
+	"mosn.io/htnn/types/registries/consul"
+)
+
+func TestNewClient(t *testing.T) {
+	reg := &Consul{}
+	config := &consul.Config{
+		ServerUrl:  "http://127.0.0.1:8500",
+		DataCenter: "test",
+	}
+	client, err := reg.NewClient(config)
+
+	assert.NoError(t, err)
+	assert.NotNil(t, client)
+
+	config = &consul.Config{
+		ServerUrl:  "::::::::::::",
+		DataCenter: "test",
+	}
+
+	client, err = reg.NewClient(config)
+
+	assert.Error(t, err)
+	assert.Nil(t, client)
+}
+
+func TestStart(t *testing.T) {
+	reg := &Consul{
+		logger: log.NewLogger(&log.RegistryLoggerOptions{
+			Name: "test",
+		}),
+		softDeletedServices: map[consulService]bool{},
+		done:                make(chan struct{}),
+		watchingServices:    map[consulService]bool{},
+	}
+	config := &consul.Config{
+		ServerUrl: "http://127.0.0.1:8500",
+	}
+
+	err := reg.Start(config)
+	assert.NoError(t, err)
+
+	err = reg.subscribe("123")
+	assert.Nil(t, err)
+
+	err = reg.unsubscribe("123")
+	assert.Nil(t, err)
+
+	err = reg.refresh()
+	assert.Nil(t, err)
+
+	err = reg.Stop()
+	assert.Nil(t, err)
+}
+
+func TestReload(t *testing.T) {
+	reg := &Consul{}
+	config := &consul.Config{
+		ServerUrl: "http://127.0.0.1:8500",
+	}
+
+	err := reg.Reload(config)
+	assert.NoError(t, err)
+}

plugins/go.mod

@@ -95,4 +95,5 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
+
 )

types/registries/consul/config.go

@@ -0,0 +1,32 @@
+// Copyright The HTNN Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consul
+
+import "mosn.io/htnn/types/pkg/registry"
+
+const (
+	Name = "consul"
+)
+
+func init() {
+	registry.AddRegistryType(Name, &RegistryType{})
+}
+
+type RegistryType struct {
+}
+
+func (reg *RegistryType) Config() registry.RegistryConfig {
+	return &Config{}
+}