From 43ebd02ba2424a42a07e8a9cc0bd61289c574f85 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Sun, 12 May 2019 13:18:23 -0400 Subject: [PATCH 01/27] removed bootstrapping functionality --- go.mod | 3 -- go.sum | 2 -- pubsub.go | 91 ++++++------------------------------------------------- 3 files changed, 9 insertions(+), 87 deletions(-) diff --git a/go.mod b/go.mod index 2f994da..b03841d 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,12 @@ module github.com/libp2p/go-libp2p-pubsub-router require ( - github.com/ipfs/go-cid v0.0.1 github.com/ipfs/go-datastore v0.0.1 github.com/ipfs/go-ipfs-ds-help v0.0.1 - github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.0.1 github.com/libp2p/go-libp2p-host v0.0.1 github.com/libp2p/go-libp2p-peer v0.0.1 - github.com/libp2p/go-libp2p-peerstore v0.0.1 github.com/libp2p/go-libp2p-pubsub v0.0.1 github.com/libp2p/go-libp2p-record v0.0.1 github.com/libp2p/go-libp2p-routing v0.0.1 diff --git a/go.sum b/go.sum index 6dc1df7..ad67013 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,6 @@ github.com/libp2p/go-libp2p-record v0.0.1 h1:zN7AS3X46qmwsw5JLxdDuI43cH5UYwovKxH github.com/libp2p/go-libp2p-record v0.0.1/go.mod h1:grzqg263Rug/sRex85QrDOLntdFAymLDLm7lxMgU79Q= github.com/libp2p/go-libp2p-routing v0.0.1 h1:hPMAWktf9rYi3ME4MG48qE7dq1ofJxiQbfdvpNntjhc= github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= -github.com/libp2p/go-libp2p-routing-helpers v0.0.1 h1:fl02yxCXoPxt3tIRUC5yxKEIkKfJoZVkAkm+JN97SFE= -github.com/libp2p/go-libp2p-routing-helpers v0.0.1/go.mod h1:zf1ook9HoOQpfnVXrF4gGorkPrGGf1g25vgH5+4SRNU= github.com/libp2p/go-libp2p-routing-helpers v0.0.2 h1:SLX7eDQE8Xo197NwNM/hM7WnH3w6fSGY9+G9HkiYwqQ= github.com/libp2p/go-libp2p-routing-helpers v0.0.2/go.mod h1:zf1ook9HoOQpfnVXrF4gGorkPrGGf1g25vgH5+4SRNU= github.com/libp2p/go-libp2p-secio v0.0.1 h1:CqE/RdsizOwItdgLe632iyft/w0tshDLmZGAiKDcUAI= diff --git a/pubsub.go b/pubsub.go index 67b9de2..6a3229e 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,22 +5,17 @@ import ( "context" "encoding/base64" "errors" - "sync" - "time" - - cid "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" dshelp "github.com/ipfs/go-ipfs-ds-help" - u "github.com/ipfs/go-ipfs-util" logging "github.com/ipfs/go-log" p2phost "github.com/libp2p/go-libp2p-host" peer "github.com/libp2p/go-libp2p-peer" - pstore "github.com/libp2p/go-libp2p-peerstore" pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" routing "github.com/libp2p/go-libp2p-routing" ropts "github.com/libp2p/go-libp2p-routing/options" + "sync" ) var log = logging.Logger("pubsub-valuestore") @@ -31,11 +26,9 @@ type watchGroup struct { } type PubsubValueStore struct { - ctx context.Context - ds ds.Datastore - host p2phost.Host - cr routing.ContentRouting - ps *pubsub.PubSub + ctx context.Context + ds ds.Datastore + ps *pubsub.PubSub // Map of keys to subscriptions. // @@ -64,10 +57,8 @@ func NewPubsubValueStore(ctx context.Context, host p2phost.Host, cr routing.Cont return &PubsubValueStore{ ctx: ctx, - ds: dssync.MutexWrap(ds.NewMapDatastore()), - host: host, // needed for pubsub bootstrap - cr: cr, // needed for pubsub bootstrap - ps: ps, + ds: dssync.MutexWrap(ds.NewMapDatastore()), + ps: ps, subs: make(map[string]*pubsub.Subscription), watching: make(map[string]*watchGroup), @@ -137,7 +128,7 @@ func (p *PubsubValueStore) Subscribe(key string) error { } p.mx.Lock() - existingSub, bootstraped := p.subs[key] + existingSub, _ := p.subs[key] if existingSub != nil { p.mx.Unlock() sub.Cancel() @@ -145,16 +136,11 @@ func (p *PubsubValueStore) Subscribe(key string) error { } p.subs[key] = sub - ctx, cancel := context.WithCancel(p.ctx) - go p.handleSubscription(sub, key, cancel) + go p.handleSubscription(sub, key) p.mx.Unlock() log.Debugf("PubsubResolve: subscribed to %s", key) - if !bootstraped { - // TODO: Deal with publish then resolve case? Cancel behaviour changes. - go bootstrapPubsub(ctx, p.cr, p.host, topic) - } return nil } @@ -287,9 +273,8 @@ func (p *PubsubValueStore) Cancel(name string) (bool, error) { return ok, nil } -func (p *PubsubValueStore) handleSubscription(sub *pubsub.Subscription, key string, cancel func()) { +func (p *PubsubValueStore) handleSubscription(sub *pubsub.Subscription, key string) { defer sub.Cancel() - defer cancel() for { msg, err := sub.Next(p.ctx) @@ -325,61 +310,3 @@ func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { } } } - -// rendezvous with peers in the name topic through provider records -// Note: rendezvous/boostrap should really be handled by the pubsub implementation itself! -func bootstrapPubsub(ctx context.Context, cr routing.ContentRouting, host p2phost.Host, name string) { - // TODO: consider changing this to `pubsub:...` - topic := "floodsub:" + name - hash := u.Hash([]byte(topic)) - rz := cid.NewCidV1(cid.Raw, hash) - - go func() { - err := cr.Provide(ctx, rz, true) - if err != nil { - log.Warningf("bootstrapPubsub: error providing rendezvous for %s: %s", topic, err.Error()) - } - - for { - select { - case <-time.After(8 * time.Hour): - err := cr.Provide(ctx, rz, true) - if err != nil { - log.Warningf("bootstrapPubsub: error providing rendezvous for %s: %s", topic, err.Error()) - } - case <-ctx.Done(): - return - } - } - }() - - rzctx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - - wg := &sync.WaitGroup{} - for pi := range cr.FindProvidersAsync(rzctx, rz, 10) { - if pi.ID == host.ID() { - continue - } - wg.Add(1) - go func(pi pstore.PeerInfo) { - defer wg.Done() - - ctx, cancel := context.WithTimeout(ctx, time.Second*10) - defer cancel() - - err := host.Connect(ctx, pi) - if err != nil { - log.Debugf("Error connecting to pubsub peer %s: %s", pi.ID, err.Error()) - return - } - - // delay to let pubsub perform its handshake - time.Sleep(time.Millisecond * 250) - - log.Debugf("Connected to pubsub peer %s", pi.ID) - }(pi) - } - - wg.Wait() -} From f57d48c477c70c2e8c86139bf41b4d67ad8d384a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 22 May 2019 07:35:20 -0400 Subject: [PATCH 02/27] start pubsub validator changes to implement LWW pubsub --- pubsub.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pubsub.go b/pubsub.go index 6a3229e..3029372 100644 --- a/pubsub.go +++ b/pubsub.go @@ -87,13 +87,15 @@ func (p *PubsubValueStore) isBetter(key string, val []byte) bool { } old, err := p.getLocal(key) - if err != nil { - // If the old one is invalid, the new one is *always* better. - return true + + // Same record is not better + if old != nil && bytes.Equal(old, val) { + return false } - // Same record. Possible DoS vector, should consider failing? - if bytes.Equal(old, val) { + if err != nil { + // If the old one is invalid and is not identical to the new record we assume the new one is better. + // Perhaps we should have levels of invalid (e.g. EOL vs bad formatting) return true } @@ -117,7 +119,7 @@ func (p *PubsubValueStore) Subscribe(key string) error { // record hasn't expired. // // Also, make sure to do this *before* subscribing. - p.ps.RegisterTopicValidator(topic, func(ctx context.Context, _ peer.ID, msg *pubsub.Message) bool { + _ = p.ps.RegisterTopicValidator(topic, func(ctx context.Context, _ peer.ID, msg *pubsub.Message) bool { return p.isBetter(key, msg.GetData()) }) @@ -156,7 +158,7 @@ func (p *PubsubValueStore) getLocal(key string) ([]byte, error) { // If the old one is invalid, the new one is *always* better. if err := p.Validator.Validate(key, val); err != nil { - return nil, err + return val, err } return val, nil } From ff890161a5a23096d884eecdd48f092851286689 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 14 Jun 2019 13:54:00 -0400 Subject: [PATCH 03/27] Implement persistence --temporary --- go.mod | 6 +- go.sum | 3 + pubsub.go | 244 ++++++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 224 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 6b013d6..0d3d888 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,17 @@ module github.com/libp2p/go-libp2p-pubsub-router require ( - github.com/ipfs/go-cid v0.0.2 + github.com/gogo/protobuf v1.2.1 github.com/ipfs/go-datastore v0.0.5 github.com/ipfs/go-ipfs-ds-help v0.0.1 - github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.1.1 github.com/libp2p/go-libp2p-core v0.0.1 + github.com/libp2p/go-libp2p-net v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.0 github.com/libp2p/go-libp2p-record v0.1.0 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-swarm v0.1.0 ) + +replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => ../../libp2p/go-libp2p-pubsub diff --git a/go.sum b/go.sum index 6479b98..df9614c 100644 --- a/go.sum +++ b/go.sum @@ -86,11 +86,14 @@ github.com/libp2p/go-libp2p-core v0.0.1 h1:HSTZtFIq/W5Ue43Zw+uWZyy2Vl5WtF0zDjKN8 github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= +github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= +github.com/libp2p/go-libp2p-net v0.1.0 h1:3t23V5cR4GXcNoFriNoZKFdUZEUDZgUkvfwkD2INvQE= +github.com/libp2p/go-libp2p-net v0.1.0/go.mod h1:R5VZbutk75tkC5YJJS61OCO1NWoajxYjCEV2RoHh3FY= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0 h1:MKh7pRNPHSh1fLPj8u/M/s/napdmeNpoi9BRy9lPN0E= diff --git a/pubsub.go b/pubsub.go index fa7035d..cd8753b 100644 --- a/pubsub.go +++ b/pubsub.go @@ -4,28 +4,31 @@ import ( "bytes" "context" "encoding/base64" - "errors" - "sync" - "time" - + "encoding/binary" + "fmt" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p-net" pubsub "github.com/libp2p/go-libp2p-pubsub" record "github.com/libp2p/go-libp2p-record" - "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" dshelp "github.com/ipfs/go-ipfs-ds-help" - u "github.com/ipfs/go-ipfs-util" logging "github.com/ipfs/go-log" + pb "github.com/libp2p/go-libp2p-pubsub-router/pb" + "io" "sync" + "time" ) var log = logging.Logger("pubsub-valuestore") +const OnJoinProto = protocol.ID("/psOnJoin/0.0.1") + type watchGroup struct { // Note: this chan must be buffered, see notifyWatchers listeners map[chan []byte]struct{} @@ -36,6 +39,11 @@ type PubsubValueStore struct { ds ds.Datastore ps *pubsub.PubSub + host host.Host + + rebroadcastInitialDelay time.Duration + rebroadcastInterval time.Duration + // Map of keys to subscriptions. // // If a key is present but the subscription is nil, we've bootstrapped @@ -60,17 +68,44 @@ func KeyToTopic(key string) string { // The constructor interface is complicated by the need to bootstrap the pubsub topic. // This could be greatly simplified if the pubsub implementation handled bootstrap itself func NewPubsubValueStore(ctx context.Context, host host.Host, cr routing.ContentRouting, ps *pubsub.PubSub, validator record.Validator) *PubsubValueStore { - return &PubsubValueStore{ + psValueStore := &PubsubValueStore{ ctx: ctx, - ds: dssync.MutexWrap(ds.NewMapDatastore()), - ps: ps, + ds: dssync.MutexWrap(ds.NewMapDatastore()), + ps: ps, + host: host, + rebroadcastInitialDelay: 100 * time.Millisecond, + rebroadcastInterval: time.Second, subs: make(map[string]*pubsub.Subscription), watching: make(map[string]*watchGroup), Validator: validator, } + + host.SetStreamHandler(OnJoinProto, func(s net.Stream) { + defer net.FullClose(s) + + msgData, err := readBytes(s) + if err != nil { + return + } + + msg := pb.RequestLatest{} + if msg.Unmarshal(msgData) != nil { + return + } + + response, err := psValueStore.getLocal(*msg.Identifier) + if err != nil { + return + } + + if writeBytes(s, response) != nil { + return + } + }) + return psValueStore } // Publish publishes an IPNS record through pubsub with default TTL @@ -84,29 +119,39 @@ func (p *PubsubValueStore) PutValue(ctx context.Context, key string, value []byt } log.Debugf("PubsubPublish: publish value for key", key) - return p.ps.Publish(topic, value) + done := make(chan error, 1) + go func() { + done <- p.ps.Publish(topic, value) + }() + + select { + case err := <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } } -func (p *PubsubValueStore) isBetter(key string, val []byte) bool { +func (p *PubsubValueStore) isBetter(key string, val []byte) (isBetter bool, isEqual bool) { if p.Validator.Validate(key, val) != nil { - return false + return false, false } old, err := p.getLocal(key) // Same record is not better if old != nil && bytes.Equal(old, val) { - return false + return false, true } if err != nil { // If the old one is invalid and is not identical to the new record we assume the new one is better. // Perhaps we should have levels of invalid (e.g. EOL vs bad formatting) - return true + return true, false } i, err := p.Validator.Select(key, [][]byte{val, old}) - return err == nil && i == 0 + return err == nil && i == 0, false } func (p *PubsubValueStore) Subscribe(key string) error { @@ -125,8 +170,14 @@ func (p *PubsubValueStore) Subscribe(key string) error { // record hasn't expired. // // Also, make sure to do this *before* subscribing. - _ = p.ps.RegisterTopicValidator(topic, func(ctx context.Context, _ peer.ID, msg *pubsub.Message) bool { - return p.isBetter(key, msg.GetData()) + myID := p.host.ID() + _ = p.ps.RegisterTopicValidator(topic, func(ctx context.Context, src peer.ID, msg *pubsub.Message) bool { + isBetter, isEqual := p.isBetter(key, msg.GetData()) + + if src == myID && isEqual { + return true + } + return isBetter }) sub, err := p.ps.Subscribe(topic) @@ -147,11 +198,32 @@ func (p *PubsubValueStore) Subscribe(key string) error { go p.handleSubscription(sub, key) p.mx.Unlock() + go p.rebroadcast(key) log.Debugf("PubsubResolve: subscribed to %s", key) return nil } +func (p *PubsubValueStore) rebroadcast(key string) { + topic := KeyToTopic(key) + time.Sleep(p.rebroadcastInitialDelay) + + ticker := time.NewTicker(p.rebroadcastInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + val, _ := p.getLocal(key) + if val != nil { + _ = p.ps.Publish(topic, val) + } + case <-p.ctx.Done(): + return + } + } +} + func (p *PubsubValueStore) getLocal(key string) ([]byte, error) { val, err := p.ds.Get(dshelp.NewKeyFromBinary([]byte(key))) if err != nil { @@ -268,7 +340,7 @@ func (p *PubsubValueStore) Cancel(name string) (bool, error) { p.watchLk.Lock() if _, wok := p.watching[name]; wok { p.watchLk.Unlock() - return false, errors.New("key has active subscriptions") + return false, fmt.Errorf("key has active subscriptions") } p.watchLk.Unlock() @@ -284,22 +356,92 @@ func (p *PubsubValueStore) Cancel(name string) (bool, error) { func (p *PubsubValueStore) handleSubscription(sub *pubsub.Subscription, key string) { defer sub.Cancel() - for { - msg, err := sub.Next(p.ctx) - if err != nil { - if err != context.Canceled { - log.Warningf("PubsubResolve: subscription error in %s: %s", key, err.Error()) + newMsg := make(chan []byte) + go func() { + for { + data, err := p.handleNewMsgs(sub, key) + if err != nil { + close(newMsg) + return } - return + newMsg <- data + } + }() + + newPeerData := make(chan []byte) + go func() { + for { + data, err := p.handleNewPeer(sub, key) + if err != nil { + close(newPeerData) + return + } + newPeerData <- data + } + }() + + for { + var data []byte + select { + case data = <-newMsg: + case data = <-newPeerData: } - if p.isBetter(key, msg.GetData()) { - err := p.ds.Put(dshelp.NewKeyFromBinary([]byte(key)), msg.GetData()) + + if isBetter, _ := p.isBetter(key, data); isBetter { + err := p.ds.Put(dshelp.NewKeyFromBinary([]byte(key)), data) if err != nil { log.Warningf("PubsubResolve: error writing update for %s: %s", key, err) } - p.notifyWatchers(key, msg.GetData()) + p.notifyWatchers(key, data) + } + } +} + +func (p *PubsubValueStore) handleNewMsgs(sub *pubsub.Subscription, key string) ([]byte, error) { + msg, err := sub.Next(p.ctx) + if err != nil { + if err != context.Canceled { + log.Warningf("PubsubResolve: subscription error in %s: %s", key, err.Error()) + } + return nil, err + } + return msg.GetData(), nil +} + +func (p *PubsubValueStore) handleNewPeer(sub *pubsub.Subscription, key string) ([]byte, error) { + peer, err := sub.NextPeerJoin(p.ctx) + if err != nil { + if err != context.Canceled { + log.Warningf("PubsubNewPeer: subscription error in %s: %s", key, err.Error()) } + return nil, err + } + + peerCtx, _ := context.WithTimeout(p.ctx, time.Second*10) + s, err := p.host.NewStream(peerCtx, peer, OnJoinProto) + if err != nil { + return nil, err + } + defer net.FullClose(s) + + msg := pb.RequestLatest{Identifier: &key} + msgData, err := msg.Marshal() + if err != nil { + return nil, err + } + + if writeBytes(s, msgData) != nil { + return nil, err + } + + s.Close() + + response, err := readBytes(s) + if err != nil { + return nil, err } + + return response, nil } func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { @@ -318,3 +460,51 @@ func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { } } } + +const sizeLengthBytes = 8 + +// readNumBytesFromReader reads a specific number of bytes from a Reader, or returns an error +func readNumBytesFromReader(r io.Reader, numBytes uint64) ([]byte, error) { + data := make([]byte, numBytes) + n, err := io.ReadFull(r, data) + if err != nil { + return data, err + } else if uint64(n) != numBytes { + return data, fmt.Errorf("Could not read full length from stream") + } + return data, nil +} + +func readBytes(r io.Reader) ([]byte, error) { + // Protocol: uint64 MessageLength followed by byte[] MarshalledMessage + + sizeData, err := readNumBytesFromReader(r, sizeLengthBytes) + if err != nil { + return nil, err + } + + size := binary.LittleEndian.Uint64(sizeData) + data, err := readNumBytesFromReader(r, size) + if err != nil { + return nil, err + } + + return data, nil +} + +func writeBytes(w io.Writer, data []byte) error { + size := len(data) + + // Protocol: uint64 MessageLength followed by byte[] MarshalledMessage + sizeData := make([]byte, sizeLengthBytes) + binary.LittleEndian.PutUint64(sizeData, uint64(size)) + + _, err := w.Write(sizeData) + if err != nil { + return err + } + + _, err = w.Write(data) + + return err +} From 90c980a9783cb6e06fcc3d269dd59d983db8693f Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 21 Jun 2019 08:26:05 -0400 Subject: [PATCH 04/27] Added message protobuf that was missing from previous commit --- pb/Makefile | 11 + pb/message.pb.go | 792 +++++++++++++++++++++++++++++++++++++++++++++++ pb/message.proto | 21 ++ 3 files changed, 824 insertions(+) create mode 100644 pb/Makefile create mode 100644 pb/message.pb.go create mode 100644 pb/message.proto diff --git a/pb/Makefile b/pb/Makefile new file mode 100644 index 0000000..eb14b57 --- /dev/null +++ b/pb/Makefile @@ -0,0 +1,11 @@ +PB = $(wildcard *.proto) +GO = $(PB:.proto=.pb.go) + +all: $(GO) + +%.pb.go: %.proto + protoc --proto_path=$(GOPATH)/src:. --gogofast_out=. $< + +clean: + rm -f *.pb.go + rm -f *.go diff --git a/pb/message.pb.go b/pb/message.pb.go new file mode 100644 index 0000000..c477b71 --- /dev/null +++ b/pb/message.pb.go @@ -0,0 +1,792 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: message.proto + +package namesys_pb + +import ( + fmt "fmt" + github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Message struct { + SessionID *int32 `protobuf:"varint,1,req,name=sessionID" json:"sessionID,omitempty"` + Type *int32 `protobuf:"varint,2,opt,name=type" json:"type,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{0} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(m, src) +} +func (m *Message) XXX_Size() int { + return m.Size() +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetSessionID() int32 { + if m != nil && m.SessionID != nil { + return *m.SessionID + } + return 0 +} + +func (m *Message) GetType() int32 { + if m != nil && m.Type != nil { + return *m.Type + } + return 0 +} + +func (m *Message) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +type RequestLatest struct { + Identifier *string `protobuf:"bytes,1,opt,name=identifier" json:"identifier,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RequestLatest) Reset() { *m = RequestLatest{} } +func (m *RequestLatest) String() string { return proto.CompactTextString(m) } +func (*RequestLatest) ProtoMessage() {} +func (*RequestLatest) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{1} +} +func (m *RequestLatest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RequestLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RequestLatest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RequestLatest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RequestLatest.Merge(m, src) +} +func (m *RequestLatest) XXX_Size() int { + return m.Size() +} +func (m *RequestLatest) XXX_DiscardUnknown() { + xxx_messageInfo_RequestLatest.DiscardUnknown(m) +} + +var xxx_messageInfo_RequestLatest proto.InternalMessageInfo + +func (m *RequestLatest) GetIdentifier() string { + if m != nil && m.Identifier != nil { + return *m.Identifier + } + return "" +} + +type SendLatest struct { + Data []byte `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SendLatest) Reset() { *m = SendLatest{} } +func (m *SendLatest) String() string { return proto.CompactTextString(m) } +func (*SendLatest) ProtoMessage() {} +func (*SendLatest) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{2} +} +func (m *SendLatest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SendLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SendLatest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SendLatest) XXX_Merge(src proto.Message) { + xxx_messageInfo_SendLatest.Merge(m, src) +} +func (m *SendLatest) XXX_Size() int { + return m.Size() +} +func (m *SendLatest) XXX_DiscardUnknown() { + xxx_messageInfo_SendLatest.DiscardUnknown(m) +} + +var xxx_messageInfo_SendLatest proto.InternalMessageInfo + +func (m *SendLatest) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func init() { + proto.RegisterType((*Message)(nil), "namesys.pb.Message") + proto.RegisterType((*RequestLatest)(nil), "namesys.pb.RequestLatest") + proto.RegisterType((*SendLatest)(nil), "namesys.pb.SendLatest") +} + +func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } + +var fileDescriptor_33c57e4bae7b9afd = []byte{ + // 181 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, + 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, + 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xf2, 0xe7, 0x62, 0xf7, 0x85, 0x48, 0x0a, 0xc9, 0x70, 0x71, + 0x16, 0xa7, 0x16, 0x17, 0x67, 0xe6, 0xe7, 0x79, 0xba, 0x48, 0x30, 0x2a, 0x30, 0x69, 0xb0, 0x06, + 0x21, 0x04, 0x84, 0x84, 0xb8, 0x58, 0x4a, 0x2a, 0x0b, 0x52, 0x25, 0x98, 0x14, 0x18, 0x35, 0x58, + 0x83, 0xc0, 0x6c, 0x90, 0x58, 0x4a, 0x62, 0x49, 0xa2, 0x04, 0xb3, 0x02, 0xa3, 0x06, 0x4f, 0x10, + 0x98, 0xad, 0xa4, 0xcf, 0xc5, 0x1b, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0xe2, 0x93, 0x58, 0x92, + 0x5a, 0x5c, 0x22, 0x24, 0xc7, 0xc5, 0x95, 0x99, 0x92, 0x9a, 0x57, 0x92, 0x99, 0x96, 0x99, 0x5a, + 0x24, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x84, 0x24, 0xa2, 0xa4, 0xc0, 0xc5, 0x15, 0x9c, 0x9a, + 0x97, 0x02, 0x55, 0x0d, 0x33, 0x92, 0x11, 0x61, 0xa4, 0x13, 0xcf, 0x89, 0x47, 0x72, 0x8c, 0x17, + 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x08, 0x08, 0x00, 0x00, 0xff, 0xff, 0xef, 0x81, 0x61, + 0xa2, 0xcd, 0x00, 0x00, 0x00, +} + +func (m *Message) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Message) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.SessionID == nil { + return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("sessionID") + } else { + dAtA[i] = 0x8 + i++ + i = encodeVarintMessage(dAtA, i, uint64(*m.SessionID)) + } + if m.Type != nil { + dAtA[i] = 0x10 + i++ + i = encodeVarintMessage(dAtA, i, uint64(*m.Type)) + } + if m.Data != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *RequestLatest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Identifier != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintMessage(dAtA, i, uint64(len(*m.Identifier))) + i += copy(dAtA[i:], *m.Identifier) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func (m *SendLatest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SendLatest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Data != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + +func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *Message) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SessionID != nil { + n += 1 + sovMessage(uint64(*m.SessionID)) + } + if m.Type != nil { + n += 1 + sovMessage(uint64(*m.Type)) + } + if m.Data != nil { + l = len(m.Data) + n += 1 + l + sovMessage(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *RequestLatest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Identifier != nil { + l = len(*m.Identifier) + n += 1 + l + sovMessage(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *SendLatest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Data != nil { + l = len(m.Data) + n += 1 + l + sovMessage(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovMessage(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozMessage(x uint64) (n int) { + return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Message) Unmarshal(dAtA []byte) error { + var hasFields [1]uint64 + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Message: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Message: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SessionID", wireType) + } + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.SessionID = &v + hasFields[0] |= uint64(0x00000001) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + var v int32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Type = &v + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + if hasFields[0]&uint64(0x00000001) == 0 { + return github_com_gogo_protobuf_proto.NewRequiredNotSetError("sessionID") + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestLatest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestLatest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestLatest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identifier", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + s := string(dAtA[iNdEx:postIndex]) + m.Identifier = &s + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SendLatest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SendLatest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SendLatest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipMessage(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthMessage + } + iNdEx += length + if iNdEx < 0 { + return 0, ErrInvalidLengthMessage + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipMessage(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + if iNdEx < 0 { + return 0, ErrInvalidLengthMessage + } + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthMessage = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMessage = fmt.Errorf("proto: integer overflow") +) diff --git a/pb/message.proto b/pb/message.proto new file mode 100644 index 0000000..08f8d98 --- /dev/null +++ b/pb/message.proto @@ -0,0 +1,21 @@ +syntax = "proto2"; + +package namesys.pb; + +message Message { + required int32 sessionID = 1; + optional int32 type = 2; + optional bytes data = 3; +} + +message RequestLatest { + optional string identifier = 1; +} + +message SendLatest { + optional bytes data = 1; +} + +message Heads { + repeated bytes heads = 1; +} \ No newline at end of file From 3ae7b1fbd3bbd1599df73bc6b74f89227749a002 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 3 Jul 2019 09:45:48 -0400 Subject: [PATCH 05/27] Use changes from ongoing pubsub PRs --- go.mod | 2 +- go.sum | 1 + pubsub.go | 21 ++++++++++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 0d3d888..471176d 100644 --- a/go.mod +++ b/go.mod @@ -14,4 +14,4 @@ require ( github.com/libp2p/go-libp2p-swarm v0.1.0 ) -replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => ../../libp2p/go-libp2p-pubsub +replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece diff --git a/go.sum b/go.sum index df9614c..f57d8cb 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece/go.mod h1:ekhyliBSJ0aBg62+j9rECrxW+UUPAj1SLS3jN7JMAGc= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= diff --git a/pubsub.go b/pubsub.go index cd8753b..426119b 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,6 +6,8 @@ import ( "encoding/base64" "encoding/binary" "fmt" + + "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" @@ -409,20 +411,25 @@ func (p *PubsubValueStore) handleNewMsgs(sub *pubsub.Subscription, key string) ( } func (p *PubsubValueStore) handleNewPeer(sub *pubsub.Subscription, key string) ([]byte, error) { - peer, err := sub.NextPeerJoin(p.ctx) - if err != nil { - if err != context.Canceled { - log.Warningf("PubsubNewPeer: subscription error in %s: %s", key, err.Error()) + peerEvt, err := sub.NextPeerEvent(p.ctx) + for { + if err != nil { + if err != context.Canceled { + log.Warningf("PubsubNewPeer: subscription error in %s: %s", key, err.Error()) + } + return nil, err + } + if peerEvt.Type == pubsub.PEER_JOIN { + break } - return nil, err } peerCtx, _ := context.WithTimeout(p.ctx, time.Second*10) - s, err := p.host.NewStream(peerCtx, peer, OnJoinProto) + s, err := p.host.NewStream(peerCtx, peerEvt.Peer, OnJoinProto) if err != nil { return nil, err } - defer net.FullClose(s) + defer helpers.FullClose(s) msg := pb.RequestLatest{Identifier: &key} msgData, err := msg.Marshal() From 6fb9435bfdf6d991af5c9bcf176173d95c3cabe2 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 3 Jul 2019 11:13:09 -0400 Subject: [PATCH 06/27] removed unused protobufs --- go.sum | 1 + pb/message.pb.go | 457 +---------------------------------------------- pb/message.proto | 14 -- 3 files changed, 8 insertions(+), 464 deletions(-) diff --git a/go.sum b/go.sum index f57d8cb..2edf9d6 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece h1:lC1y2VJ4vAuUNaVtLsUgyoRuIWUWMEy74WPK7anoyvk= github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece/go.mod h1:ekhyliBSJ0aBg62+j9rECrxW+UUPAj1SLS3jN7JMAGc= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= diff --git a/pb/message.pb.go b/pb/message.pb.go index c477b71..0cd026a 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -5,7 +5,6 @@ package namesys_pb import ( fmt "fmt" - github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -22,69 +21,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package -type Message struct { - SessionID *int32 `protobuf:"varint,1,req,name=sessionID" json:"sessionID,omitempty"` - Type *int32 `protobuf:"varint,2,opt,name=type" json:"type,omitempty"` - Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *Message) Reset() { *m = Message{} } -func (m *Message) String() string { return proto.CompactTextString(m) } -func (*Message) ProtoMessage() {} -func (*Message) Descriptor() ([]byte, []int) { - return fileDescriptor_33c57e4bae7b9afd, []int{0} -} -func (m *Message) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Message.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Message) XXX_Merge(src proto.Message) { - xxx_messageInfo_Message.Merge(m, src) -} -func (m *Message) XXX_Size() int { - return m.Size() -} -func (m *Message) XXX_DiscardUnknown() { - xxx_messageInfo_Message.DiscardUnknown(m) -} - -var xxx_messageInfo_Message proto.InternalMessageInfo - -func (m *Message) GetSessionID() int32 { - if m != nil && m.SessionID != nil { - return *m.SessionID - } - return 0 -} - -func (m *Message) GetType() int32 { - if m != nil && m.Type != nil { - return *m.Type - } - return 0 -} - -func (m *Message) GetData() []byte { - if m != nil { - return m.Data - } - return nil -} - type RequestLatest struct { Identifier *string `protobuf:"bytes,1,opt,name=identifier" json:"identifier,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -96,7 +32,7 @@ func (m *RequestLatest) Reset() { *m = RequestLatest{} } func (m *RequestLatest) String() string { return proto.CompactTextString(m) } func (*RequestLatest) ProtoMessage() {} func (*RequestLatest) Descriptor() ([]byte, []int) { - return fileDescriptor_33c57e4bae7b9afd, []int{1} + return fileDescriptor_33c57e4bae7b9afd, []int{0} } func (m *RequestLatest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -132,114 +68,21 @@ func (m *RequestLatest) GetIdentifier() string { return "" } -type SendLatest struct { - Data []byte `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *SendLatest) Reset() { *m = SendLatest{} } -func (m *SendLatest) String() string { return proto.CompactTextString(m) } -func (*SendLatest) ProtoMessage() {} -func (*SendLatest) Descriptor() ([]byte, []int) { - return fileDescriptor_33c57e4bae7b9afd, []int{2} -} -func (m *SendLatest) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *SendLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_SendLatest.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalTo(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *SendLatest) XXX_Merge(src proto.Message) { - xxx_messageInfo_SendLatest.Merge(m, src) -} -func (m *SendLatest) XXX_Size() int { - return m.Size() -} -func (m *SendLatest) XXX_DiscardUnknown() { - xxx_messageInfo_SendLatest.DiscardUnknown(m) -} - -var xxx_messageInfo_SendLatest proto.InternalMessageInfo - -func (m *SendLatest) GetData() []byte { - if m != nil { - return m.Data - } - return nil -} - func init() { - proto.RegisterType((*Message)(nil), "namesys.pb.Message") proto.RegisterType((*RequestLatest)(nil), "namesys.pb.RequestLatest") - proto.RegisterType((*SendLatest)(nil), "namesys.pb.SendLatest") } func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 181 bytes of a gzipped FileDescriptorProto + // 106 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, - 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xf2, 0xe7, 0x62, 0xf7, 0x85, 0x48, 0x0a, 0xc9, 0x70, 0x71, - 0x16, 0xa7, 0x16, 0x17, 0x67, 0xe6, 0xe7, 0x79, 0xba, 0x48, 0x30, 0x2a, 0x30, 0x69, 0xb0, 0x06, - 0x21, 0x04, 0x84, 0x84, 0xb8, 0x58, 0x4a, 0x2a, 0x0b, 0x52, 0x25, 0x98, 0x14, 0x18, 0x35, 0x58, - 0x83, 0xc0, 0x6c, 0x90, 0x58, 0x4a, 0x62, 0x49, 0xa2, 0x04, 0xb3, 0x02, 0xa3, 0x06, 0x4f, 0x10, - 0x98, 0xad, 0xa4, 0xcf, 0xc5, 0x1b, 0x94, 0x5a, 0x58, 0x9a, 0x5a, 0x5c, 0xe2, 0x93, 0x58, 0x92, - 0x5a, 0x5c, 0x22, 0x24, 0xc7, 0xc5, 0x95, 0x99, 0x92, 0x9a, 0x57, 0x92, 0x99, 0x96, 0x99, 0x5a, - 0x24, 0xc1, 0xa8, 0xc0, 0xa8, 0xc1, 0x19, 0x84, 0x24, 0xa2, 0xa4, 0xc0, 0xc5, 0x15, 0x9c, 0x9a, - 0x97, 0x02, 0x55, 0x0d, 0x33, 0x92, 0x11, 0x61, 0xa4, 0x13, 0xcf, 0x89, 0x47, 0x72, 0x8c, 0x17, - 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x08, 0x08, 0x00, 0x00, 0xff, 0xff, 0xef, 0x81, 0x61, - 0xa2, 0xcd, 0x00, 0x00, 0x00, -} - -func (m *Message) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Message) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.SessionID == nil { - return 0, github_com_gogo_protobuf_proto.NewRequiredNotSetError("sessionID") - } else { - dAtA[i] = 0x8 - i++ - i = encodeVarintMessage(dAtA, i, uint64(*m.SessionID)) - } - if m.Type != nil { - dAtA[i] = 0x10 - i++ - i = encodeVarintMessage(dAtA, i, uint64(*m.Type)) - } - if m.Data != nil { - dAtA[i] = 0x1a - i++ - i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) - i += copy(dAtA[i:], m.Data) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil + 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, + 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, + 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x71, 0xe2, 0x39, + 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x01, 0x01, 0x00, 0x00, + 0xff, 0xff, 0xba, 0x07, 0x1b, 0x54, 0x5a, 0x00, 0x00, 0x00, } func (m *RequestLatest) Marshal() (dAtA []byte, err error) { @@ -269,33 +112,6 @@ func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *SendLatest) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalTo(dAtA) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *SendLatest) MarshalTo(dAtA []byte) (int, error) { - var i int - _ = i - var l int - _ = l - if m.Data != nil { - dAtA[i] = 0xa - i++ - i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) - i += copy(dAtA[i:], m.Data) - } - if m.XXX_unrecognized != nil { - i += copy(dAtA[i:], m.XXX_unrecognized) - } - return i, nil -} - func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -305,28 +121,6 @@ func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *Message) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.SessionID != nil { - n += 1 + sovMessage(uint64(*m.SessionID)) - } - if m.Type != nil { - n += 1 + sovMessage(uint64(*m.Type)) - } - if m.Data != nil { - l = len(m.Data) - n += 1 + l + sovMessage(uint64(l)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - func (m *RequestLatest) Size() (n int) { if m == nil { return 0 @@ -343,22 +137,6 @@ func (m *RequestLatest) Size() (n int) { return n } -func (m *SendLatest) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Data != nil { - l = len(m.Data) - n += 1 + l + sovMessage(uint64(l)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - func sovMessage(x uint64) (n int) { for { n++ @@ -372,139 +150,6 @@ func sovMessage(x uint64) (n int) { func sozMessage(x uint64) (n int) { return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *Message) Unmarshal(dAtA []byte) error { - var hasFields [1]uint64 - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Message: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Message: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SessionID", wireType) - } - var v int32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.SessionID = &v - hasFields[0] |= uint64(0x00000001) - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) - } - var v int32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Type = &v - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipMessage(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - if hasFields[0]&uint64(0x00000001) == 0 { - return github_com_gogo_protobuf_proto.NewRequiredNotSetError("sessionID") - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *RequestLatest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -592,94 +237,6 @@ func (m *RequestLatest) Unmarshal(dAtA []byte) error { } return nil } -func (m *SendLatest) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SendLatest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SendLatest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthMessage - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthMessage - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipMessage(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthMessage - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func skipMessage(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pb/message.proto b/pb/message.proto index 08f8d98..d815da2 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -2,20 +2,6 @@ syntax = "proto2"; package namesys.pb; -message Message { - required int32 sessionID = 1; - optional int32 type = 2; - optional bytes data = 3; -} - message RequestLatest { optional string identifier = 1; -} - -message SendLatest { - optional bytes data = 1; -} - -message Heads { - repeated bytes heads = 1; } \ No newline at end of file From f3f8dd40ca43c81678dc78fbadd925c0571f52ba Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 3 Jul 2019 13:14:30 -0400 Subject: [PATCH 07/27] records past EOL should fail --- pubsub.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pubsub.go b/pubsub.go index 426119b..db4fb56 100644 --- a/pubsub.go +++ b/pubsub.go @@ -140,18 +140,16 @@ func (p *PubsubValueStore) isBetter(key string, val []byte) (isBetter bool, isEq } old, err := p.getLocal(key) + if err != nil { + // If the old one is invalid, the new one is *always* better. + return true, false + } // Same record is not better if old != nil && bytes.Equal(old, val) { return false, true } - if err != nil { - // If the old one is invalid and is not identical to the new record we assume the new one is better. - // Perhaps we should have levels of invalid (e.g. EOL vs bad formatting) - return true, false - } - i, err := p.Validator.Select(key, [][]byte{val, old}) return err == nil && i == 0, false } @@ -216,8 +214,8 @@ func (p *PubsubValueStore) rebroadcast(key string) { for { select { case <-ticker.C: - val, _ := p.getLocal(key) - if val != nil { + val, err := p.getLocal(key) + if err == nil { _ = p.ps.Publish(topic, val) } case <-p.ctx.Done(): @@ -238,7 +236,7 @@ func (p *PubsubValueStore) getLocal(key string) ([]byte, error) { // If the old one is invalid, the new one is *always* better. if err := p.Validator.Validate(key, val); err != nil { - return val, err + return nil, err } return val, nil } From 9e9d7788071f73464560986d56dad8820c7789ec Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 3 Jul 2019 13:16:02 -0400 Subject: [PATCH 08/27] reorder imports --- pubsub.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pubsub.go b/pubsub.go index db4fb56..ee20f5d 100644 --- a/pubsub.go +++ b/pubsub.go @@ -6,6 +6,9 @@ import ( "encoding/base64" "encoding/binary" "fmt" + "io" + "sync" + "time" "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" @@ -15,16 +18,13 @@ import ( "github.com/libp2p/go-libp2p-net" pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub-router/pb" record "github.com/libp2p/go-libp2p-record" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" dshelp "github.com/ipfs/go-ipfs-ds-help" logging "github.com/ipfs/go-log" - pb "github.com/libp2p/go-libp2p-pubsub-router/pb" - "io" - "sync" - "time" ) var log = logging.Logger("pubsub-valuestore") From a24054d2f9d1b95b6adb1c793fa286626d1ebee5 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 01:32:03 -0400 Subject: [PATCH 09/27] Improved the get-latest protocol (protobufs for request and response, response can have nil data, use standard gogo protobuf tools). Changed the protocol string for the get-latest protocol. Separated out get-latest protocol and gave it explicit tests. Added more tests and modified an existing one to take into account the use of the get-latest protocol. Update go.mod to use newer (still forked) version of pubsub that has better peer event handling. Changed rebroadcast to use one goroutine instead of one per topic (also upped the rebroadcast interval). * Fixed existing issue with hanging on failed pubsub subscription. Better handling of contexts. Misc. code cleanup. --- getlatest.go | 107 +++++++++++++++++++ getlatest_test.go | 99 ++++++++++++++++++ go.mod | 3 +- go.sum | 8 +- pb/message.pb.go | 232 ++++++++++++++++++++++++++++++++++++++++- pb/message.proto | 7 +- pubsub.go | 258 ++++++++++++++++++++-------------------------- pubsub_test.go | 54 +++++++++- 8 files changed, 607 insertions(+), 161 deletions(-) create mode 100644 getlatest.go create mode 100644 getlatest_test.go diff --git a/getlatest.go b/getlatest.go new file mode 100644 index 0000000..ab282b8 --- /dev/null +++ b/getlatest.go @@ -0,0 +1,107 @@ +package namesys + +import ( + "bufio" + "context" + "io" + "time" + + ggio "github.com/gogo/protobuf/io" + "github.com/gogo/protobuf/proto" + + "github.com/libp2p/go-libp2p-core/helpers" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + pb "github.com/libp2p/go-libp2p-pubsub-router/pb" +) + +type getLatestProtocol struct { + host host.Host +} + +func newGetLatestProtocol(host host.Host, getLocal func(key string) ([]byte, error)) *getLatestProtocol { + p := &getLatestProtocol{host} + + host.SetStreamHandler(PSGetLatestProto, func(s network.Stream) { + p.Receive(s, getLocal) + }) + + return p +} + +func (p *getLatestProtocol) Receive(s network.Stream, getLocal func(key string) ([]byte, error)) { + r := ggio.NewDelimitedReader(s, 1<<20) + msg := &pb.RequestLatest{} + if err := r.ReadMsg(msg); err != nil { + if err != io.EOF { + s.Reset() + log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) + } else { + // Just be nice. They probably won't read this + // but it doesn't hurt to send it. + s.Close() + } + return + } + + response, err := getLocal(*msg.Identifier) + var respProto pb.RespondLatest + + if err != nil || response == nil { + nodata := true + respProto = pb.RespondLatest{Nodata: &nodata} + } else { + respProto = pb.RespondLatest{Data: response} + } + + if err := writeBytes(s, &respProto); err != nil { + s.Reset() + log.Infof("error writing response to %s: %s", s.Conn().RemotePeer(), err) + return + } + helpers.FullClose(s) +} + +func (p getLatestProtocol) Send(ctx context.Context, pid peer.ID, key string) ([]byte, error) { + peerCtx, _ := context.WithTimeout(ctx, time.Second*10) + s, err := p.host.NewStream(peerCtx, pid, PSGetLatestProto) + if err != nil { + return nil, err + } + + if err := s.SetDeadline(time.Now().Add(time.Second * 5)); err != nil { + return nil, err + } + + defer helpers.FullClose(s) + + msg := pb.RequestLatest{Identifier: &key} + + if err := writeBytes(s, &msg); err != nil { + s.Reset() + return nil, err + } + + s.Close() + + r := ggio.NewDelimitedReader(s, 1<<20) + response := &pb.RespondLatest{} + if err := r.ReadMsg(response); err != nil { + return nil, err + } + + return response.Data, nil +} + +func writeBytes(w io.Writer, msg proto.Message) error { + bufw := bufio.NewWriter(w) + wc := ggio.NewDelimitedWriter(bufw) + + if err := wc.WriteMsg(msg); err != nil { + return err + } + + return bufw.Flush() +} diff --git a/getlatest_test.go b/getlatest_test.go new file mode 100644 index 0000000..4cc31d3 --- /dev/null +++ b/getlatest_test.go @@ -0,0 +1,99 @@ +package namesys + +import ( + "bytes" + "context" + "errors" + "testing" + + "github.com/libp2p/go-libp2p-core/host" +) + +func connect(t *testing.T, a, b host.Host) { + pinfo := a.Peerstore().PeerInfo(a.ID()) + err := b.Connect(context.Background(), pinfo) + if err != nil { + t.Fatal(err) + } +} + +type datastore struct { + data map[string][]byte +} + +func (d *datastore) Lookup(key string) ([]byte, error) { + v, ok := d.data[key] + if !ok { + return nil, errors.New("key not found") + } + return v, nil +} + +func TestGetLatestProtocolTrip(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hosts := newNetHosts(ctx, t, 2) + connect(t, hosts[0], hosts[1]) + + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} + h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + + d2 := &datastore{map[string][]byte{"key": []byte("value2")}} + h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + + getLatest(t, ctx, h1, h2, "key", []byte("value2")) + getLatest(t, ctx, h2, h1, "key", []byte("value1")) +} + +func TestGetLatestProtocolNil(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hosts := newNetHosts(ctx, t, 2) + connect(t, hosts[0], hosts[1]) + + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} + h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + + d2 := &datastore{make(map[string][]byte)} + h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + + getLatest(t, ctx, h1, h2, "key", nil) + getLatest(t, ctx, h2, h1, "key", []byte("value1")) +} + +func TestGetLatestProtocolRepeated(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hosts := newNetHosts(ctx, t, 2) + connect(t, hosts[0], hosts[1]) + + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} + h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + + d2 := &datastore{make(map[string][]byte)} + h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + + for i := 0; i < 10; i++ { + getLatest(t, ctx, h1, h2, "key", nil) + getLatest(t, ctx, h2, h1, "key", []byte("value1")) + } +} + +func getLatest(t *testing.T, ctx context.Context, + requester *getLatestProtocol, responder *getLatestProtocol, key string, expected []byte) { + data, err := requester.Send(ctx, responder.host.ID(), key) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, expected) { + t.Fatalf("expected: %v, received: %v", string(expected), string(data)) + } + + if (data == nil && expected != nil) || (data != nil && expected == nil) { + t.Fatalf("expected []byte{} or nil and received the opposite") + } +} diff --git a/go.mod b/go.mod index 471176d..94458b1 100644 --- a/go.mod +++ b/go.mod @@ -7,11 +7,10 @@ require ( github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.1.1 github.com/libp2p/go-libp2p-core v0.0.1 - github.com/libp2p/go-libp2p-net v0.1.0 github.com/libp2p/go-libp2p-pubsub v0.1.0 github.com/libp2p/go-libp2p-record v0.1.0 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-swarm v0.1.0 ) -replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece +replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd diff --git a/go.sum b/go.sum index 2edf9d6..8ae368f 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece h1:lC1y2VJ4vAuUNaVtLsUgyoRuIWUWMEy74WPK7anoyvk= -github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190701154427-d91967657ece/go.mod h1:ekhyliBSJ0aBg62+j9rECrxW+UUPAj1SLS3jN7JMAGc= +github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd h1:P1//443gNhGLDpMpzwnaNDgKoHpXrlt+iIEVgwlTHoI= +github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd/go.mod h1:ekhyliBSJ0aBg62+j9rECrxW+UUPAj1SLS3jN7JMAGc= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -94,14 +94,10 @@ github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3x github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1 h1:E1xaJBQnbSiTHGI1gaBKmKhu1TUKkErKJnE8iGvirYI= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= -github.com/libp2p/go-libp2p-net v0.1.0 h1:3t23V5cR4GXcNoFriNoZKFdUZEUDZgUkvfwkD2INvQE= -github.com/libp2p/go-libp2p-net v0.1.0/go.mod h1:R5VZbutk75tkC5YJJS61OCO1NWoajxYjCEV2RoHh3FY= github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0 h1:MKh7pRNPHSh1fLPj8u/M/s/napdmeNpoi9BRy9lPN0E= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-pubsub v0.1.0 h1:SmQeMa7IUv5vadh0fYgYsafWCBA1sCy5d/68kIYqGcU= -github.com/libp2p/go-libp2p-pubsub v0.1.0/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= diff --git a/pb/message.pb.go b/pb/message.pb.go index 0cd026a..91e88fe 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -68,21 +68,80 @@ func (m *RequestLatest) GetIdentifier() string { return "" } +type RespondLatest struct { + Data []byte `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` + Nodata *bool `protobuf:"varint,2,opt,name=nodata" json:"nodata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *RespondLatest) Reset() { *m = RespondLatest{} } +func (m *RespondLatest) String() string { return proto.CompactTextString(m) } +func (*RespondLatest) ProtoMessage() {} +func (*RespondLatest) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{1} +} +func (m *RespondLatest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RespondLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RespondLatest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RespondLatest) XXX_Merge(src proto.Message) { + xxx_messageInfo_RespondLatest.Merge(m, src) +} +func (m *RespondLatest) XXX_Size() int { + return m.Size() +} +func (m *RespondLatest) XXX_DiscardUnknown() { + xxx_messageInfo_RespondLatest.DiscardUnknown(m) +} + +var xxx_messageInfo_RespondLatest proto.InternalMessageInfo + +func (m *RespondLatest) GetData() []byte { + if m != nil { + return m.Data + } + return nil +} + +func (m *RespondLatest) GetNodata() bool { + if m != nil && m.Nodata != nil { + return *m.Nodata + } + return false +} + func init() { proto.RegisterType((*RequestLatest)(nil), "namesys.pb.RequestLatest") + proto.RegisterType((*RespondLatest)(nil), "namesys.pb.RespondLatest") } func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 106 bytes of a gzipped FileDescriptorProto + // 145 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, - 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x71, 0xe2, 0x39, - 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x01, 0x01, 0x00, 0x00, - 0xff, 0xff, 0xba, 0x07, 0x1b, 0x54, 0x5a, 0x00, 0x00, 0x00, + 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0xb2, 0x06, + 0x69, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0x6a, 0x10, 0xe2, 0x62, 0x49, 0x49, 0x2c, 0x49, 0x04, + 0x2b, 0xe5, 0x09, 0x02, 0xb3, 0x85, 0xc4, 0xb8, 0xd8, 0xf2, 0xf2, 0xc1, 0xa2, 0x4c, 0x0a, 0x8c, + 0x1a, 0x1c, 0x41, 0x50, 0x9e, 0x13, 0xcf, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, + 0x78, 0x24, 0xc7, 0x08, 0x08, 0x00, 0x00, 0xff, 0xff, 0x21, 0x3f, 0xdb, 0xfb, 0x97, 0x00, 0x00, + 0x00, } func (m *RequestLatest) Marshal() (dAtA []byte, err error) { @@ -112,6 +171,43 @@ func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *RespondLatest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RespondLatest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Data != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) + i += copy(dAtA[i:], m.Data) + } + if m.Nodata != nil { + dAtA[i] = 0x10 + i++ + if *m.Nodata { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -137,6 +233,25 @@ func (m *RequestLatest) Size() (n int) { return n } +func (m *RespondLatest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Data != nil { + l = len(m.Data) + n += 1 + l + sovMessage(uint64(l)) + } + if m.Nodata != nil { + n += 2 + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovMessage(x uint64) (n int) { for { n++ @@ -237,6 +352,115 @@ func (m *RequestLatest) Unmarshal(dAtA []byte) error { } return nil } +func (m *RespondLatest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RespondLatest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RespondLatest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Nodata", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + b := bool(v != 0) + m.Nodata = &b + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipMessage(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pb/message.proto b/pb/message.proto index d815da2..4a04228 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -3,5 +3,10 @@ syntax = "proto2"; package namesys.pb; message RequestLatest { - optional string identifier = 1; + optional string identifier = 1; +} + +message RespondLatest { + optional bytes data = 1; + optional bool nodata = 2; } \ No newline at end of file diff --git a/pubsub.go b/pubsub.go index ee20f5d..ac6519b 100644 --- a/pubsub.go +++ b/pubsub.go @@ -4,21 +4,16 @@ import ( "bytes" "context" "encoding/base64" - "encoding/binary" "fmt" - "io" "sync" "time" - "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-core/routing" - "github.com/libp2p/go-libp2p-net" pubsub "github.com/libp2p/go-libp2p-pubsub" - pb "github.com/libp2p/go-libp2p-pubsub-router/pb" record "github.com/libp2p/go-libp2p-record" ds "github.com/ipfs/go-datastore" @@ -29,7 +24,7 @@ import ( var log = logging.Logger("pubsub-valuestore") -const OnJoinProto = protocol.ID("/psOnJoin/0.0.1") +const PSGetLatestProto = protocol.ID("/pubsub-get-latest/0.0.1") type watchGroup struct { // Note: this chan must be buffered, see notifyWatchers @@ -41,18 +36,18 @@ type PubsubValueStore struct { ds ds.Datastore ps *pubsub.PubSub - host host.Host + host host.Host + getLatest *getLatestProtocol rebroadcastInitialDelay time.Duration rebroadcastInterval time.Duration // Map of keys to subscriptions. - // - // If a key is present but the subscription is nil, we've bootstrapped - // but haven't subscribed. mx sync.Mutex subs map[string]*pubsub.Subscription + cancels map[string]context.CancelFunc + watchLk sync.Mutex watching map[string]*watchGroup @@ -77,36 +72,19 @@ func NewPubsubValueStore(ctx context.Context, host host.Host, cr routing.Content ps: ps, host: host, rebroadcastInitialDelay: 100 * time.Millisecond, - rebroadcastInterval: time.Second, + rebroadcastInterval: time.Minute * 10, subs: make(map[string]*pubsub.Subscription), + cancels: make(map[string]context.CancelFunc), watching: make(map[string]*watchGroup), Validator: validator, } - host.SetStreamHandler(OnJoinProto, func(s net.Stream) { - defer net.FullClose(s) + psValueStore.getLatest = newGetLatestProtocol(host, psValueStore.getLocal) - msgData, err := readBytes(s) - if err != nil { - return - } + go psValueStore.rebroadcast(ctx) - msg := pb.RequestLatest{} - if msg.Unmarshal(msgData) != nil { - return - } - - response, err := psValueStore.getLocal(*msg.Identifier) - if err != nil { - return - } - - if writeBytes(s, response) != nil { - return - } - }) return psValueStore } @@ -121,37 +99,39 @@ func (p *PubsubValueStore) PutValue(ctx context.Context, key string, value []byt } log.Debugf("PubsubPublish: publish value for key", key) - done := make(chan error, 1) - go func() { - done <- p.ps.Publish(topic, value) - }() select { - case err := <-done: + case err := <-p.psPublishChannel(topic, value): return err case <-ctx.Done(): return ctx.Err() } } -func (p *PubsubValueStore) isBetter(key string, val []byte) (isBetter bool, isEqual bool) { +// compare compares the input value with the current value. +// Returns 0 if equal, greater than 0 if better, less than 0 if worse +func (p *PubsubValueStore) compare(key string, val []byte) int { if p.Validator.Validate(key, val) != nil { - return false, false + return -1 } old, err := p.getLocal(key) if err != nil { // If the old one is invalid, the new one is *always* better. - return true, false + return 1 } // Same record is not better if old != nil && bytes.Equal(old, val) { - return false, true + return 0 } i, err := p.Validator.Select(key, [][]byte{val, old}) - return err == nil && i == 0, false + if err == nil && i == 0 { + return 1 + } else { + return -1 + } } func (p *PubsubValueStore) Subscribe(key string) error { @@ -172,17 +152,13 @@ func (p *PubsubValueStore) Subscribe(key string) error { // Also, make sure to do this *before* subscribing. myID := p.host.ID() _ = p.ps.RegisterTopicValidator(topic, func(ctx context.Context, src peer.ID, msg *pubsub.Message) bool { - isBetter, isEqual := p.isBetter(key, msg.GetData()) + cmp := p.compare(key, msg.GetData()) - if src == myID && isEqual { - return true - } - return isBetter + return cmp > 0 || cmp == 0 && src == myID }) sub, err := p.ps.Subscribe(topic) if err != nil { - p.mx.Unlock() return err } @@ -194,18 +170,19 @@ func (p *PubsubValueStore) Subscribe(key string) error { return nil } + ctx, cancel := context.WithCancel(p.ctx) + p.cancels[key] = cancel + p.subs[key] = sub - go p.handleSubscription(sub, key) + go p.handleSubscription(ctx, sub, key) p.mx.Unlock() - go p.rebroadcast(key) log.Debugf("PubsubResolve: subscribed to %s", key) return nil } -func (p *PubsubValueStore) rebroadcast(key string) { - topic := KeyToTopic(key) +func (p *PubsubValueStore) rebroadcast(ctx context.Context) { time.Sleep(p.rebroadcastInitialDelay) ticker := time.NewTicker(p.rebroadcastInterval) @@ -214,16 +191,40 @@ func (p *PubsubValueStore) rebroadcast(key string) { for { select { case <-ticker.C: - val, err := p.getLocal(key) - if err == nil { - _ = p.ps.Publish(topic, val) + var keys []string + p.mx.Lock() + keys = make([]string, 0, len(p.subs)) + for p, _ := range p.subs { + keys = append(keys, p) } - case <-p.ctx.Done(): + p.mx.Unlock() + if len(keys) > 0 { + for _, k := range keys { + val, err := p.getLocal(k) + if err == nil { + topic := KeyToTopic(k) + select { + case <-p.psPublishChannel(topic, val): + case <-ctx.Done(): + return + } + } + } + } + case <-ctx.Done(): return } } } +func (p *PubsubValueStore) psPublishChannel(topic string, value []byte) chan error { + done := make(chan error, 1) + go func() { + done <- p.ps.Publish(topic, value) + }() + return done +} + func (p *PubsubValueStore) getLocal(key string) ([]byte, error) { val, err := p.ds.Get(dshelp.NewKeyFromBinary([]byte(key))) if err != nil { @@ -350,44 +351,74 @@ func (p *PubsubValueStore) Cancel(name string) (bool, error) { delete(p.subs, name) } + if cancel, ok := p.cancels[name]; ok { + cancel() + delete(p.cancels, name) + } + return ok, nil } -func (p *PubsubValueStore) handleSubscription(sub *pubsub.Subscription, key string) { +func (p *PubsubValueStore) handleSubscription(ctx context.Context, sub *pubsub.Subscription, key string) { defer sub.Cancel() newMsg := make(chan []byte) go func() { + defer close(newMsg) for { - data, err := p.handleNewMsgs(sub, key) + data, err := p.handleNewMsgs(ctx, sub, key) if err != nil { - close(newMsg) return } - newMsg <- data + select { + case newMsg <- data: + case <-ctx.Done(): + return + } } }() newPeerData := make(chan []byte) go func() { + defer close(newPeerData) for { - data, err := p.handleNewPeer(sub, key) - if err != nil { - close(newPeerData) - return + data, err := p.handleNewPeer(ctx, sub, key) + if err == nil { + if data != nil { + select { + case newPeerData <- data: + case <-ctx.Done(): + return + } + } + } else { + select { + case <-ctx.Done(): + return + default: + log.Errorf("PubsubPeerJoin: error interacting with new peer", err) + } } - newPeerData <- data } }() for { var data []byte + var ok bool select { - case data = <-newMsg: - case data = <-newPeerData: + case data, ok = <-newMsg: + if !ok { + return + } + case data, ok = <-newPeerData: + if !ok { + return + } + case <-ctx.Done(): + return } - if isBetter, _ := p.isBetter(key, data); isBetter { + if p.compare(key, data) > 0 { err := p.ds.Put(dshelp.NewKeyFromBinary([]byte(key)), data) if err != nil { log.Warningf("PubsubResolve: error writing update for %s: %s", key, err) @@ -397,8 +428,8 @@ func (p *PubsubValueStore) handleSubscription(sub *pubsub.Subscription, key stri } } -func (p *PubsubValueStore) handleNewMsgs(sub *pubsub.Subscription, key string) ([]byte, error) { - msg, err := sub.Next(p.ctx) +func (p *PubsubValueStore) handleNewMsgs(ctx context.Context, sub *pubsub.Subscription, key string) ([]byte, error) { + msg, err := sub.Next(ctx) if err != nil { if err != context.Canceled { log.Warningf("PubsubResolve: subscription error in %s: %s", key, err.Error()) @@ -408,45 +439,30 @@ func (p *PubsubValueStore) handleNewMsgs(sub *pubsub.Subscription, key string) ( return msg.GetData(), nil } -func (p *PubsubValueStore) handleNewPeer(sub *pubsub.Subscription, key string) ([]byte, error) { - peerEvt, err := sub.NextPeerEvent(p.ctx) +func (p *PubsubValueStore) handleNewPeer(ctx context.Context, sub *pubsub.Subscription, key string) ([]byte, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + var pid peer.ID + for { + peerEvt, err := sub.NextPeerEvent(ctx) if err != nil { if err != context.Canceled { log.Warningf("PubsubNewPeer: subscription error in %s: %s", key, err.Error()) } return nil, err } - if peerEvt.Type == pubsub.PEER_JOIN { + if peerEvt.Type == pubsub.PeerJoin { + pid = peerEvt.Peer break } } - peerCtx, _ := context.WithTimeout(p.ctx, time.Second*10) - s, err := p.host.NewStream(peerCtx, peerEvt.Peer, OnJoinProto) - if err != nil { - return nil, err - } - defer helpers.FullClose(s) - - msg := pb.RequestLatest{Identifier: &key} - msgData, err := msg.Marshal() - if err != nil { - return nil, err - } - - if writeBytes(s, msgData) != nil { - return nil, err - } - - s.Close() - - response, err := readBytes(s) - if err != nil { - return nil, err - } - - return response, nil + return p.getLatest.Send(ctx, pid, key) } func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { @@ -465,51 +481,3 @@ func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { } } } - -const sizeLengthBytes = 8 - -// readNumBytesFromReader reads a specific number of bytes from a Reader, or returns an error -func readNumBytesFromReader(r io.Reader, numBytes uint64) ([]byte, error) { - data := make([]byte, numBytes) - n, err := io.ReadFull(r, data) - if err != nil { - return data, err - } else if uint64(n) != numBytes { - return data, fmt.Errorf("Could not read full length from stream") - } - return data, nil -} - -func readBytes(r io.Reader) ([]byte, error) { - // Protocol: uint64 MessageLength followed by byte[] MarshalledMessage - - sizeData, err := readNumBytesFromReader(r, sizeLengthBytes) - if err != nil { - return nil, err - } - - size := binary.LittleEndian.Uint64(sizeData) - data, err := readNumBytesFromReader(r, size) - if err != nil { - return nil, err - } - - return data, nil -} - -func writeBytes(w io.Writer, data []byte) error { - size := len(data) - - // Protocol: uint64 MessageLength followed by byte[] MarshalledMessage - sizeData := make([]byte, sizeLengthBytes) - binary.LittleEndian.PutUint64(sizeData, uint64(size)) - - _, err := w.Write(sizeData) - if err != nil { - return err - } - - _, err = w.Write(data) - - return err -} diff --git a/pubsub_test.go b/pubsub_test.go index f3a2a2e..0085257 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -105,6 +105,47 @@ func setupTest(ctx context.Context, t *testing.T) (*PubsubValueStore, []*PubsubV } // tests +func TestEarlyPublish(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hosts := newNetHosts(ctx, t, 5) + + key := "/namespace/key" + val := []byte("valid for key 1") + + vss := make([]*PubsubValueStore, len(hosts)) + for i := 0; i < len(vss); i++ { + fs, err := pubsub.NewFloodSub(ctx, hosts[i]) + if err != nil { + t.Fatal(err) + } + + vss[i] = NewPubsubValueStore(ctx, hosts[i], rhelper.Null{}, fs, testValidator{}) + } + + pub := vss[0] + vss = vss[1:] + + if err := pub.PutValue(ctx, key, val); err != nil { + t.Fatal(err) + } + + for i, vs := range vss { + connect(t, hosts[i], hosts[i+1]) + if err := vs.Subscribe(key); err != nil { + t.Fatal(err) + } + } + + // Wait for Get Latest protocol to retrieve data + time.Sleep(time.Second * 1) + for i, vs := range vss { + checkValue(ctx, t, i, vs, key, val) + } + +} + func TestPubsubPublishSubscribe(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -192,16 +233,24 @@ func TestPubsubPublishSubscribe(t *testing.T) { } time.Sleep(time.Millisecond * 100) + // Get missed value? nval = []byte("valid for key 3") err = pub.PutValue(ctx, key, nval) if err != nil { t.Fatal(err) } - // check we still have the old value in the vssolver + // resubscribe + for _, vs := range vss { + if err := vs.Subscribe(key); err != nil { + t.Fatal(err) + } + } + + // check that we get the new value time.Sleep(time.Second * 1) for i, vs := range vss { - checkValue(ctx, t, i, vs, key, val) + checkValue(ctx, t, i, vs, key, nval) } } @@ -258,7 +307,6 @@ func TestWatch(t *testing.T) { if v != "valid for key 2" { t.Errorf("got unexpected value: %s", v) } - } func checkNotFound(ctx context.Context, t *testing.T, i int, vs routing.ValueStore, key string) { From 092d0e188d68f0755948113cf26cc5b4877f7697 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 02:16:48 -0400 Subject: [PATCH 10/27] better context cancel in get-latest protocol --- getlatest.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/getlatest.go b/getlatest.go index ab282b8..935e959 100644 --- a/getlatest.go +++ b/getlatest.go @@ -65,7 +65,9 @@ func (p *getLatestProtocol) Receive(s network.Stream, getLocal func(key string) } func (p getLatestProtocol) Send(ctx context.Context, pid peer.ID, key string) ([]byte, error) { - peerCtx, _ := context.WithTimeout(ctx, time.Second*10) + peerCtx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + s, err := p.host.NewStream(peerCtx, pid, PSGetLatestProto) if err != nil { return nil, err From 3bf24fa78f71a3bcdaa9a29118d29b3e0dbc51f9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 15:42:11 -0400 Subject: [PATCH 11/27] restore bootstrapping ... for now --- go.mod | 2 ++ pubsub.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 94458b1..d5ca32d 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,10 @@ module github.com/libp2p/go-libp2p-pubsub-router require ( github.com/gogo/protobuf v1.2.1 + github.com/ipfs/go-cid v0.0.2 github.com/ipfs/go-datastore v0.0.5 github.com/ipfs/go-ipfs-ds-help v0.0.1 + github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.1.1 github.com/libp2p/go-libp2p-core v0.0.1 diff --git a/pubsub.go b/pubsub.go index ac6519b..79571ac 100644 --- a/pubsub.go +++ b/pubsub.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "fmt" + "github.com/ipfs/go-cid" "sync" "time" @@ -19,6 +20,7 @@ import ( ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" dshelp "github.com/ipfs/go-ipfs-ds-help" + u "github.com/ipfs/go-ipfs-util" logging "github.com/ipfs/go-log" ) @@ -34,6 +36,7 @@ type watchGroup struct { type PubsubValueStore struct { ctx context.Context ds ds.Datastore + cr routing.ContentRouting ps *pubsub.PubSub host host.Host @@ -43,6 +46,9 @@ type PubsubValueStore struct { rebroadcastInterval time.Duration // Map of keys to subscriptions. + // + // If a key is present but the subscription is nil, we've bootstrapped + // but haven't subscribed. mx sync.Mutex subs map[string]*pubsub.Subscription @@ -68,6 +74,8 @@ func NewPubsubValueStore(ctx context.Context, host host.Host, cr routing.Content psValueStore := &PubsubValueStore{ ctx: ctx, + cr: cr, // needed for pubsub bootstrap + ds: dssync.MutexWrap(ds.NewMapDatastore()), ps: ps, host: host, @@ -163,7 +171,7 @@ func (p *PubsubValueStore) Subscribe(key string) error { } p.mx.Lock() - existingSub, _ := p.subs[key] + existingSub, bootstrapped := p.subs[key] if existingSub != nil { p.mx.Unlock() sub.Cancel() @@ -174,11 +182,16 @@ func (p *PubsubValueStore) Subscribe(key string) error { p.cancels[key] = cancel p.subs[key] = sub - go p.handleSubscription(ctx, sub, key) + go p.handleSubscription(ctx, sub, key, cancel) p.mx.Unlock() log.Debugf("PubsubResolve: subscribed to %s", key) + if !bootstrapped { + // TODO: Deal with publish then resolve case? Cancel behaviour changes. + go bootstrapPubsub(ctx, p.cr, p.host, topic) + } + return nil } @@ -359,8 +372,9 @@ func (p *PubsubValueStore) Cancel(name string) (bool, error) { return ok, nil } -func (p *PubsubValueStore) handleSubscription(ctx context.Context, sub *pubsub.Subscription, key string) { +func (p *PubsubValueStore) handleSubscription(ctx context.Context, sub *pubsub.Subscription, key string, cancel func()) { defer sub.Cancel() + defer cancel() newMsg := make(chan []byte) go func() { @@ -481,3 +495,61 @@ func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { } } } + +// rendezvous with peers in the name topic through provider records +// Note: rendezvous/boostrap should really be handled by the pubsub implementation itself! +func bootstrapPubsub(ctx context.Context, cr routing.ContentRouting, host host.Host, name string) { + // TODO: consider changing this to `pubsub:...` + topic := "floodsub:" + name + hash := u.Hash([]byte(topic)) + rz := cid.NewCidV1(cid.Raw, hash) + + go func() { + err := cr.Provide(ctx, rz, true) + if err != nil { + log.Warningf("bootstrapPubsub: error providing rendezvous for %s: %s", topic, err.Error()) + } + + for { + select { + case <-time.After(8 * time.Hour): + err := cr.Provide(ctx, rz, true) + if err != nil { + log.Warningf("bootstrapPubsub: error providing rendezvous for %s: %s", topic, err.Error()) + } + case <-ctx.Done(): + return + } + } + }() + + rzctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + wg := &sync.WaitGroup{} + for pi := range cr.FindProvidersAsync(rzctx, rz, 10) { + if pi.ID == host.ID() { + continue + } + wg.Add(1) + go func(pi peer.AddrInfo) { + defer wg.Done() + + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + + err := host.Connect(ctx, pi) + if err != nil { + log.Debugf("Error connecting to pubsub peer %s: %s", pi.ID, err.Error()) + return + } + + // delay to let pubsub perform its handshake + time.Sleep(time.Millisecond * 250) + + log.Debugf("Connected to pubsub peer %s", pi.ID) + }(pi) + } + + wg.Wait() +} From e9b0864fe01c96441864539ff21d2d596c7a545a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 15:45:03 -0400 Subject: [PATCH 12/27] In get-latest tests wait a bit after connecting hosts so they have time to connect. --- getlatest_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/getlatest_test.go b/getlatest_test.go index 4cc31d3..ea80c2b 100644 --- a/getlatest_test.go +++ b/getlatest_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "testing" + "time" "github.com/libp2p/go-libp2p-core/host" ) @@ -36,6 +37,9 @@ func TestGetLatestProtocolTrip(t *testing.T) { hosts := newNetHosts(ctx, t, 2) connect(t, hosts[0], hosts[1]) + // wait for hosts to get connected + time.Sleep(time.Millisecond * 100) + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} h1 := newGetLatestProtocol(hosts[0], d1.Lookup) @@ -53,6 +57,9 @@ func TestGetLatestProtocolNil(t *testing.T) { hosts := newNetHosts(ctx, t, 2) connect(t, hosts[0], hosts[1]) + // wait for hosts to get connected + time.Sleep(time.Millisecond * 100) + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} h1 := newGetLatestProtocol(hosts[0], d1.Lookup) @@ -70,6 +77,9 @@ func TestGetLatestProtocolRepeated(t *testing.T) { hosts := newNetHosts(ctx, t, 2) connect(t, hosts[0], hosts[1]) + // wait for hosts to get connected + time.Sleep(time.Millisecond * 100) + d1 := &datastore{map[string][]byte{"key": []byte("value1")}} h1 := newGetLatestProtocol(hosts[0], d1.Lookup) From f85f2bc260683aa21b0a702ed8b1ba21cd1be6c6 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 18:44:49 -0400 Subject: [PATCH 13/27] Changed get-latest protocol to have responses with a status code. Wrapped protocol read/writes in contexts. A little refactoring. Protobuf Makefile builds on Windows too. --- getlatest.go | 108 ++++++++++++++++++++++++++++++-------------- getlatest_test.go | 56 +++++++++++++++++++---- pb/Makefile | 8 +++- pb/message.pb.go | 111 ++++++++++++++++++++++++++++------------------ pb/message.proto | 13 ++++-- pubsub.go | 4 +- 6 files changed, 208 insertions(+), 92 deletions(-) diff --git a/getlatest.go b/getlatest.go index 935e959..673f357 100644 --- a/getlatest.go +++ b/getlatest.go @@ -1,8 +1,8 @@ package namesys import ( - "bufio" "context" + "errors" "io" "time" @@ -17,27 +17,33 @@ import ( pb "github.com/libp2p/go-libp2p-pubsub-router/pb" ) +var GetLatestErr = errors.New("get-latest: received error") + type getLatestProtocol struct { + ctx context.Context host host.Host } -func newGetLatestProtocol(host host.Host, getLocal func(key string) ([]byte, error)) *getLatestProtocol { - p := &getLatestProtocol{host} +func newGetLatestProtocol(ctx context.Context, host host.Host, getLocal func(key string) ([]byte, error)) *getLatestProtocol { + p := &getLatestProtocol{ctx, host} host.SetStreamHandler(PSGetLatestProto, func(s network.Stream) { - p.Receive(s, getLocal) + p.receive(s, getLocal) }) return p } -func (p *getLatestProtocol) Receive(s network.Stream, getLocal func(key string) ([]byte, error)) { - r := ggio.NewDelimitedReader(s, 1<<20) +func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { msg := &pb.RequestLatest{} - if err := r.ReadMsg(msg); err != nil { + if err := readMsg(p.ctx, s, msg); err != nil { if err != io.EOF { - s.Reset() log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) + respProto := pb.RespondLatest{Status: pb.RespondLatest_ERR} + if err := writeMsg(p.ctx, s, &respProto); err != nil { + return + } + helpers.FullClose(s) } else { // Just be nice. They probably won't read this // but it doesn't hurt to send it. @@ -46,25 +52,22 @@ func (p *getLatestProtocol) Receive(s network.Stream, getLocal func(key string) return } - response, err := getLocal(*msg.Identifier) + response, err := getLocal(msg.Identifier) var respProto pb.RespondLatest - if err != nil || response == nil { - nodata := true - respProto = pb.RespondLatest{Nodata: &nodata} + if err != nil { + respProto = pb.RespondLatest{Status: pb.RespondLatest_NOT_FOUND} } else { respProto = pb.RespondLatest{Data: response} } - if err := writeBytes(s, &respProto); err != nil { - s.Reset() - log.Infof("error writing response to %s: %s", s.Conn().RemotePeer(), err) + if err := writeMsg(p.ctx, s, &respProto); err != nil { return } helpers.FullClose(s) } -func (p getLatestProtocol) Send(ctx context.Context, pid peer.ID, key string) ([]byte, error) { +func (p getLatestProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte, error) { peerCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() @@ -72,38 +75,75 @@ func (p getLatestProtocol) Send(ctx context.Context, pid peer.ID, key string) ([ if err != nil { return nil, err } - - if err := s.SetDeadline(time.Now().Add(time.Second * 5)); err != nil { - return nil, err - } - defer helpers.FullClose(s) - msg := pb.RequestLatest{Identifier: &key} + msg := &pb.RequestLatest{Identifier: key} - if err := writeBytes(s, &msg); err != nil { - s.Reset() + if err := writeMsg(ctx, s, msg); err != nil { return nil, err } - s.Close() - r := ggio.NewDelimitedReader(s, 1<<20) response := &pb.RespondLatest{} - if err := r.ReadMsg(response); err != nil { + if err := readMsg(ctx, s, response); err != nil { return nil, err } - return response.Data, nil + switch response.Status { + case pb.RespondLatest_SUCCESS: + return response.Data, nil + case pb.RespondLatest_NOT_FOUND: + return nil, nil + case pb.RespondLatest_ERR: + return nil, GetLatestErr + default: + return nil, errors.New("get-latest: received unknown status code") + } } -func writeBytes(w io.Writer, msg proto.Message) error { - bufw := bufio.NewWriter(w) - wc := ggio.NewDelimitedWriter(bufw) +func writeMsg(ctx context.Context, s network.Stream, msg proto.Message) error { + done := make(chan error) + go func() { + wc := ggio.NewDelimitedWriter(s) - if err := wc.WriteMsg(msg); err != nil { - return err + if err := wc.WriteMsg(msg); err != nil { + done <- err + return + } + + done <- nil + }() + + var retErr error + select { + case retErr = <-done: + case <-ctx.Done(): + retErr = ctx.Err() + } + + if retErr != nil { + s.Reset() + log.Infof("error writing response to %s: %s", s.Conn().RemotePeer(), retErr) } + return retErr +} - return bufw.Flush() +func readMsg(ctx context.Context, s network.Stream, msg proto.Message) error { + done := make(chan error) + go func() { + r := ggio.NewDelimitedReader(s, 1<<20) + if err := r.ReadMsg(msg); err != nil { + done <- err + return + } + done <- nil + }() + + select { + case err := <-done: + return err + case <-ctx.Done(): + s.Reset() + return ctx.Err() + } } diff --git a/getlatest_test.go b/getlatest_test.go index ea80c2b..41fe589 100644 --- a/getlatest_test.go +++ b/getlatest_test.go @@ -3,11 +3,15 @@ package namesys import ( "bytes" "context" + "encoding/binary" "errors" "testing" "time" + "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" + + pb "github.com/libp2p/go-libp2p-pubsub-router/pb" ) func connect(t *testing.T, a, b host.Host) { @@ -41,16 +45,16 @@ func TestGetLatestProtocolTrip(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{map[string][]byte{"key": []byte("value2")}} - h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) getLatest(t, ctx, h1, h2, "key", []byte("value2")) getLatest(t, ctx, h2, h1, "key", []byte("value1")) } -func TestGetLatestProtocolNil(t *testing.T) { +func TestGetLatestProtocolNotFound(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -61,15 +65,51 @@ func TestGetLatestProtocolNil(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{make(map[string][]byte)} - h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) getLatest(t, ctx, h1, h2, "key", nil) getLatest(t, ctx, h2, h1, "key", []byte("value1")) } +func TestGetLatestProtocolErr(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + hosts := newNetHosts(ctx, t, 2) + connect(t, hosts[0], hosts[1]) + + // wait for hosts to get connected + time.Sleep(time.Millisecond * 100) + + d1 := &datastore{make(map[string][]byte)} + h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) + + // bad send protocol to force an error + s, err := hosts[1].NewStream(ctx, h1.host.ID(), PSGetLatestProto) + if err != nil { + t.Fatal(err) + } + defer helpers.FullClose(s) + + buf := make([]byte, binary.MaxVarintLen64) + binary.PutUvarint(buf, ^uint64(0)) + if _, err := s.Write(buf); err != nil { + t.Fatal(err) + } + + response := &pb.RespondLatest{} + if err := readMsg(ctx, s, response); err != nil { + t.Fatal(err) + } + + if response.Status != pb.RespondLatest_ERR { + t.Fatal("should have received an error") + } +} + func TestGetLatestProtocolRepeated(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -81,10 +121,10 @@ func TestGetLatestProtocolRepeated(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(hosts[0], d1.Lookup) + h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{make(map[string][]byte)} - h2 := newGetLatestProtocol(hosts[1], d2.Lookup) + h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) for i := 0; i < 10; i++ { getLatest(t, ctx, h1, h2, "key", nil) @@ -94,7 +134,7 @@ func TestGetLatestProtocolRepeated(t *testing.T) { func getLatest(t *testing.T, ctx context.Context, requester *getLatestProtocol, responder *getLatestProtocol, key string, expected []byte) { - data, err := requester.Send(ctx, responder.host.ID(), key) + data, err := requester.Get(ctx, responder.host.ID(), key) if err != nil { t.Fatal(err) } diff --git a/pb/Makefile b/pb/Makefile index eb14b57..0353d62 100644 --- a/pb/Makefile +++ b/pb/Makefile @@ -1,10 +1,16 @@ PB = $(wildcard *.proto) GO = $(PB:.proto=.pb.go) +ifeq ($(OS),Windows_NT) + GOPATH_DELIMITER = \; +else + GOPATH_DELIMITER = : +endif + all: $(GO) %.pb.go: %.proto - protoc --proto_path=$(GOPATH)/src:. --gogofast_out=. $< + protoc --proto_path=$(GOPATH)/src$(GOPATH_DELIMITER). --gogofast_out=. $< clean: rm -f *.pb.go diff --git a/pb/message.pb.go b/pb/message.pb.go index 91e88fe..f728d34 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -21,8 +21,36 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +type RespondLatest_StatusCode int32 + +const ( + RespondLatest_SUCCESS RespondLatest_StatusCode = 0 + RespondLatest_NOT_FOUND RespondLatest_StatusCode = 1 + RespondLatest_ERR RespondLatest_StatusCode = 2 +) + +var RespondLatest_StatusCode_name = map[int32]string{ + 0: "SUCCESS", + 1: "NOT_FOUND", + 2: "ERR", +} + +var RespondLatest_StatusCode_value = map[string]int32{ + "SUCCESS": 0, + "NOT_FOUND": 1, + "ERR": 2, +} + +func (x RespondLatest_StatusCode) String() string { + return proto.EnumName(RespondLatest_StatusCode_name, int32(x)) +} + +func (RespondLatest_StatusCode) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{1, 0} +} + type RequestLatest struct { - Identifier *string `protobuf:"bytes,1,opt,name=identifier" json:"identifier,omitempty"` + Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -62,18 +90,18 @@ func (m *RequestLatest) XXX_DiscardUnknown() { var xxx_messageInfo_RequestLatest proto.InternalMessageInfo func (m *RequestLatest) GetIdentifier() string { - if m != nil && m.Identifier != nil { - return *m.Identifier + if m != nil { + return m.Identifier } return "" } type RespondLatest struct { - Data []byte `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` - Nodata *bool `protobuf:"varint,2,opt,name=nodata" json:"nodata,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Status RespondLatest_StatusCode `protobuf:"varint,2,opt,name=status,proto3,enum=namesys.pb.RespondLatest_StatusCode" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *RespondLatest) Reset() { *m = RespondLatest{} } @@ -116,14 +144,15 @@ func (m *RespondLatest) GetData() []byte { return nil } -func (m *RespondLatest) GetNodata() bool { - if m != nil && m.Nodata != nil { - return *m.Nodata +func (m *RespondLatest) GetStatus() RespondLatest_StatusCode { + if m != nil { + return m.Status } - return false + return RespondLatest_SUCCESS } func init() { + proto.RegisterEnum("namesys.pb.RespondLatest_StatusCode", RespondLatest_StatusCode_name, RespondLatest_StatusCode_value) proto.RegisterType((*RequestLatest)(nil), "namesys.pb.RequestLatest") proto.RegisterType((*RespondLatest)(nil), "namesys.pb.RespondLatest") } @@ -131,17 +160,21 @@ func init() { func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 145 bytes of a gzipped FileDescriptorProto + // 214 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, - 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0xb2, 0x06, - 0x69, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0x6a, 0x10, 0xe2, 0x62, 0x49, 0x49, 0x2c, 0x49, 0x04, - 0x2b, 0xe5, 0x09, 0x02, 0xb3, 0x85, 0xc4, 0xb8, 0xd8, 0xf2, 0xf2, 0xc1, 0xa2, 0x4c, 0x0a, 0x8c, - 0x1a, 0x1c, 0x41, 0x50, 0x9e, 0x13, 0xcf, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, - 0x78, 0x24, 0xc7, 0x08, 0x08, 0x00, 0x00, 0xff, 0xff, 0x21, 0x3f, 0xdb, 0xfb, 0x97, 0x00, 0x00, - 0x00, + 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0x9a, 0xc2, + 0x08, 0xd2, 0x51, 0x5c, 0x90, 0x9f, 0x97, 0x02, 0xd5, 0x21, 0xc4, 0xc5, 0x92, 0x92, 0x58, 0x92, + 0x08, 0x56, 0xcb, 0x13, 0x04, 0x66, 0x0b, 0xd9, 0x70, 0xb1, 0x15, 0x97, 0x24, 0x96, 0x94, 0x16, + 0x4b, 0x30, 0x29, 0x30, 0x6a, 0xf0, 0x19, 0xa9, 0xe8, 0x21, 0xec, 0xd4, 0x43, 0xd1, 0xae, 0x17, + 0x0c, 0x56, 0xe7, 0x9c, 0x9f, 0x92, 0x1a, 0x04, 0xd5, 0xa3, 0x64, 0xc8, 0xc5, 0x85, 0x10, 0x15, + 0xe2, 0xe6, 0x62, 0x0f, 0x0e, 0x75, 0x76, 0x76, 0x0d, 0x0e, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, + 0xf4, 0xf3, 0x0f, 0x89, 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x14, 0x62, 0xe7, 0x62, 0x76, + 0x0d, 0x0a, 0x12, 0x60, 0x72, 0xe2, 0x39, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, + 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0x1e, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x22, + 0x78, 0xb9, 0xf9, 0x00, 0x00, 0x00, } func (m *RequestLatest) Marshal() (dAtA []byte, err error) { @@ -159,11 +192,11 @@ func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Identifier != nil { + if len(m.Identifier) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintMessage(dAtA, i, uint64(len(*m.Identifier))) - i += copy(dAtA[i:], *m.Identifier) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Identifier))) + i += copy(dAtA[i:], m.Identifier) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -186,21 +219,16 @@ func (m *RespondLatest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.Data != nil { + if len(m.Data) > 0 { dAtA[i] = 0xa i++ i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) i += copy(dAtA[i:], m.Data) } - if m.Nodata != nil { + if m.Status != 0 { dAtA[i] = 0x10 i++ - if *m.Nodata { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ + i = encodeVarintMessage(dAtA, i, uint64(m.Status)) } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) @@ -223,8 +251,8 @@ func (m *RequestLatest) Size() (n int) { } var l int _ = l - if m.Identifier != nil { - l = len(*m.Identifier) + l = len(m.Identifier) + if l > 0 { n += 1 + l + sovMessage(uint64(l)) } if m.XXX_unrecognized != nil { @@ -239,12 +267,12 @@ func (m *RespondLatest) Size() (n int) { } var l int _ = l - if m.Data != nil { - l = len(m.Data) + l = len(m.Data) + if l > 0 { n += 1 + l + sovMessage(uint64(l)) } - if m.Nodata != nil { - n += 2 + if m.Status != 0 { + n += 1 + sovMessage(uint64(m.Status)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -324,8 +352,7 @@ func (m *RequestLatest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - s := string(dAtA[iNdEx:postIndex]) - m.Identifier = &s + m.Identifier = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -417,9 +444,9 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Nodata", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) } - var v int + m.Status = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowMessage @@ -429,13 +456,11 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + m.Status |= RespondLatest_StatusCode(b&0x7F) << shift if b < 0x80 { break } } - b := bool(v != 0) - m.Nodata = &b default: iNdEx = preIndex skippy, err := skipMessage(dAtA[iNdEx:]) diff --git a/pb/message.proto b/pb/message.proto index 4a04228..1ec433d 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -1,12 +1,17 @@ -syntax = "proto2"; +syntax = "proto3"; package namesys.pb; message RequestLatest { - optional string identifier = 1; + string identifier = 1; } message RespondLatest { - optional bytes data = 1; - optional bool nodata = 2; + bytes data = 1; + StatusCode status = 2; + enum StatusCode { + SUCCESS = 0; + NOT_FOUND = 1; + ERR = 2; + } } \ No newline at end of file diff --git a/pubsub.go b/pubsub.go index 79571ac..c9c737a 100644 --- a/pubsub.go +++ b/pubsub.go @@ -89,7 +89,7 @@ func NewPubsubValueStore(ctx context.Context, host host.Host, cr routing.Content Validator: validator, } - psValueStore.getLatest = newGetLatestProtocol(host, psValueStore.getLocal) + psValueStore.getLatest = newGetLatestProtocol(ctx, host, psValueStore.getLocal) go psValueStore.rebroadcast(ctx) @@ -476,7 +476,7 @@ func (p *PubsubValueStore) handleNewPeer(ctx context.Context, sub *pubsub.Subscr } } - return p.getLatest.Send(ctx, pid, key) + return p.getLatest.Get(ctx, pid, key) } func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { From 97562631bdf4688ce2aa21cd5122b6af6e943534 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 19:18:48 -0400 Subject: [PATCH 14/27] get-latest responds with ERR message even when sender sends an incorrect EOF --- getlatest.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/getlatest.go b/getlatest.go index 673f357..13874b6 100644 --- a/getlatest.go +++ b/getlatest.go @@ -3,7 +3,6 @@ package namesys import ( "context" "errors" - "io" "time" ggio "github.com/gogo/protobuf/io" @@ -37,18 +36,12 @@ func newGetLatestProtocol(ctx context.Context, host host.Host, getLocal func(key func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { msg := &pb.RequestLatest{} if err := readMsg(p.ctx, s, msg); err != nil { - if err != io.EOF { - log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) - respProto := pb.RespondLatest{Status: pb.RespondLatest_ERR} - if err := writeMsg(p.ctx, s, &respProto); err != nil { - return - } - helpers.FullClose(s) - } else { - // Just be nice. They probably won't read this - // but it doesn't hurt to send it. - s.Close() + log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) + respProto := pb.RespondLatest{Status: pb.RespondLatest_ERR} + if err := writeMsg(p.ctx, s, &respProto); err != nil { + return } + helpers.FullClose(s) return } From a4d4d6cae76ac4668bdf973a9d44b4fb89fa4803 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 19:30:23 -0400 Subject: [PATCH 15/27] Removed ERR from protobuf and we just reset the stream when we encounter errors in the protocol. --- getlatest.go | 13 +++---------- getlatest_test.go | 40 ---------------------------------------- pb/message.pb.go | 24 ++++++++++-------------- pb/message.proto | 1 - 4 files changed, 13 insertions(+), 65 deletions(-) diff --git a/getlatest.go b/getlatest.go index 13874b6..2a5c041 100644 --- a/getlatest.go +++ b/getlatest.go @@ -16,8 +16,6 @@ import ( pb "github.com/libp2p/go-libp2p-pubsub-router/pb" ) -var GetLatestErr = errors.New("get-latest: received error") - type getLatestProtocol struct { ctx context.Context host host.Host @@ -34,14 +32,12 @@ func newGetLatestProtocol(ctx context.Context, host host.Host, getLocal func(key } func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { + defer helpers.FullClose(s) + msg := &pb.RequestLatest{} if err := readMsg(p.ctx, s, msg); err != nil { log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) - respProto := pb.RespondLatest{Status: pb.RespondLatest_ERR} - if err := writeMsg(p.ctx, s, &respProto); err != nil { - return - } - helpers.FullClose(s) + s.Reset() return } @@ -57,7 +53,6 @@ func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) if err := writeMsg(p.ctx, s, &respProto); err != nil { return } - helpers.FullClose(s) } func (p getLatestProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte, error) { @@ -87,8 +82,6 @@ func (p getLatestProtocol) Get(ctx context.Context, pid peer.ID, key string) ([] return response.Data, nil case pb.RespondLatest_NOT_FOUND: return nil, nil - case pb.RespondLatest_ERR: - return nil, GetLatestErr default: return nil, errors.New("get-latest: received unknown status code") } diff --git a/getlatest_test.go b/getlatest_test.go index 41fe589..964ec3e 100644 --- a/getlatest_test.go +++ b/getlatest_test.go @@ -3,15 +3,11 @@ package namesys import ( "bytes" "context" - "encoding/binary" "errors" "testing" "time" - "github.com/libp2p/go-libp2p-core/helpers" "github.com/libp2p/go-libp2p-core/host" - - pb "github.com/libp2p/go-libp2p-pubsub-router/pb" ) func connect(t *testing.T, a, b host.Host) { @@ -74,42 +70,6 @@ func TestGetLatestProtocolNotFound(t *testing.T) { getLatest(t, ctx, h2, h1, "key", []byte("value1")) } -func TestGetLatestProtocolErr(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - hosts := newNetHosts(ctx, t, 2) - connect(t, hosts[0], hosts[1]) - - // wait for hosts to get connected - time.Sleep(time.Millisecond * 100) - - d1 := &datastore{make(map[string][]byte)} - h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) - - // bad send protocol to force an error - s, err := hosts[1].NewStream(ctx, h1.host.ID(), PSGetLatestProto) - if err != nil { - t.Fatal(err) - } - defer helpers.FullClose(s) - - buf := make([]byte, binary.MaxVarintLen64) - binary.PutUvarint(buf, ^uint64(0)) - if _, err := s.Write(buf); err != nil { - t.Fatal(err) - } - - response := &pb.RespondLatest{} - if err := readMsg(ctx, s, response); err != nil { - t.Fatal(err) - } - - if response.Status != pb.RespondLatest_ERR { - t.Fatal("should have received an error") - } -} - func TestGetLatestProtocolRepeated(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/pb/message.pb.go b/pb/message.pb.go index f728d34..f1037bd 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -26,19 +26,16 @@ type RespondLatest_StatusCode int32 const ( RespondLatest_SUCCESS RespondLatest_StatusCode = 0 RespondLatest_NOT_FOUND RespondLatest_StatusCode = 1 - RespondLatest_ERR RespondLatest_StatusCode = 2 ) var RespondLatest_StatusCode_name = map[int32]string{ 0: "SUCCESS", 1: "NOT_FOUND", - 2: "ERR", } var RespondLatest_StatusCode_value = map[string]int32{ "SUCCESS": 0, "NOT_FOUND": 1, - "ERR": 2, } func (x RespondLatest_StatusCode) String() string { @@ -160,21 +157,20 @@ func init() { func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 214 bytes of a gzipped FileDescriptorProto + // 205 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, - 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0x9a, 0xc2, - 0x08, 0xd2, 0x51, 0x5c, 0x90, 0x9f, 0x97, 0x02, 0xd5, 0x21, 0xc4, 0xc5, 0x92, 0x92, 0x58, 0x92, - 0x08, 0x56, 0xcb, 0x13, 0x04, 0x66, 0x0b, 0xd9, 0x70, 0xb1, 0x15, 0x97, 0x24, 0x96, 0x94, 0x16, - 0x4b, 0x30, 0x29, 0x30, 0x6a, 0xf0, 0x19, 0xa9, 0xe8, 0x21, 0xec, 0xd4, 0x43, 0xd1, 0xae, 0x17, - 0x0c, 0x56, 0xe7, 0x9c, 0x9f, 0x92, 0x1a, 0x04, 0xd5, 0xa3, 0x64, 0xc8, 0xc5, 0x85, 0x10, 0x15, - 0xe2, 0xe6, 0x62, 0x0f, 0x0e, 0x75, 0x76, 0x76, 0x0d, 0x0e, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, - 0xf4, 0xf3, 0x0f, 0x89, 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x14, 0x62, 0xe7, 0x62, 0x76, - 0x0d, 0x0a, 0x12, 0x60, 0x72, 0xe2, 0x39, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, - 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0x1e, 0x35, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x22, - 0x78, 0xb9, 0xf9, 0x00, 0x00, 0x00, + 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0xea, 0x66, + 0x04, 0xe9, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0xea, 0x10, 0xe2, 0x62, 0x49, 0x49, 0x2c, 0x49, + 0x04, 0xab, 0xe5, 0x09, 0x02, 0xb3, 0x85, 0x6c, 0xb8, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, 0x4a, 0x8b, + 0x25, 0x98, 0x14, 0x18, 0x35, 0xf8, 0x8c, 0x54, 0xf4, 0x10, 0x76, 0xea, 0xa1, 0x68, 0xd7, 0x0b, + 0x06, 0xab, 0x73, 0xce, 0x4f, 0x49, 0x0d, 0x82, 0xea, 0x51, 0xd2, 0xe0, 0xe2, 0x42, 0x88, 0x0a, + 0x71, 0x73, 0xb1, 0x07, 0x87, 0x3a, 0x3b, 0xbb, 0x06, 0x07, 0x0b, 0x30, 0x08, 0xf1, 0x72, 0x71, + 0xfa, 0xf9, 0x87, 0xc4, 0xbb, 0xf9, 0x87, 0xfa, 0xb9, 0x08, 0x30, 0x3a, 0xf1, 0x9c, 0x78, 0x24, + 0xc7, 0x78, 0xe1, 0x91, 0x1c, 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0xff, 0x19, 0x03, + 0x02, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x3c, 0x29, 0xab, 0xf0, 0x00, 0x00, 0x00, } func (m *RequestLatest) Marshal() (dAtA []byte, err error) { diff --git a/pb/message.proto b/pb/message.proto index 1ec433d..c0d6c5c 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -12,6 +12,5 @@ message RespondLatest { enum StatusCode { SUCCESS = 0; NOT_FOUND = 1; - ERR = 2; } } \ No newline at end of file From 97ac549d9542eabc28dca6073f8f2b31ab557eff Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Aug 2019 19:38:16 -0400 Subject: [PATCH 16/27] protobuf Makefile supports spaces in path name --- pb/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb/Makefile b/pb/Makefile index 0353d62..172ca4b 100644 --- a/pb/Makefile +++ b/pb/Makefile @@ -2,7 +2,7 @@ PB = $(wildcard *.proto) GO = $(PB:.proto=.pb.go) ifeq ($(OS),Windows_NT) - GOPATH_DELIMITER = \; + GOPATH_DELIMITER = ; else GOPATH_DELIMITER = : endif @@ -10,7 +10,7 @@ endif all: $(GO) %.pb.go: %.proto - protoc --proto_path=$(GOPATH)/src$(GOPATH_DELIMITER). --gogofast_out=. $< + protoc --proto_path="$(GOPATH)/src$(GOPATH_DELIMITER)." --gogofast_out=. $< clean: rm -f *.pb.go From b0eeb77707edbfaf23bd56f0fa8a06b3c2e1bee9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Sat, 10 Aug 2019 23:40:57 -0400 Subject: [PATCH 17/27] fixed potential goroutine leak. switched order of protobuf fields. --- getlatest.go | 4 +-- pb/message.pb.go | 88 ++++++++++++++++++++++++------------------------ pb/message.proto | 4 +-- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/getlatest.go b/getlatest.go index 2a5c041..e8fcc3a 100644 --- a/getlatest.go +++ b/getlatest.go @@ -88,7 +88,7 @@ func (p getLatestProtocol) Get(ctx context.Context, pid peer.ID, key string) ([] } func writeMsg(ctx context.Context, s network.Stream, msg proto.Message) error { - done := make(chan error) + done := make(chan error, 1) go func() { wc := ggio.NewDelimitedWriter(s) @@ -115,7 +115,7 @@ func writeMsg(ctx context.Context, s network.Stream, msg proto.Message) error { } func readMsg(ctx context.Context, s network.Stream, msg proto.Message) error { - done := make(chan error) + done := make(chan error, 1) go func() { r := ggio.NewDelimitedReader(s, 1<<20) if err := r.ReadMsg(msg); err != nil { diff --git a/pb/message.pb.go b/pb/message.pb.go index f1037bd..a0a72d8 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -94,8 +94,8 @@ func (m *RequestLatest) GetIdentifier() string { } type RespondLatest struct { - Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` - Status RespondLatest_StatusCode `protobuf:"varint,2,opt,name=status,proto3,enum=namesys.pb.RespondLatest_StatusCode" json:"status,omitempty"` + Status RespondLatest_StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=namesys.pb.RespondLatest_StatusCode" json:"status,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -134,18 +134,18 @@ func (m *RespondLatest) XXX_DiscardUnknown() { var xxx_messageInfo_RespondLatest proto.InternalMessageInfo -func (m *RespondLatest) GetData() []byte { +func (m *RespondLatest) GetStatus() RespondLatest_StatusCode { if m != nil { - return m.Data + return m.Status } - return nil + return RespondLatest_SUCCESS } -func (m *RespondLatest) GetStatus() RespondLatest_StatusCode { +func (m *RespondLatest) GetData() []byte { if m != nil { - return m.Status + return m.Data } - return RespondLatest_SUCCESS + return nil } func init() { @@ -163,14 +163,14 @@ var fileDescriptor_33c57e4bae7b9afd = []byte{ 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0xea, 0x66, - 0x04, 0xe9, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0xea, 0x10, 0xe2, 0x62, 0x49, 0x49, 0x2c, 0x49, - 0x04, 0xab, 0xe5, 0x09, 0x02, 0xb3, 0x85, 0x6c, 0xb8, 0xd8, 0x8a, 0x4b, 0x12, 0x4b, 0x4a, 0x8b, - 0x25, 0x98, 0x14, 0x18, 0x35, 0xf8, 0x8c, 0x54, 0xf4, 0x10, 0x76, 0xea, 0xa1, 0x68, 0xd7, 0x0b, - 0x06, 0xab, 0x73, 0xce, 0x4f, 0x49, 0x0d, 0x82, 0xea, 0x51, 0xd2, 0xe0, 0xe2, 0x42, 0x88, 0x0a, - 0x71, 0x73, 0xb1, 0x07, 0x87, 0x3a, 0x3b, 0xbb, 0x06, 0x07, 0x0b, 0x30, 0x08, 0xf1, 0x72, 0x71, - 0xfa, 0xf9, 0x87, 0xc4, 0xbb, 0xf9, 0x87, 0xfa, 0xb9, 0x08, 0x30, 0x3a, 0xf1, 0x9c, 0x78, 0x24, - 0xc7, 0x78, 0xe1, 0x91, 0x1c, 0xe3, 0x83, 0x47, 0x72, 0x8c, 0x49, 0x6c, 0x60, 0xff, 0x19, 0x03, - 0x02, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x3c, 0x29, 0xab, 0xf0, 0x00, 0x00, 0x00, + 0x04, 0xe9, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0xea, 0xb0, 0xe1, 0x62, 0x2b, 0x2e, 0x49, 0x2c, + 0x29, 0x2d, 0x06, 0xab, 0xe6, 0x33, 0x52, 0xd1, 0x43, 0x98, 0xaf, 0x87, 0xa2, 0x54, 0x2f, 0x18, + 0xac, 0xce, 0x39, 0x3f, 0x25, 0x35, 0x08, 0xaa, 0x47, 0x48, 0x88, 0x8b, 0x25, 0x25, 0xb1, 0x24, + 0x51, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x56, 0xd2, 0xe0, 0xe2, 0x42, 0xa8, 0x14, + 0xe2, 0xe6, 0x62, 0x0f, 0x0e, 0x75, 0x76, 0x76, 0x0d, 0x0e, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, + 0xf4, 0xf3, 0x0f, 0x89, 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x74, 0xe2, 0x39, 0xf1, 0x48, + 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0xfe, 0x33, 0x06, + 0x04, 0x00, 0x00, 0xff, 0xff, 0x2d, 0x05, 0x6e, 0x5c, 0xf0, 0x00, 0x00, 0x00, } func (m *RequestLatest) Marshal() (dAtA []byte, err error) { @@ -215,17 +215,17 @@ func (m *RespondLatest) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Status != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintMessage(dAtA, i, uint64(m.Status)) + } if len(m.Data) > 0 { - dAtA[i] = 0xa + dAtA[i] = 0x12 i++ i = encodeVarintMessage(dAtA, i, uint64(len(m.Data))) i += copy(dAtA[i:], m.Data) } - if m.Status != 0 { - dAtA[i] = 0x10 - i++ - i = encodeVarintMessage(dAtA, i, uint64(m.Status)) - } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -263,13 +263,13 @@ func (m *RespondLatest) Size() (n int) { } var l int _ = l + if m.Status != 0 { + n += 1 + sovMessage(uint64(m.Status)) + } l = len(m.Data) if l > 0 { n += 1 + l + sovMessage(uint64(l)) } - if m.Status != 0 { - n += 1 + sovMessage(uint64(m.Status)) - } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -405,6 +405,25 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + m.Status = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Status |= RespondLatest_StatusCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) } @@ -438,25 +457,6 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { m.Data = []byte{} } iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) - } - m.Status = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowMessage - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Status |= RespondLatest_StatusCode(b&0x7F) << shift - if b < 0x80 { - break - } - } default: iNdEx = preIndex skippy, err := skipMessage(dAtA[iNdEx:]) diff --git a/pb/message.proto b/pb/message.proto index c0d6c5c..0985f06 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -7,10 +7,10 @@ message RequestLatest { } message RespondLatest { - bytes data = 1; - StatusCode status = 2; + StatusCode status = 1; enum StatusCode { SUCCESS = 0; NOT_FOUND = 1; } + bytes data = 2; } \ No newline at end of file From 4fbc97b69e0e558f79bf55ef8e1020dac602d5a0 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Sat, 10 Aug 2019 23:53:30 -0400 Subject: [PATCH 18/27] Small Makefile refactor --- pb/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pb/Makefile b/pb/Makefile index 172ca4b..38a4972 100644 --- a/pb/Makefile +++ b/pb/Makefile @@ -2,15 +2,15 @@ PB = $(wildcard *.proto) GO = $(PB:.proto=.pb.go) ifeq ($(OS),Windows_NT) - GOPATH_DELIMITER = ; + LIST_SEPARATOR = ; else - GOPATH_DELIMITER = : + LIST_SEPARATOR = : endif all: $(GO) %.pb.go: %.proto - protoc --proto_path="$(GOPATH)/src$(GOPATH_DELIMITER)." --gogofast_out=. $< + protoc --proto_path="$(GOPATH)/src$(LIST_SEPARATOR)." --gogofast_out=. $< clean: rm -f *.pb.go From aa737d11b292d687df68aa97252763d88de936d5 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 12 Aug 2019 17:34:53 -0400 Subject: [PATCH 19/27] changed get-latest protocol to be called fetch. some refactoring --- getlatest.go | 17 ++++++++++------- getlatest_test.go | 34 +++++++++++++++++----------------- pubsub.go | 11 ++++------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/getlatest.go b/getlatest.go index e8fcc3a..a20cfbc 100644 --- a/getlatest.go +++ b/getlatest.go @@ -12,26 +12,29 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" pb "github.com/libp2p/go-libp2p-pubsub-router/pb" ) -type getLatestProtocol struct { +const FetchProtoID = protocol.ID("/libp2p/fetch/0.0.1") + +type fetchProtocol struct { ctx context.Context host host.Host } -func newGetLatestProtocol(ctx context.Context, host host.Host, getLocal func(key string) ([]byte, error)) *getLatestProtocol { - p := &getLatestProtocol{ctx, host} +func newFetchProtocol(ctx context.Context, host host.Host, getLocal func(key string) ([]byte, error)) *fetchProtocol { + p := &fetchProtocol{ctx, host} - host.SetStreamHandler(PSGetLatestProto, func(s network.Stream) { + host.SetStreamHandler(FetchProtoID, func(s network.Stream) { p.receive(s, getLocal) }) return p } -func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { +func (p *fetchProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { defer helpers.FullClose(s) msg := &pb.RequestLatest{} @@ -55,11 +58,11 @@ func (p *getLatestProtocol) receive(s network.Stream, getLocal func(key string) } } -func (p getLatestProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte, error) { +func (p fetchProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte, error) { peerCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() - s, err := p.host.NewStream(peerCtx, pid, PSGetLatestProto) + s, err := p.host.NewStream(peerCtx, pid, FetchProtoID) if err != nil { return nil, err } diff --git a/getlatest_test.go b/getlatest_test.go index 964ec3e..3506d85 100644 --- a/getlatest_test.go +++ b/getlatest_test.go @@ -30,7 +30,7 @@ func (d *datastore) Lookup(key string) ([]byte, error) { return v, nil } -func TestGetLatestProtocolTrip(t *testing.T) { +func TestFetchProtocolTrip(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -41,16 +41,16 @@ func TestGetLatestProtocolTrip(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) + h1 := newFetchProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{map[string][]byte{"key": []byte("value2")}} - h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) + h2 := newFetchProtocol(ctx, hosts[1], d2.Lookup) - getLatest(t, ctx, h1, h2, "key", []byte("value2")) - getLatest(t, ctx, h2, h1, "key", []byte("value1")) + fetchCheck(t, ctx, h1, h2, "key", []byte("value2")) + fetchCheck(t, ctx, h2, h1, "key", []byte("value1")) } -func TestGetLatestProtocolNotFound(t *testing.T) { +func TestFetchProtocolNotFound(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -61,16 +61,16 @@ func TestGetLatestProtocolNotFound(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) + h1 := newFetchProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{make(map[string][]byte)} - h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) + h2 := newFetchProtocol(ctx, hosts[1], d2.Lookup) - getLatest(t, ctx, h1, h2, "key", nil) - getLatest(t, ctx, h2, h1, "key", []byte("value1")) + fetchCheck(t, ctx, h1, h2, "key", nil) + fetchCheck(t, ctx, h2, h1, "key", []byte("value1")) } -func TestGetLatestProtocolRepeated(t *testing.T) { +func TestFetchProtocolRepeated(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -81,19 +81,19 @@ func TestGetLatestProtocolRepeated(t *testing.T) { time.Sleep(time.Millisecond * 100) d1 := &datastore{map[string][]byte{"key": []byte("value1")}} - h1 := newGetLatestProtocol(ctx, hosts[0], d1.Lookup) + h1 := newFetchProtocol(ctx, hosts[0], d1.Lookup) d2 := &datastore{make(map[string][]byte)} - h2 := newGetLatestProtocol(ctx, hosts[1], d2.Lookup) + h2 := newFetchProtocol(ctx, hosts[1], d2.Lookup) for i := 0; i < 10; i++ { - getLatest(t, ctx, h1, h2, "key", nil) - getLatest(t, ctx, h2, h1, "key", []byte("value1")) + fetchCheck(t, ctx, h1, h2, "key", nil) + fetchCheck(t, ctx, h2, h1, "key", []byte("value1")) } } -func getLatest(t *testing.T, ctx context.Context, - requester *getLatestProtocol, responder *getLatestProtocol, key string, expected []byte) { +func fetchCheck(t *testing.T, ctx context.Context, + requester *fetchProtocol, responder *fetchProtocol, key string, expected []byte) { data, err := requester.Get(ctx, responder.host.ID(), key) if err != nil { t.Fatal(err) diff --git a/pubsub.go b/pubsub.go index c9c737a..0a69440 100644 --- a/pubsub.go +++ b/pubsub.go @@ -11,7 +11,6 @@ import ( "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-core/routing" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -26,8 +25,6 @@ import ( var log = logging.Logger("pubsub-valuestore") -const PSGetLatestProto = protocol.ID("/pubsub-get-latest/0.0.1") - type watchGroup struct { // Note: this chan must be buffered, see notifyWatchers listeners map[chan []byte]struct{} @@ -39,8 +36,8 @@ type PubsubValueStore struct { cr routing.ContentRouting ps *pubsub.PubSub - host host.Host - getLatest *getLatestProtocol + host host.Host + fetch *fetchProtocol rebroadcastInitialDelay time.Duration rebroadcastInterval time.Duration @@ -89,7 +86,7 @@ func NewPubsubValueStore(ctx context.Context, host host.Host, cr routing.Content Validator: validator, } - psValueStore.getLatest = newGetLatestProtocol(ctx, host, psValueStore.getLocal) + psValueStore.fetch = newFetchProtocol(ctx, host, psValueStore.getLocal) go psValueStore.rebroadcast(ctx) @@ -476,7 +473,7 @@ func (p *PubsubValueStore) handleNewPeer(ctx context.Context, sub *pubsub.Subscr } } - return p.getLatest.Get(ctx, pid, key) + return p.fetch.Get(ctx, pid, key) } func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { From ded823d18018e44b60d10acf414b5b15ff88de80 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 12 Aug 2019 17:36:31 -0400 Subject: [PATCH 20/27] renamed protocol files to match protocol rename --- getlatest.go => fetch.go | 0 getlatest_test.go => fetch_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename getlatest.go => fetch.go (100%) rename getlatest_test.go => fetch_test.go (100%) diff --git a/getlatest.go b/fetch.go similarity index 100% rename from getlatest.go rename to fetch.go diff --git a/getlatest_test.go b/fetch_test.go similarity index 100% rename from getlatest_test.go rename to fetch_test.go From 76d8fcaaf7c2761d23e79ba773138cfb90433138 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 14 Aug 2019 13:56:53 -0400 Subject: [PATCH 21/27] made function passed into the fetch protocol a typedef --- fetch.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/fetch.go b/fetch.go index a20cfbc..7e48ec4 100644 --- a/fetch.go +++ b/fetch.go @@ -24,17 +24,19 @@ type fetchProtocol struct { host host.Host } -func newFetchProtocol(ctx context.Context, host host.Host, getLocal func(key string) ([]byte, error)) *fetchProtocol { +type getValue func(key string) ([]byte, error) + +func newFetchProtocol(ctx context.Context, host host.Host, getData getValue) *fetchProtocol { p := &fetchProtocol{ctx, host} host.SetStreamHandler(FetchProtoID, func(s network.Stream) { - p.receive(s, getLocal) + p.receive(s, getData) }) return p } -func (p *fetchProtocol) receive(s network.Stream, getLocal func(key string) ([]byte, error)) { +func (p *fetchProtocol) receive(s network.Stream, getData getValue) { defer helpers.FullClose(s) msg := &pb.RequestLatest{} @@ -44,7 +46,7 @@ func (p *fetchProtocol) receive(s network.Stream, getLocal func(key string) ([]b return } - response, err := getLocal(msg.Identifier) + response, err := getData(msg.Identifier) var respProto pb.RespondLatest if err != nil { From 1913974e71cc4c4ceeba04fe7f199fe9744b4008 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 15 Aug 2019 08:43:01 -0400 Subject: [PATCH 22/27] renames in the fetch protobufs --- fetch.go | 16 +++--- pb/message.pb.go | 146 +++++++++++++++++++++++------------------------ pb/message.proto | 6 +- 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/fetch.go b/fetch.go index 7e48ec4..6325051 100644 --- a/fetch.go +++ b/fetch.go @@ -39,7 +39,7 @@ func newFetchProtocol(ctx context.Context, host host.Host, getData getValue) *fe func (p *fetchProtocol) receive(s network.Stream, getData getValue) { defer helpers.FullClose(s) - msg := &pb.RequestLatest{} + msg := &pb.FetchRequest{} if err := readMsg(p.ctx, s, msg); err != nil { log.Infof("error reading request from %s: %s", s.Conn().RemotePeer(), err) s.Reset() @@ -47,12 +47,12 @@ func (p *fetchProtocol) receive(s network.Stream, getData getValue) { } response, err := getData(msg.Identifier) - var respProto pb.RespondLatest + var respProto pb.FetchResponse if err != nil { - respProto = pb.RespondLatest{Status: pb.RespondLatest_NOT_FOUND} + respProto = pb.FetchResponse{Status: pb.FetchResponse_NOT_FOUND} } else { - respProto = pb.RespondLatest{Data: response} + respProto = pb.FetchResponse{Data: response} } if err := writeMsg(p.ctx, s, &respProto); err != nil { @@ -70,22 +70,22 @@ func (p fetchProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte } defer helpers.FullClose(s) - msg := &pb.RequestLatest{Identifier: key} + msg := &pb.FetchRequest{Identifier: key} if err := writeMsg(ctx, s, msg); err != nil { return nil, err } s.Close() - response := &pb.RespondLatest{} + response := &pb.FetchResponse{} if err := readMsg(ctx, s, response); err != nil { return nil, err } switch response.Status { - case pb.RespondLatest_SUCCESS: + case pb.FetchResponse_OK: return response.Data, nil - case pb.RespondLatest_NOT_FOUND: + case pb.FetchResponse_NOT_FOUND: return nil, nil default: return nil, errors.New("get-latest: received unknown status code") diff --git a/pb/message.pb.go b/pb/message.pb.go index a0a72d8..2284380 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -21,50 +21,50 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package -type RespondLatest_StatusCode int32 +type FetchResponse_StatusCode int32 const ( - RespondLatest_SUCCESS RespondLatest_StatusCode = 0 - RespondLatest_NOT_FOUND RespondLatest_StatusCode = 1 + FetchResponse_OK FetchResponse_StatusCode = 0 + FetchResponse_NOT_FOUND FetchResponse_StatusCode = 1 ) -var RespondLatest_StatusCode_name = map[int32]string{ - 0: "SUCCESS", +var FetchResponse_StatusCode_name = map[int32]string{ + 0: "OK", 1: "NOT_FOUND", } -var RespondLatest_StatusCode_value = map[string]int32{ - "SUCCESS": 0, +var FetchResponse_StatusCode_value = map[string]int32{ + "OK": 0, "NOT_FOUND": 1, } -func (x RespondLatest_StatusCode) String() string { - return proto.EnumName(RespondLatest_StatusCode_name, int32(x)) +func (x FetchResponse_StatusCode) String() string { + return proto.EnumName(FetchResponse_StatusCode_name, int32(x)) } -func (RespondLatest_StatusCode) EnumDescriptor() ([]byte, []int) { +func (FetchResponse_StatusCode) EnumDescriptor() ([]byte, []int) { return fileDescriptor_33c57e4bae7b9afd, []int{1, 0} } -type RequestLatest struct { +type FetchRequest struct { Identifier string `protobuf:"bytes,1,opt,name=identifier,proto3" json:"identifier,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *RequestLatest) Reset() { *m = RequestLatest{} } -func (m *RequestLatest) String() string { return proto.CompactTextString(m) } -func (*RequestLatest) ProtoMessage() {} -func (*RequestLatest) Descriptor() ([]byte, []int) { +func (m *FetchRequest) Reset() { *m = FetchRequest{} } +func (m *FetchRequest) String() string { return proto.CompactTextString(m) } +func (*FetchRequest) ProtoMessage() {} +func (*FetchRequest) Descriptor() ([]byte, []int) { return fileDescriptor_33c57e4bae7b9afd, []int{0} } -func (m *RequestLatest) XXX_Unmarshal(b []byte) error { +func (m *FetchRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *RequestLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *FetchRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_RequestLatest.Marshal(b, m, deterministic) + return xxx_messageInfo_FetchRequest.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalTo(b) @@ -74,45 +74,45 @@ func (m *RequestLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return b[:n], nil } } -func (m *RequestLatest) XXX_Merge(src proto.Message) { - xxx_messageInfo_RequestLatest.Merge(m, src) +func (m *FetchRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_FetchRequest.Merge(m, src) } -func (m *RequestLatest) XXX_Size() int { +func (m *FetchRequest) XXX_Size() int { return m.Size() } -func (m *RequestLatest) XXX_DiscardUnknown() { - xxx_messageInfo_RequestLatest.DiscardUnknown(m) +func (m *FetchRequest) XXX_DiscardUnknown() { + xxx_messageInfo_FetchRequest.DiscardUnknown(m) } -var xxx_messageInfo_RequestLatest proto.InternalMessageInfo +var xxx_messageInfo_FetchRequest proto.InternalMessageInfo -func (m *RequestLatest) GetIdentifier() string { +func (m *FetchRequest) GetIdentifier() string { if m != nil { return m.Identifier } return "" } -type RespondLatest struct { - Status RespondLatest_StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=namesys.pb.RespondLatest_StatusCode" json:"status,omitempty"` +type FetchResponse struct { + Status FetchResponse_StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=namesys.pb.FetchResponse_StatusCode" json:"status,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *RespondLatest) Reset() { *m = RespondLatest{} } -func (m *RespondLatest) String() string { return proto.CompactTextString(m) } -func (*RespondLatest) ProtoMessage() {} -func (*RespondLatest) Descriptor() ([]byte, []int) { +func (m *FetchResponse) Reset() { *m = FetchResponse{} } +func (m *FetchResponse) String() string { return proto.CompactTextString(m) } +func (*FetchResponse) ProtoMessage() {} +func (*FetchResponse) Descriptor() ([]byte, []int) { return fileDescriptor_33c57e4bae7b9afd, []int{1} } -func (m *RespondLatest) XXX_Unmarshal(b []byte) error { +func (m *FetchResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *RespondLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *FetchResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_RespondLatest.Marshal(b, m, deterministic) + return xxx_messageInfo_FetchResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalTo(b) @@ -122,26 +122,26 @@ func (m *RespondLatest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error return b[:n], nil } } -func (m *RespondLatest) XXX_Merge(src proto.Message) { - xxx_messageInfo_RespondLatest.Merge(m, src) +func (m *FetchResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_FetchResponse.Merge(m, src) } -func (m *RespondLatest) XXX_Size() int { +func (m *FetchResponse) XXX_Size() int { return m.Size() } -func (m *RespondLatest) XXX_DiscardUnknown() { - xxx_messageInfo_RespondLatest.DiscardUnknown(m) +func (m *FetchResponse) XXX_DiscardUnknown() { + xxx_messageInfo_FetchResponse.DiscardUnknown(m) } -var xxx_messageInfo_RespondLatest proto.InternalMessageInfo +var xxx_messageInfo_FetchResponse proto.InternalMessageInfo -func (m *RespondLatest) GetStatus() RespondLatest_StatusCode { +func (m *FetchResponse) GetStatus() FetchResponse_StatusCode { if m != nil { return m.Status } - return RespondLatest_SUCCESS + return FetchResponse_OK } -func (m *RespondLatest) GetData() []byte { +func (m *FetchResponse) GetData() []byte { if m != nil { return m.Data } @@ -149,31 +149,31 @@ func (m *RespondLatest) GetData() []byte { } func init() { - proto.RegisterEnum("namesys.pb.RespondLatest_StatusCode", RespondLatest_StatusCode_name, RespondLatest_StatusCode_value) - proto.RegisterType((*RequestLatest)(nil), "namesys.pb.RequestLatest") - proto.RegisterType((*RespondLatest)(nil), "namesys.pb.RespondLatest") + proto.RegisterEnum("namesys.pb.FetchResponse_StatusCode", FetchResponse_StatusCode_name, FetchResponse_StatusCode_value) + proto.RegisterType((*FetchRequest)(nil), "namesys.pb.FetchRequest") + proto.RegisterType((*FetchResponse)(nil), "namesys.pb.FetchResponse") } func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 205 bytes of a gzipped FileDescriptorProto + // 201 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, - 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe7, 0xe2, 0x0d, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, - 0xf1, 0x49, 0x2c, 0x49, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, - 0x4c, 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0xea, 0x66, - 0x04, 0xe9, 0x28, 0x2e, 0xc8, 0xcf, 0x4b, 0x81, 0xea, 0xb0, 0xe1, 0x62, 0x2b, 0x2e, 0x49, 0x2c, - 0x29, 0x2d, 0x06, 0xab, 0xe6, 0x33, 0x52, 0xd1, 0x43, 0x98, 0xaf, 0x87, 0xa2, 0x54, 0x2f, 0x18, - 0xac, 0xce, 0x39, 0x3f, 0x25, 0x35, 0x08, 0xaa, 0x47, 0x48, 0x88, 0x8b, 0x25, 0x25, 0xb1, 0x24, - 0x51, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x56, 0xd2, 0xe0, 0xe2, 0x42, 0xa8, 0x14, - 0xe2, 0xe6, 0x62, 0x0f, 0x0e, 0x75, 0x76, 0x76, 0x0d, 0x0e, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, - 0xf4, 0xf3, 0x0f, 0x89, 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x74, 0xe2, 0x39, 0xf1, 0x48, - 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0xfe, 0x33, 0x06, - 0x04, 0x00, 0x00, 0xff, 0xff, 0x2d, 0x05, 0x6e, 0x5c, 0xf0, 0x00, 0x00, 0x00, -} - -func (m *RequestLatest) Marshal() (dAtA []byte, err error) { + 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe3, 0xe2, 0x71, 0x4b, 0x2d, 0x49, 0xce, 0x08, 0x4a, + 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, 0x4c, + 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0x6a, 0x63, 0xe4, + 0xe2, 0x85, 0x6a, 0x28, 0x2e, 0xc8, 0xcf, 0x2b, 0x4e, 0x15, 0xb2, 0xe1, 0x62, 0x2b, 0x2e, 0x49, + 0x2c, 0x29, 0x2d, 0x06, 0xab, 0xe6, 0x33, 0x52, 0xd1, 0x43, 0x18, 0xaf, 0x87, 0xa2, 0x54, 0x2f, + 0x18, 0xac, 0xce, 0x39, 0x3f, 0x25, 0x35, 0x08, 0xaa, 0x47, 0x48, 0x88, 0x8b, 0x25, 0x25, 0xb1, + 0x24, 0x51, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x56, 0x52, 0xe6, 0xe2, 0x42, 0xa8, + 0x14, 0x62, 0xe3, 0x62, 0xf2, 0xf7, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, 0xf4, 0xf3, 0x0f, 0x89, + 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x74, 0xe2, 0x39, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, + 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0x3e, 0x33, 0x06, 0x04, 0x00, 0x00, 0xff, + 0xff, 0xe9, 0x3b, 0xba, 0x5b, 0xea, 0x00, 0x00, 0x00, +} + +func (m *FetchRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -183,7 +183,7 @@ func (m *RequestLatest) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { +func (m *FetchRequest) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -200,7 +200,7 @@ func (m *RequestLatest) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *RespondLatest) Marshal() (dAtA []byte, err error) { +func (m *FetchResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -210,7 +210,7 @@ func (m *RespondLatest) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *RespondLatest) MarshalTo(dAtA []byte) (int, error) { +func (m *FetchResponse) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -241,7 +241,7 @@ func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *RequestLatest) Size() (n int) { +func (m *FetchRequest) Size() (n int) { if m == nil { return 0 } @@ -257,7 +257,7 @@ func (m *RequestLatest) Size() (n int) { return n } -func (m *RespondLatest) Size() (n int) { +func (m *FetchResponse) Size() (n int) { if m == nil { return 0 } @@ -289,7 +289,7 @@ func sovMessage(x uint64) (n int) { func sozMessage(x uint64) (n int) { return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *RequestLatest) Unmarshal(dAtA []byte) error { +func (m *FetchRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -312,10 +312,10 @@ func (m *RequestLatest) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RequestLatest: wiretype end group for non-group") + return fmt.Errorf("proto: FetchRequest: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RequestLatest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FetchRequest: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -375,7 +375,7 @@ func (m *RequestLatest) Unmarshal(dAtA []byte) error { } return nil } -func (m *RespondLatest) Unmarshal(dAtA []byte) error { +func (m *FetchResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -398,10 +398,10 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RespondLatest: wiretype end group for non-group") + return fmt.Errorf("proto: FetchResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RespondLatest: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FetchResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -418,7 +418,7 @@ func (m *RespondLatest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Status |= RespondLatest_StatusCode(b&0x7F) << shift + m.Status |= FetchResponse_StatusCode(b&0x7F) << shift if b < 0x80 { break } diff --git a/pb/message.proto b/pb/message.proto index 0985f06..c1dd31d 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -2,14 +2,14 @@ syntax = "proto3"; package namesys.pb; -message RequestLatest { +message FetchRequest { string identifier = 1; } -message RespondLatest { +message FetchResponse { StatusCode status = 1; enum StatusCode { - SUCCESS = 0; + OK = 0; NOT_FOUND = 1; } bytes data = 2; From fdbeaecc2f1c81432a90eff42350f449961155cf Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 15 Aug 2019 13:50:20 -0400 Subject: [PATCH 23/27] Makefile is more MSYS friendly but you still need a weird GOPATH --- pb/Makefile | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pb/Makefile b/pb/Makefile index 38a4972..e9ab2ab 100644 --- a/pb/Makefile +++ b/pb/Makefile @@ -1,16 +1,10 @@ PB = $(wildcard *.proto) GO = $(PB:.proto=.pb.go) -ifeq ($(OS),Windows_NT) - LIST_SEPARATOR = ; -else - LIST_SEPARATOR = : -endif - all: $(GO) %.pb.go: %.proto - protoc --proto_path="$(GOPATH)/src$(LIST_SEPARATOR)." --gogofast_out=. $< + protoc --proto_path="$(GOPATH)/src" --proto_path="." --gogofast_out=. $< clean: rm -f *.pb.go From b303f910bbcaccfefea495d23cdda8d505fa1046 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 16 Aug 2019 09:38:06 -0400 Subject: [PATCH 24/27] Updated go.mod to use unreleased version of pubsub. Refactored Fetch protocol interface. --- fetch.go | 4 ++-- fetch_test.go | 2 +- go.mod | 4 +--- go.sum | 2 ++ pubsub.go | 2 +- pubsub_test.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/fetch.go b/fetch.go index 6325051..28c746a 100644 --- a/fetch.go +++ b/fetch.go @@ -60,7 +60,7 @@ func (p *fetchProtocol) receive(s network.Stream, getData getValue) { } } -func (p fetchProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte, error) { +func (p fetchProtocol) Fetch(ctx context.Context, pid peer.ID, key string) ([]byte, error) { peerCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() @@ -88,7 +88,7 @@ func (p fetchProtocol) Get(ctx context.Context, pid peer.ID, key string) ([]byte case pb.FetchResponse_NOT_FOUND: return nil, nil default: - return nil, errors.New("get-latest: received unknown status code") + return nil, errors.New("fetch: received unknown status code") } } diff --git a/fetch_test.go b/fetch_test.go index 3506d85..a8eae86 100644 --- a/fetch_test.go +++ b/fetch_test.go @@ -94,7 +94,7 @@ func TestFetchProtocolRepeated(t *testing.T) { func fetchCheck(t *testing.T, ctx context.Context, requester *fetchProtocol, responder *fetchProtocol, key string, expected []byte) { - data, err := requester.Get(ctx, responder.host.ID(), key) + data, err := requester.Fetch(ctx, responder.host.ID(), key) if err != nil { t.Fatal(err) } diff --git a/go.mod b/go.mod index d5ca32d..05983c0 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,8 @@ require ( github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.1.1 github.com/libp2p/go-libp2p-core v0.0.1 - github.com/libp2p/go-libp2p-pubsub v0.1.0 + github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4 github.com/libp2p/go-libp2p-record v0.1.0 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-swarm v0.1.0 ) - -replace github.com/libp2p/go-libp2p-pubsub v0.1.0 => github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd diff --git a/go.sum b/go.sum index 8ae368f..33ca91a 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUje github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0 h1:MKh7pRNPHSh1fLPj8u/M/s/napdmeNpoi9BRy9lPN0E= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= +github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4 h1:untTAXKOcipK+9Oz3yVthhwkOrPPdkjG8QLpHFB0vIc= +github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8= diff --git a/pubsub.go b/pubsub.go index 0a69440..fc0a4aa 100644 --- a/pubsub.go +++ b/pubsub.go @@ -473,7 +473,7 @@ func (p *PubsubValueStore) handleNewPeer(ctx context.Context, sub *pubsub.Subscr } } - return p.fetch.Get(ctx, pid, key) + return p.fetch.Fetch(ctx, pid, key) } func (p *PubsubValueStore) notifyWatchers(key string, data []byte) { diff --git a/pubsub_test.go b/pubsub_test.go index 0085257..4c9dd39 100644 --- a/pubsub_test.go +++ b/pubsub_test.go @@ -138,7 +138,7 @@ func TestEarlyPublish(t *testing.T) { } } - // Wait for Get Latest protocol to retrieve data + // Wait for Fetch protocol to retrieve data time.Sleep(time.Second * 1) for i, vs := range vss { checkValue(ctx, t, i, vs, key, val) From 0787cb500f6412d860f684abea03f8eb958dfc2e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 16 Aug 2019 09:53:40 -0400 Subject: [PATCH 25/27] rebroadcast initial delay using timer --- pubsub.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pubsub.go b/pubsub.go index fc0a4aa..27d556b 100644 --- a/pubsub.go +++ b/pubsub.go @@ -193,7 +193,11 @@ func (p *PubsubValueStore) Subscribe(key string) error { } func (p *PubsubValueStore) rebroadcast(ctx context.Context) { - time.Sleep(p.rebroadcastInitialDelay) + select { + case <-time.After(p.rebroadcastInitialDelay): + case <-ctx.Done(): + return + } ticker := time.NewTicker(p.rebroadcastInterval) defer ticker.Stop() From 41f6fb8236a6341e420723c77e1dcc4eaa135af9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 16 Aug 2019 10:01:22 -0400 Subject: [PATCH 26/27] Added Error status code to Fetch protobuf. Currently unused. --- pb/message.pb.go | 24 ++++++++++++++---------- pb/message.proto | 1 + 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pb/message.pb.go b/pb/message.pb.go index 2284380..a1f5145 100644 --- a/pb/message.pb.go +++ b/pb/message.pb.go @@ -26,16 +26,19 @@ type FetchResponse_StatusCode int32 const ( FetchResponse_OK FetchResponse_StatusCode = 0 FetchResponse_NOT_FOUND FetchResponse_StatusCode = 1 + FetchResponse_ERROR FetchResponse_StatusCode = 2 ) var FetchResponse_StatusCode_name = map[int32]string{ 0: "OK", 1: "NOT_FOUND", + 2: "ERROR", } var FetchResponse_StatusCode_value = map[string]int32{ "OK": 0, "NOT_FOUND": 1, + "ERROR": 2, } func (x FetchResponse_StatusCode) String() string { @@ -157,20 +160,21 @@ func init() { func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } var fileDescriptor_33c57e4bae7b9afd = []byte{ - // 201 bytes of a gzipped FileDescriptorProto + // 212 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcd, 0x4d, 0x2d, 0x2e, 0x4e, 0x4c, 0x4f, 0xd5, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xca, 0x4b, 0xcc, 0x4d, 0x2d, 0xae, 0x2c, 0xd6, 0x2b, 0x48, 0x52, 0xd2, 0xe3, 0xe2, 0x71, 0x4b, 0x2d, 0x49, 0xce, 0x08, 0x4a, 0x2d, 0x2c, 0x4d, 0x2d, 0x2e, 0x11, 0x92, 0xe3, 0xe2, 0xca, 0x4c, 0x49, 0xcd, 0x2b, 0xc9, 0x4c, - 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0x6a, 0x63, 0xe4, - 0xe2, 0x85, 0x6a, 0x28, 0x2e, 0xc8, 0xcf, 0x2b, 0x4e, 0x15, 0xb2, 0xe1, 0x62, 0x2b, 0x2e, 0x49, - 0x2c, 0x29, 0x2d, 0x06, 0xab, 0xe6, 0x33, 0x52, 0xd1, 0x43, 0x18, 0xaf, 0x87, 0xa2, 0x54, 0x2f, - 0x18, 0xac, 0xce, 0x39, 0x3f, 0x25, 0x35, 0x08, 0xaa, 0x47, 0x48, 0x88, 0x8b, 0x25, 0x25, 0xb1, - 0x24, 0x51, 0x82, 0x49, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x56, 0x52, 0xe6, 0xe2, 0x42, 0xa8, - 0x14, 0x62, 0xe3, 0x62, 0xf2, 0xf7, 0x16, 0x60, 0x10, 0xe2, 0xe5, 0xe2, 0xf4, 0xf3, 0x0f, 0x89, - 0x77, 0xf3, 0x0f, 0xf5, 0x73, 0x11, 0x60, 0x74, 0xe2, 0x39, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, - 0x39, 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x93, 0xd8, 0xc0, 0x3e, 0x33, 0x06, 0x04, 0x00, 0x00, 0xff, - 0xff, 0xe9, 0x3b, 0xba, 0x5b, 0xea, 0x00, 0x00, 0x00, + 0xcb, 0x4c, 0x2d, 0x92, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x0c, 0x42, 0x12, 0x51, 0x9a, 0xc8, 0xc8, + 0xc5, 0x0b, 0xd5, 0x50, 0x5c, 0x90, 0x9f, 0x57, 0x9c, 0x2a, 0x64, 0xc3, 0xc5, 0x56, 0x5c, 0x92, + 0x58, 0x52, 0x5a, 0x0c, 0x56, 0xcd, 0x67, 0xa4, 0xa2, 0x87, 0x30, 0x5e, 0x0f, 0x45, 0xa9, 0x5e, + 0x30, 0x58, 0x9d, 0x73, 0x7e, 0x4a, 0x6a, 0x10, 0x54, 0x8f, 0x90, 0x10, 0x17, 0x4b, 0x4a, 0x62, + 0x49, 0xa2, 0x04, 0x93, 0x02, 0xa3, 0x06, 0x4f, 0x10, 0x98, 0xad, 0xa4, 0xc7, 0xc5, 0x85, 0x50, + 0x29, 0xc4, 0xc6, 0xc5, 0xe4, 0xef, 0x2d, 0xc0, 0x20, 0xc4, 0xcb, 0xc5, 0xe9, 0xe7, 0x1f, 0x12, + 0xef, 0xe6, 0x1f, 0xea, 0xe7, 0x22, 0xc0, 0x28, 0xc4, 0xc9, 0xc5, 0xea, 0x1a, 0x14, 0xe4, 0x1f, + 0x24, 0xc0, 0xe4, 0xc4, 0x73, 0xe2, 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, + 0x31, 0x26, 0xb1, 0x81, 0x3d, 0x69, 0x0c, 0x08, 0x00, 0x00, 0xff, 0xff, 0xc8, 0x6e, 0x00, 0xfe, + 0xf5, 0x00, 0x00, 0x00, } func (m *FetchRequest) Marshal() (dAtA []byte, err error) { diff --git a/pb/message.proto b/pb/message.proto index c1dd31d..2a161a8 100644 --- a/pb/message.proto +++ b/pb/message.proto @@ -11,6 +11,7 @@ message FetchResponse { enum StatusCode { OK = 0; NOT_FOUND = 1; + ERROR = 2; } bytes data = 2; } \ No newline at end of file From 981b38cc187c0b8a752d75ad3400a65a5e2c015f Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 19 Aug 2019 13:20:58 -0400 Subject: [PATCH 27/27] use go-libp2p-pubsub v0.1.1. Fix `Fetch` function to be a pointer receiver instead of value. --- fetch.go | 2 +- go.mod | 2 +- go.sum | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/fetch.go b/fetch.go index 28c746a..c8a11bf 100644 --- a/fetch.go +++ b/fetch.go @@ -60,7 +60,7 @@ func (p *fetchProtocol) receive(s network.Stream, getData getValue) { } } -func (p fetchProtocol) Fetch(ctx context.Context, pid peer.ID, key string) ([]byte, error) { +func (p *fetchProtocol) Fetch(ctx context.Context, pid peer.ID, key string) ([]byte, error) { peerCtx, cancel := context.WithTimeout(ctx, time.Second*10) defer cancel() diff --git a/go.mod b/go.mod index 05983c0..2267a0d 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/ipfs/go-log v0.0.1 github.com/libp2p/go-libp2p-blankhost v0.1.1 github.com/libp2p/go-libp2p-core v0.0.1 - github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4 + github.com/libp2p/go-libp2p-pubsub v0.1.1 github.com/libp2p/go-libp2p-record v0.1.0 github.com/libp2p/go-libp2p-routing-helpers v0.1.0 github.com/libp2p/go-libp2p-swarm v0.1.0 diff --git a/go.sum b/go.sum index 33ca91a..0e93bd3 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,6 @@ github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd h1:P1//443gNhGLDpMpzwnaNDgKoHpXrlt+iIEVgwlTHoI= -github.com/aschmahmann/go-libp2p-pubsub v0.0.4-0.20190807152749-d7996289bbcd/go.mod h1:ekhyliBSJ0aBg62+j9rECrxW+UUPAj1SLS3jN7JMAGc= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32 h1:qkOC5Gd33k54tobS36cXdAzJbeHaduLtnLQQwNoIi78= github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= @@ -88,7 +86,6 @@ github.com/libp2p/go-libp2p-core v0.0.1 h1:HSTZtFIq/W5Ue43Zw+uWZyy2Vl5WtF0zDjKN8 github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= -github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= @@ -98,8 +95,8 @@ github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUje github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0 h1:MKh7pRNPHSh1fLPj8u/M/s/napdmeNpoi9BRy9lPN0E= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= -github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4 h1:untTAXKOcipK+9Oz3yVthhwkOrPPdkjG8QLpHFB0vIc= -github.com/libp2p/go-libp2p-pubsub v0.1.1-0.20190807100218-9f04364996b4/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= +github.com/libp2p/go-libp2p-pubsub v0.1.1 h1:phDnQvO3H3hAgaEEQi6yt3LILqIYVXaw05bxzezrEwQ= +github.com/libp2p/go-libp2p-pubsub v0.1.1/go.mod h1:ZwlKzRSe1eGvSIdU5bD7+8RZN/Uzw0t1Bp9R1znpR/Q= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= github.com/libp2p/go-libp2p-routing-helpers v0.1.0 h1:BaFvpyv8TyhCN7TihawTiKuzeu8/Pyw7ZnMA4IvqIN8=