From fdcc3b52282fd11e0e5dcea567d8642b7667fb65 Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Tue, 2 Jul 2024 19:05:18 -0500 Subject: [PATCH 01/24] chore: add stats for bundle message delays, stale contact info (#2118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation - Need better visibility into message delays and disambiguate between bundle delay and message delay - Ensure stale contacts aren't disseminating through the network ## Change Summary - Add statsd metrics for earliest and latest message timestamp in bundle and bundle creation timestamp. All timestamps are in farcaster time - Add statsd metric for old contact info being gossiped ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR focuses on enhancing message bundle submission in the Hubble application by adding stats for message delays and contact info staleness. ### Detailed summary - Added stats for bundle message delays and stale contact info - Updated `submitMessageBundle` method parameters in multiple files - Updated `submitMessageBundle` method in `Hub` class to include additional stats and parameters > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/dull-dancers-float.md | 5 ++ apps/hubble/src/hubble.ts | 67 +++++++++++++++++-- apps/hubble/src/network/p2p/gossipNode.ts | 2 +- apps/hubble/src/network/sync/syncEngine.ts | 2 +- .../hubble/src/test/e2e/hubbleNetwork.test.ts | 6 +- apps/hubble/src/test/mocks.ts | 2 + 6 files changed, 72 insertions(+), 12 deletions(-) create mode 100644 .changeset/dull-dancers-float.md diff --git a/.changeset/dull-dancers-float.md b/.changeset/dull-dancers-float.md new file mode 100644 index 0000000000..440b5ba934 --- /dev/null +++ b/.changeset/dull-dancers-float.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +chore: add stats for bundle message delays, stale contact info diff --git a/apps/hubble/src/hubble.ts b/apps/hubble/src/hubble.ts index 77cf7f5c9b..caca74e9e9 100644 --- a/apps/hubble/src/hubble.ts +++ b/apps/hubble/src/hubble.ts @@ -5,6 +5,7 @@ import { ContactInfoContent, ContactInfoContentBody, FarcasterNetwork, + fromFarcasterTime, getInsecureHubRpcClient, getSSLHubRpcClient, GossipAddressInfo, @@ -17,6 +18,7 @@ import { Message, OnChainEvent, onChainEventTypeToJSON, + toFarcasterTime, UserNameProof, validations, } from "@farcaster/hub-nodejs"; @@ -131,7 +133,12 @@ export interface HubInterface { identity: string; hubOperatorFid?: number; submitMessage(message: Message, source?: HubSubmitSource): HubAsyncResult; - submitMessageBundle(messageBundle: MessageBundle, source?: HubSubmitSource): Promise[]>; + submitMessageBundle( + creationFarcasterTime: number, + messageBundle: MessageBundle, + source?: HubSubmitSource, + peerId?: PeerId, + ): Promise[]>; validateMessage(message: Message): HubAsyncResult; submitUserNameProof(usernameProof: UserNameProof, source?: HubSubmitSource): HubAsyncResult; submitOnChainEvent(event: OnChainEvent, source?: HubSubmitSource): HubAsyncResult; @@ -1340,14 +1347,13 @@ export class Hub implements HubInterface { await this.gossipNode.reportValid(msgId, peerIdFromString(source.toString()).toBytes(), false); } } - statsd().timing("gossip.message_delay", gossipMessageDelay); const mergeResult = result.isOk() ? "success" : "failure"; - statsd().timing(`gossip.message_delay.${mergeResult}`, gossipMessageDelay); + statsd().timing("gossip.message_delay", gossipMessageDelay, { status: mergeResult }); return result.map(() => undefined); } else if (gossipMessage.messageBundle) { const bundle = gossipMessage.messageBundle; - const results = await this.submitMessageBundle(bundle, "gossip"); + const results = await this.submitMessageBundle(messageFirstGossipedTime, bundle, "gossip", source); // If at least one is Ok, report as valid const atLeastOneOk = results.find((r) => r.isOk()); @@ -1414,6 +1420,7 @@ export class Hub implements HubInterface { // Don't process messages that are too old if (message.timestamp && message.timestamp < Date.now() - MAX_CONTACT_INFO_AGE_MS) { + statsd().increment("gossip.contact_info.too_old", 1); log.debug({ message }, "contact info message is too old"); return false; } @@ -1666,7 +1673,12 @@ export class Hub implements HubInterface { /* RPC Handler API */ /* -------------------------------------------------------------------------- */ - async submitMessageBundle(messageBundle: MessageBundle, source?: HubSubmitSource): Promise[]> { + async submitMessageBundle( + creationFarcasterTime: number, + messageBundle: MessageBundle, + source?: HubSubmitSource, + peerId?: PeerId, + ): Promise[]> { if (this.syncEngine.syncTrieQSize > MAX_SYNCTRIE_QUEUE_SIZE) { log.warn({ syncTrieQSize: this.syncEngine.syncTrieQSize }, "SubmitMessage rejected: Sync trie queue is full"); // Since we're rejecting the full bundle, return an error for each message @@ -1679,6 +1691,8 @@ export class Hub implements HubInterface { const allResults: Map> = new Map(); const dedupedMessages: { i: number; message: Message }[] = []; + let earliestTimestamp = Infinity; + let latestTimestamp = -Infinity; if (source === "gossip") { // Go over all the messages and see if they are in the DB. If they are, don't bother processing them const messagesExist = await areMessagesInDb(this.rocksDB, messageBundle.messages); @@ -1688,12 +1702,51 @@ export class Hub implements HubInterface { log.debug({ source }, "submitMessageBundle rejected: Message already exists"); allResults.set(i, err(new HubError("bad_request.duplicate", "message has already been merged"))); } else { - dedupedMessages.push({ i, message: ensureMessageData(messageBundle.messages[i] as Message) }); + const message = ensureMessageData(messageBundle.messages[i] as Message); + earliestTimestamp = Math.min(earliestTimestamp, message.data?.timestamp ?? Infinity); + latestTimestamp = Math.max(latestTimestamp, message.data?.timestamp ?? -Infinity); + dedupedMessages.push({ i, message }); } } } else { - dedupedMessages.push(...messageBundle.messages.map((message, i) => ({ i, message: ensureMessageData(message) }))); + const initialResult = { + earliest: Infinity, + latest: -Infinity, + deduped: [] as { i: number; message: Message }[], + }; + const { deduped, earliest, latest } = messageBundle.messages.reduce((acc, message, i) => { + const messageData = ensureMessageData(message); + acc.earliest = Math.min(acc.earliest, messageData.data?.timestamp ?? Infinity); + acc.latest = Math.max(acc.latest, messageData.data?.timestamp ?? -Infinity); + acc.deduped.push({ i, message: messageData }); + return acc; + }, initialResult); + earliestTimestamp = earliest; + latestTimestamp = latest; + dedupedMessages.push(...deduped); } + const tags: { [key: string]: string } = { + ...(source ? { source } : {}), + ...(peerId ? { peer_id: peerId.toString() } : {}), + }; + + statsd().gauge("hub.submit_message_bundle.size", dedupedMessages.length, tags); + statsd().gauge( + "hub.submit_message_bundle.earliest_timestamp_ms", + fromFarcasterTime(earliestTimestamp).unwrapOr(0), + tags, + ); + statsd().gauge( + "hub.submit_message_bundle.latest_timestamp_ms", + fromFarcasterTime(latestTimestamp).unwrapOr(0), + tags, + ); + statsd().gauge( + "hub.submit_message_bundle.creation_time_ms", + fromFarcasterTime(creationFarcasterTime).unwrapOr(0), + tags, + ); + statsd().gauge("hub.submit_message_bundle.max_delay_ms", creationFarcasterTime - earliestTimestamp, tags); // Merge the messages const mergeResults = await this.engine.mergeMessages(dedupedMessages.map((m) => m.message)); diff --git a/apps/hubble/src/network/p2p/gossipNode.ts b/apps/hubble/src/network/p2p/gossipNode.ts index 77e5412c02..029d6a5545 100644 --- a/apps/hubble/src/network/p2p/gossipNode.ts +++ b/apps/hubble/src/network/p2p/gossipNode.ts @@ -34,7 +34,7 @@ import { sleep } from "../../utils/crypto.js"; /** The maximum number of pending merge messages before we drop new incoming gossip or sync messages. */ export const MAX_SYNCTRIE_QUEUE_SIZE = 100_000; /** The TTL for messages in the seen cache */ -export const GOSSIP_SEEN_TTL = 1000 * 60 * 5; +export const GOSSIP_SEEN_TTL = 1000 * 60 * 5; // 5 minutes /** The maximum amount of time to dial a peer in libp2p network in milliseconds */ export const LIBP2P_CONNECT_TIMEOUT_MS = 2000; diff --git a/apps/hubble/src/network/sync/syncEngine.ts b/apps/hubble/src/network/sync/syncEngine.ts index 3d97562213..8c54f062e4 100644 --- a/apps/hubble/src/network/sync/syncEngine.ts +++ b/apps/hubble/src/network/sync/syncEngine.ts @@ -1187,7 +1187,7 @@ class SyncEngine extends TypedEmitter { statsd().gauge("syncengine.merge_q", this._syncMergeQ); const startTime = Date.now(); - const results = await this._hub.submitMessageBundle(MessageBundle.create({ messages }), "sync"); + const results = await this._hub.submitMessageBundle(startTime, MessageBundle.create({ messages }), "sync"); for (let i = 0; i < results.length; i++) { const result = results[i] as HubResult; diff --git a/apps/hubble/src/test/e2e/hubbleNetwork.test.ts b/apps/hubble/src/test/e2e/hubbleNetwork.test.ts index 6c85ffa3dd..1dc5d01777 100644 --- a/apps/hubble/src/test/e2e/hubbleNetwork.test.ts +++ b/apps/hubble/src/test/e2e/hubbleNetwork.test.ts @@ -147,7 +147,7 @@ describe("hubble gossip and sync tests", () => { messages: castAddMessages, }); // Submit it to hub1 via rpc so it gets gossiped - const bundleMergeResult = await hub1.submitMessageBundle(messageBundle, "rpc"); + const bundleMergeResult = await hub1.submitMessageBundle(Date.now(), messageBundle, "rpc"); expect(bundleMergeResult.length).toEqual(5); for (let i = 0; i < 5; i++) { expect(bundleMergeResult[i]?.isOk()).toBeTruthy(); @@ -167,7 +167,7 @@ describe("hubble gossip and sync tests", () => { } // Submitting the same bundle again should result in a duplicate error if we submit via gossip - const errResult2 = await hub1.submitMessageBundle(messageBundle, "gossip"); + const errResult2 = await hub1.submitMessageBundle(Date.now(), messageBundle, "gossip"); // Expect all the messages to be duplicates for (let i = 0; i < 5; i++) { expect(errResult2[i]?.isErr()).toBeTruthy(); @@ -193,7 +193,7 @@ describe("hubble gossip and sync tests", () => { hash: new Uint8Array([0, 1, 2, 1, 2]), messages: castAddMessages2, }); - const errResult3 = await hub1.submitMessageBundle(messageBundle2, "gossip"); + const errResult3 = await hub1.submitMessageBundle(Date.now(), messageBundle2, "gossip"); expect(errResult3.length).toEqual(5); expect(errResult3[0]?.isOk()).toBeTruthy(); expect(errResult3[1]?.isErr()).toBeTruthy(); diff --git a/apps/hubble/src/test/mocks.ts b/apps/hubble/src/test/mocks.ts index e166c3262a..77e2d11ef7 100644 --- a/apps/hubble/src/test/mocks.ts +++ b/apps/hubble/src/test/mocks.ts @@ -43,8 +43,10 @@ export class MockHub implements HubInterface { } async submitMessageBundle( + _creationTimeMs: number, messageBundle: MessageBundle, source?: HubSubmitSource | undefined, + _peerId?: PeerId, ): Promise[]> { const results: HubResult[] = []; for (const message of messageBundle.messages) { From aa02a48db494b2e55b8ab71fcf26be1b3cb078be Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Tue, 2 Jul 2024 21:03:18 -0500 Subject: [PATCH 02/24] fix: validate gossip message for clock skew (#2119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation - Messages were found in gossip with very high timestamps that would be very far in the future ## Change Summary - Reject gossip messages that are more than 10 minutes in the future ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR focuses on fixing clock skew validation for gossip messages in the `@farcaster/hubble` module. ### Detailed summary - Added validation for clock skew in gossip messages - Introduced `ALLOWED_CLOCK_SKEW_SECONDS` constant - Improved error handling for future timestamps in gossip messages > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/silent-planes-promise.md | 5 +++++ apps/hubble/src/hubble.ts | 20 +++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .changeset/silent-planes-promise.md diff --git a/.changeset/silent-planes-promise.md b/.changeset/silent-planes-promise.md new file mode 100644 index 0000000000..9442bd517f --- /dev/null +++ b/.changeset/silent-planes-promise.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +fix: validate gossip message for clock skew diff --git a/apps/hubble/src/hubble.ts b/apps/hubble/src/hubble.ts index caca74e9e9..2072dcc5ba 100644 --- a/apps/hubble/src/hubble.ts +++ b/apps/hubble/src/hubble.ts @@ -18,7 +18,6 @@ import { Message, OnChainEvent, onChainEventTypeToJSON, - toFarcasterTime, UserNameProof, validations, } from "@farcaster/hub-nodejs"; @@ -127,6 +126,7 @@ export const FARCASTER_VERSIONS_SCHEDULE: VersionSchedule[] = [ const MAX_CONTACT_INFO_AGE_MS = 1000 * 60 * 60; // 60 minutes const CONTACT_INFO_UPDATE_THRESHOLD_MS = 1000 * 60 * 30; // 30 minutes +const ALLOWED_CLOCK_SKEW_SECONDS = 60 * 10; // 10 minutes export interface HubInterface { engine: Engine; @@ -1274,6 +1274,24 @@ export class Hub implements HubInterface { const messageFirstGossipedTime = gossipMessage.timestamp ?? 0; const gossipMessageDelay = currentTime - messageFirstGossipedTime; if (gossipMessage.timestamp) { + if (gossipMessage.timestamp > currentTime && gossipMessage.timestamp - currentTime > ALLOWED_CLOCK_SKEW_SECONDS) { + log.error( + { + allowedClockSkew: ALLOWED_CLOCK_SKEW_SECONDS, + currentTime, + gossipMessageTimestamp: gossipMessage.timestamp, + source: source.toString(), + }, + "Received gossip message with future timestamp", + ); + await this.gossipNode.reportValid(msgId, peerIdFromString(source.toString()).toBytes(), false); + return err( + new HubError( + "bad_request.invalid_param", + "Invalid Farcaster timestamp in gossip message - future timestamp found in seconds from Farcaster Epoch", + ), + ); + } // If message is older than seenTTL, we will try to merge it, but report it as invalid so it doesn't // propogate across the network const cutOffTime = getFarcasterTime().unwrapOr(0) - GOSSIP_SEEN_TTL / 1000; From 094fe86f06a44ccfbb2beb5e28333228df38d12e Mon Sep 17 00:00:00 2001 From: Sergei Volkov Date: Wed, 3 Jul 2024 06:04:40 +0400 Subject: [PATCH 03/24] fix(docs): corrected errors in installation instructions (#2087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes issue #2086 ## Motivation Due to errors or outdated information in the Farcaster Hub installation guides, the Hub cannot be synchronized due to lack of memory which will prevent it from starting and working correctly. This error is very important because it creates difficulties for new users at the very initial stage of work and familiarization with Farcaster Hub ## Change Summary Changes have been made regarding the minimum amount of memory required for Farcaster Hub to work correctly. The old value is 60 GB in the GCP installation guide and 140 GB is the old value in the main installation guide. Farcaster Hub currently takes up about 150 GB. The old memory values have been changed to 160 GB, which allows you to run the Hub and have a reserve of correct operation for a period of time ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ x] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context According to the GCP guide we create a virtual server and set the server parameters via Google Cloud Console by copying the data from the guide into the created file "main.tf" ![1 screenshot](https://github.com/farcasterxyz/hub-monorepo/assets/80642652/ae6bbd6e-a3ee-4129-978d-896d932d8c35) The virtual server has been configured with these memory settings ![2 screenshot](https://github.com/farcasterxyz/hub-monorepo/assets/80642652/ae81634a-d66f-436f-a107-5417f26cf4b4) Then after running the script specified in the main installation guide on the created server Hub starts synchronization, but after a while it stops with the error "code 2". ![3 screenshot](https://github.com/farcasterxyz/hub-monorepo/assets/80642652/3696c741-1234-4834-8218-ed71c179e1af) This error means that Hub cannot continue synchronization due to lack of memory ![4 screenshot](https://github.com/farcasterxyz/hub-monorepo/assets/80642652/57aa5e65-b419-4385-ba5c-ca678c4bb60a) Warpcast: @vsu --- ## PR-Codex overview This PR increases the disk size for the Ubuntu 20.04 LTS image in the GCP tutorial and adjusts the storage requirement to 160 GB in the installation guide. ### Detailed summary - Increased disk size for Ubuntu 20.04 LTS image to 160 GB in GCP tutorial - Adjusted storage requirement to 160 GB in the installation guide > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/www/docs/intro/install.md | 2 +- apps/hubble/www/docs/tutorials/gcp.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/hubble/www/docs/intro/install.md b/apps/hubble/www/docs/intro/install.md index 92758daf21..92eccbc330 100644 --- a/apps/hubble/www/docs/intro/install.md +++ b/apps/hubble/www/docs/intro/install.md @@ -8,7 +8,7 @@ Hubble can be installed in 30 minutes, and a full sync can take 1-2 hours to com - 16 GB of RAM - 4 CPU cores or vCPUs -- 140 GB of free storage +- 160 GB of free storage - A public IP address with ports 2282 - 2285 exposed See [tutorials](./tutorials.html) for instructions on how to set up cloud providers to run Hubble. diff --git a/apps/hubble/www/docs/tutorials/gcp.md b/apps/hubble/www/docs/tutorials/gcp.md index c5cd7aa959..5cd79d8c06 100644 --- a/apps/hubble/www/docs/tutorials/gcp.md +++ b/apps/hubble/www/docs/tutorials/gcp.md @@ -44,7 +44,7 @@ resource "google_compute_instance" "farcaster-hub-vm" { boot_disk { initialize_params { image = "ubuntu-2004-focal-v20231213" # Ubuntu 20.04 LTS image URL - size = 60 # 60 GB disk size + size = 160 # 160 GB disk size } } From 2a82b3dcd600401b76e8d5d01cef819629f74a0e Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Wed, 3 Jul 2024 11:48:57 -0500 Subject: [PATCH 04/24] feat: add unique peer map to sync engine to represent current active peers (#2120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation - Current peer store keeps track of all hubs that successfully connected - However, peers may connect and never be seen again due to churn or change in Peer ID - We add TTL Map of peers that only adds peers, never deletes, and expires any peer that hasn't gossiped updates in 24 hours ## Change Summary - add unique peer map to sync engine to represent current active peers ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR introduces a unique peer map to track active peers in the sync engine of the Hubble app. It also adds a high-performance TTLMap for efficient entry expiration management. ### Detailed summary - Added unique peer map for active peers in the sync engine - Implemented TTLMap for automatic expiration of entries - Updated handling of peer discovery events - Enhanced peer contact management in the sync engine > The following files were skipped due to too many changes: `apps/hubble/src/utils/ttl_map.ts` > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/popular-shoes-bathe.md | 5 + apps/hubble/src/hubble.ts | 23 ++++ apps/hubble/src/network/p2p/gossipNode.ts | 3 + apps/hubble/src/network/sync/syncEngine.ts | 40 +++++- apps/hubble/src/rpc/server.ts | 2 +- apps/hubble/src/utils/ttl_map.test.ts | 142 +++++++++++++++++++ apps/hubble/src/utils/ttl_map.ts | 152 +++++++++++++++++++++ 7 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 .changeset/popular-shoes-bathe.md create mode 100644 apps/hubble/src/utils/ttl_map.test.ts create mode 100644 apps/hubble/src/utils/ttl_map.ts diff --git a/.changeset/popular-shoes-bathe.md b/.changeset/popular-shoes-bathe.md new file mode 100644 index 0000000000..af1864f2aa --- /dev/null +++ b/.changeset/popular-shoes-bathe.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +feat: add unique peer map to sync engine to represent current active peers diff --git a/apps/hubble/src/hubble.ts b/apps/hubble/src/hubble.ts index 2072dcc5ba..7be20f7c0c 100644 --- a/apps/hubble/src/hubble.ts +++ b/apps/hubble/src/hubble.ts @@ -1260,6 +1260,7 @@ export class Hub implements HubInterface { await this.gossipNode.gossipContactInfo(contactInfo); statsd().gauge("peer_store.count", await this.gossipNode.peerStoreCount()); + statsd().gauge("active_peers.count", this.syncEngine.getActivePeerCount()); return Promise.resolve(ok(undefined)); } } @@ -1685,6 +1686,28 @@ export class Hub implements HubInterface { this.syncEngine.removeContactInfoForPeerId(connection.remotePeer.toString()); statsd().increment("peer_disconnect.count"); }); + + this.gossipNode.on("peerDiscovery", async (peerInfo) => { + // NB: The current code is not in use - we would require a source to emit peerDiscovery events. + // This is a placeholder for future use. + + // Add discovered peer to sync engine + const peerId = peerInfo.id; + const peerAddresses = peerInfo.multiaddrs; + + // sorts addresses by Public IPs first + const addr = peerAddresses.sort((a, b) => + publicAddressesFirst({ multiaddr: a, isCertified: false }, { multiaddr: b, isCertified: false }), + )[0]; + if (addr === undefined) { + log.info( + { function: "peerDiscovery", peerId: peerId.toString() }, + "peer found but no address is available to request sync", + ); + return; + } + await this.gossipNode.addPeerToAddressBook(peerId, addr); + }); } /* -------------------------------------------------------------------------- */ diff --git a/apps/hubble/src/network/p2p/gossipNode.ts b/apps/hubble/src/network/p2p/gossipNode.ts index 029d6a5545..df3b9b43a0 100644 --- a/apps/hubble/src/network/p2p/gossipNode.ts +++ b/apps/hubble/src/network/p2p/gossipNode.ts @@ -1,5 +1,6 @@ import { PublishResult } from "@libp2p/interface-pubsub"; import { Worker } from "worker_threads"; +import { PeerInfo } from "@libp2p/interface-peer-info"; import { ContactInfoContent, FarcasterNetwork, @@ -50,6 +51,8 @@ interface NodeEvents { peerConnect: (connection: Connection) => void; /** Triggers when a peer disconnects and includes the libp2p Connection object */ peerDisconnect: (connection: Connection) => void; + /** Triggers when a peer is discovered and includes the libp2p PeerInfo object */ + peerDiscovery: (peerInfo: PeerInfo) => void; } /** Optional arguments provided when creating a Farcaster Gossip Node */ diff --git a/apps/hubble/src/network/sync/syncEngine.ts b/apps/hubble/src/network/sync/syncEngine.ts index 8c54f062e4..be879b0d76 100644 --- a/apps/hubble/src/network/sync/syncEngine.ts +++ b/apps/hubble/src/network/sync/syncEngine.ts @@ -52,6 +52,14 @@ import { PeerScore, PeerScorer } from "./peerScore.js"; import { getOnChainEvent } from "../../storage/db/onChainEvent.js"; import { getUserNameProof } from "../../storage/db/nameRegistryEvent.js"; import { MaxPriorityQueue } from "@datastructures-js/priority-queue"; +import { TTLMap } from "../../utils/ttl_map.js"; +import * as buffer from "node:buffer"; +import { peerIdFromString } from "@libp2p/peer-id"; + +// Time to live for peer contact info in the Peer TTLMap +const PEER_TTL_MAP_EXPIRATION_TIME_MILLISECONDS = 1000 * 60 * 60 * 24; // 24 hours +// Time interval to run cleanup on the Peer TTLMap +const PEER_TTL_MAP_CLEANUP_INTERVAL_MILLISECONDS = 1000 * 60 * 60 * 36; // 36 hours // Number of seconds to wait for the network to "settle" before syncing. We will only // attempt to sync messages that are older than this time. @@ -246,6 +254,7 @@ class SyncEngine extends TypedEmitter { private _syncProfiler?: SyncEngineProfiler; private currentHubPeerContacts: Map = new Map(); + private uniquePeerMap: TTLMap; // Number of messages waiting to get into the SyncTrie. private _syncTrieQ = 0; @@ -282,6 +291,10 @@ class SyncEngine extends TypedEmitter { ) { super(); + this.uniquePeerMap = new TTLMap( + PEER_TTL_MAP_EXPIRATION_TIME_MILLISECONDS, + PEER_TTL_MAP_CLEANUP_INTERVAL_MILLISECONDS, + ); this._db = rocksDb; this._trie = new MerkleTrie(rocksDb); this._l2EventsProvider = l2EventsProvider; @@ -541,12 +554,17 @@ class SyncEngine extends TypedEmitter { return this.currentHubPeerContacts.size; } + public getActivePeerCount(): number { + return this.uniquePeerMap.size(); + } + public getContactInfoForPeerId(peerId: string): PeerContact | undefined { return this.currentHubPeerContacts.get(peerId); } public getCurrentHubPeerContacts() { - return this.currentHubPeerContacts.values(); + // return non-expired active peers + return this.uniquePeerMap.getAll(); } public addContactInfoForPeerId( @@ -577,7 +595,9 @@ class SyncEngine extends TypedEmitter { }, "Updated Peer ContactInfo", ); - this.currentHubPeerContacts.set(peerId.toString(), { peerId, contactInfo }); + const id = peerId.toString(); + this.uniquePeerMap.set(id, contactInfo); + this.currentHubPeerContacts.set(id, { peerId, contactInfo }); return ok(undefined); } else { return err(new HubError("bad_request.duplicate", "recent contact update found for peer")); @@ -585,6 +605,11 @@ class SyncEngine extends TypedEmitter { } public removeContactInfoForPeerId(peerId: string) { + // NB: We don't remove the peer from the uniquePeerMap here - there can be many (valid) reasons to remove a + // peer from current hub peer contacts that are due to peer behavior but not indicative of whether that peer is active. + // For example, the peer may have a bad peer score or be disallowed for this hub, but it may still be an active peer. + // Since the unique peer map is meant to return the set of all active non-expired peers, we only remove + // peers from it when their TTL expires (i.e, if we don't get updates to their contact info for 24 hours) this.currentHubPeerContacts.delete(peerId); } @@ -636,7 +661,16 @@ class SyncEngine extends TypedEmitter { // Use a buffer of 5% of our messages so the peer with the highest message count does not get picked // disproportionately const messageThreshold = snapshotResult.value.numMessages * 0.95; - peers = Array.from(this.currentHubPeerContacts.values()).filter((p) => p.contactInfo.count > messageThreshold); + peers = this.uniquePeerMap + .getAll() + .filter((p) => this.currentHubPeerContacts.has(p[0]) && p[1].count > messageThreshold) + .map( + (p) => + ({ + peerId: peerIdFromString(p[0]), + contactInfo: p[1], + }) as PeerContact, + ); } if (peers.length === 0) { diff --git a/apps/hubble/src/rpc/server.ts b/apps/hubble/src/rpc/server.ts index 83ff5134b7..5cd0c5c8f0 100644 --- a/apps/hubble/src/rpc/server.ts +++ b/apps/hubble/src/rpc/server.ts @@ -495,7 +495,7 @@ export default class Server { return; } - const contactInfoArray = Array.from(currentHubPeerContacts).map((peerContact) => peerContact.contactInfo); + const contactInfoArray = Array.from(currentHubPeerContacts).map((peerContact) => peerContact[1]); callback(null, ContactInfoResponse.create({ contacts: contactInfoArray })); })(); }, diff --git a/apps/hubble/src/utils/ttl_map.test.ts b/apps/hubble/src/utils/ttl_map.test.ts new file mode 100644 index 0000000000..b6e0107dff --- /dev/null +++ b/apps/hubble/src/utils/ttl_map.test.ts @@ -0,0 +1,142 @@ +import { TTLMap } from "./ttl_map.js"; +import { jest } from "@jest/globals"; + +describe("TTLMap", () => { + jest.useFakeTimers(); + + test("size should accurately reflect non-expired entries", () => { + const map = new TTLMap(1000); + + expect(map.size()).toBe(0); + + map.set("a", 1); + expect(map.size()).toBe(1); + + map.set("b", 2); + expect(map.size()).toBe(2); + + jest.advanceTimersByTime(500); + map.set("c", 3); + expect(map.size()).toBe(3); + + jest.advanceTimersByTime(501); + expect(map.size()).toBe(1); + + map.get("a"); // This should trigger cleanup of expired entry + expect(map.size()).toBe(1); + }); + + test("set should not double count when updating existing entries", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + expect(map.size()).toBe(1); + + map.set("a", 2); // Updating existing non-expired entry + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(1001); + map.set("a", 3); // Updating expired entry + expect(map.size()).toBe(1); + }); + + test("delete should correctly update size for expired and non-expired entries", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + map.set("b", 2); + expect(map.size()).toBe(2); + + map.delete("a"); + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(1001); + expect(map.delete("b")).toBe(true); // Deleting expired entry + expect(map.size()).toBe(0); + }); + + test("get should update size when retrieving expired entries", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(1001); + expect(map.get("a")).toBeUndefined(); + expect(map.size()).toBe(0); + }); + + test("resetTTL should correctly handle expired and non-expired entries", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(500); + map.resetTTL("a"); + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(750); + expect(map.get("a")).toBe(1); + expect(map.size()).toBe(1); + + jest.advanceTimersByTime(251); + map.resetTTL("a"); + expect(map.size()).toBe(1); + expect(map.get("a")).toBe(1); + }); + + test("getAll should return only non-expired entries", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + map.set("b", 2); + map.set("c", 3); + + jest.advanceTimersByTime(500); + map.set("d", 4); + + jest.advanceTimersByTime(501); + + const allEntries = map.getAll(); + expect(allEntries).toHaveLength(1); + expect(allEntries[0]).toEqual(["d", 4]); + expect(map.size()).toBe(1); + }); + + test("clear should reset size to zero", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + map.set("b", 2); + expect(map.size()).toBe(2); + + map.clear(); + expect(map.size()).toBe(0); + + map.set("c", 3); + expect(map.size()).toBe(1); + }); + + test("size should be accurate after multiple operations", () => { + const map = new TTLMap(1000); + + map.set("a", 1); + map.set("b", 2); + map.set("c", 3); + expect(map.size()).toBe(3); + + jest.advanceTimersByTime(500); + map.delete("b"); + map.set("d", 4); + expect(map.size()).toBe(3); + + jest.advanceTimersByTime(501); + map.get("a"); // This should trigger cleanup + expect(map.size()).toBe(1); + + map.set("e", 5); + map.resetTTL("d"); + expect(map.size()).toBe(2); + }); +}); diff --git a/apps/hubble/src/utils/ttl_map.ts b/apps/hubble/src/utils/ttl_map.ts new file mode 100644 index 0000000000..c84af24170 --- /dev/null +++ b/apps/hubble/src/utils/ttl_map.ts @@ -0,0 +1,152 @@ +/** + * TTLMap - A high-performance Time-To-Live Map implementation + * + * This data structure provides a Map-like interface with automatic expiration of entries. + * It is designed for scenarios with thousands of elements and frequent bulk retrieval operations. + * + * Key features: + * - Constant-time complexity for get, set, and delete operations + * - Efficient bulk retrieval of non-expired entries + * - Coarse-grained TTL defined at the Map level + * - Ability to reset TTL for individual entries + * + * @template K The type of keys in the map + * @template V The type of values in the map + */ +export class TTLMap { + private map: Map; + private readonly ttl: number; + private lastCleanup: number; + private readonly cleanupInterval: number; + private nonExpiredCount: number; + + /** + * @param ttl Time-to-live in milliseconds for entries + * @param cleanupInterval Optional interval for running the cleanup process (default: ttl / 2) + */ + constructor(ttl: number, cleanupInterval?: number) { + this.map = new Map(); + this.ttl = ttl; + this.lastCleanup = Date.now(); + this.cleanupInterval = cleanupInterval || Math.floor(ttl / 2); + this.nonExpiredCount = 0; + } + + /** + * Sets a key-value pair in the map with the current TTL + * @param key The key to set + * @param value The value to set + * @returns The TTLMap instance for method chaining + */ + set(key: K, value: V): this { + const now = Date.now(); + const expiresAt = now + this.ttl; + const existingEntry = this.map.get(key); + + if (!existingEntry) { + this.nonExpiredCount++; + } + + this.map.set(key, { value, expiresAt }); + this.cleanupIfNeeded(); + return this; + } + + /** + * Retrieves a value from the map if it exists and hasn't expired + * @param key The key to retrieve + * @returns The value if found and not expired, undefined otherwise + */ + get(key: K): V | undefined { + const entry = this.map.get(key); + if (entry) { + if (entry.expiresAt > Date.now()) { + return entry.value; + } else { + this.map.delete(key); + this.nonExpiredCount--; + } + } + return undefined; + } + + /** + * Deletes a key-value pair from the map + * @param key The key to delete + * @returns true if the element was in the map and not expired, false otherwise + */ + delete(key: K): boolean { + const entry = this.map.get(key); + if (entry) { + if (entry.expiresAt > Date.now()) { + this.nonExpiredCount--; + } + return this.map.delete(key); + } + return false; + } + + /** + * Resets the TTL for a given key without changing its value + * @param key The key to reset the TTL for + * @returns true if the key exists and TTL was reset, false otherwise + */ + resetTTL(key: K): boolean { + const entry = this.map.get(key); + if (entry) { + entry.expiresAt = Date.now() + this.ttl; + return true; + } + return false; + } + + /** + * Retrieves all non-expired entries from the map + * This method is optimized for frequent calls and large datasets + * @returns An array of [key, value] pairs for all non-expired entries + */ + getAll(): [K, V][] { + this.cleanupIfNeeded(); + const now = Date.now(); + return Array.from(this.map.entries()) + .filter(([, { expiresAt }]) => expiresAt > now) + .map(([key, { value }]) => [key, value]); + } + + /** + * Clears all entries from the map + */ + clear(): void { + this.nonExpiredCount = 0; + this.map.clear(); + } + + /** + * Returns the number of non-expired entries in the map + * @returns The number of non-expired entries in the map + */ + size(): number { + this.cleanupIfNeeded(); + return this.nonExpiredCount; + } + + /** + * Performs cleanup of expired entries if the cleanup interval has elapsed + * This method is called internally and doesn't need to be invoked manually + */ + private cleanupIfNeeded(): void { + const now = Date.now(); + if (now - this.lastCleanup > this.cleanupInterval) { + let nonExpired = 0; + this.map.forEach((entry, key) => { + if (entry.expiresAt <= now) { + this.map.delete(key); + } else { + nonExpired++; + } + }); + this.lastCleanup = now; + this.nonExpiredCount = nonExpired; + } + } +} From 02dddc269db0c0b34b0b9597f72a8f337e2b2d99 Mon Sep 17 00:00:00 2001 From: osrm <90407222+osrm@users.noreply.github.com> Date: Thu, 4 Jul 2024 01:49:36 +0900 Subject: [PATCH 05/24] fix: corrected typo errors in CHANGELOG.md (#2109) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrected typos, ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary Describe the changes being made in 1-2 concise sentences. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ ] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR updates the CHANGELOG for the Hubble app version 1.13.2. It includes fixes for HTTP endpoint responses and IP address fetching. ### Detailed summary - Fixed HTTP endpoint response wording - Improved IP address fetching by preferring IPv4 addresses > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/hubble/CHANGELOG.md b/apps/hubble/CHANGELOG.md index 6095ce38c7..6c366c5564 100644 --- a/apps/hubble/CHANGELOG.md +++ b/apps/hubble/CHANGELOG.md @@ -23,7 +23,7 @@ - 2d26d305: CLI tool for measuring sync health - b150e900: fix: Use stricter socket timeout for gossip -- eacf29c9: fix: http endoint return not found instead of internal database error +- eacf29c9: fix: http endpoint return not found instead of internal database error - @farcaster/hub-nodejs@0.11.18 ## 1.13.2 @@ -608,7 +608,7 @@ - 08b652e: fix: Add txIndex to onchain events, fix wrong index being used in the primary key - b36eef2: fix: Extract snapshot on the fly while downloading snapshot - 93e43a8: fix: Use hashes to compare upgrade 'hubble.sh' versions -- 7daaae4: fix: Simplify IP addr fetching, prefering ipv4 +- 7daaae4: fix: Simplify IP addr fetching, preferring ipv4 - ac1f6ac: fix: Fetch envoy config during hubble.sh - baf983f: fix: Consume the FID rate limit only after a successful merge - Updated dependencies [08b652e] @@ -742,7 +742,7 @@ - f00d7d2: fix: Move validatorOrRevokeMessage and storageCache iterators to be managed - 115f1b5: feat: Do the validateOrRevokeMessages job fid-by-fid - 998979d: feat: Warn if there are no incoming connections -- c1bb21c: fix: When retring messages due to failed signers, use a queue +- c1bb21c: fix: When retrying messages due to failed signers, use a queue - 376ae0f: feat: Use a web based network config for hubble - @farcaster/hub-nodejs@0.9.1 From b6bad85b8a7f9ae4c70a8719b63838952332e8bd Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Wed, 3 Jul 2024 12:13:10 -0500 Subject: [PATCH 06/24] fix: update start time to farcaster time when submitting message bundles from sync (#2121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary Describe the changes being made in 1-2 concise sentences. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview The focus of this PR is to replace the usage of `Date.now()` with `getFarcasterTime().unwrapOr(0)` in `syncEngine.ts` for more accurate time tracking during message bundle submission. ### Detailed summary - Replaced `Date.now()` with `getFarcasterTime().unwrapOr(0)` for startTime in `syncEngine.ts` - Improved accuracy in time tracking during message bundle submission. > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/src/network/sync/syncEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hubble/src/network/sync/syncEngine.ts b/apps/hubble/src/network/sync/syncEngine.ts index be879b0d76..2da9f68f8b 100644 --- a/apps/hubble/src/network/sync/syncEngine.ts +++ b/apps/hubble/src/network/sync/syncEngine.ts @@ -1220,7 +1220,7 @@ class SyncEngine extends TypedEmitter { this._syncMergeQ += messages.length; statsd().gauge("syncengine.merge_q", this._syncMergeQ); - const startTime = Date.now(); + const startTime = getFarcasterTime().unwrapOr(0); const results = await this._hub.submitMessageBundle(startTime, MessageBundle.create({ messages }), "sync"); for (let i = 0; i < results.length; i++) { From 4055f7001e4c0b1579ba9a6d806e02ba626dee99 Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Fri, 5 Jul 2024 15:26:05 -0500 Subject: [PATCH 07/24] chore: update cleanup interval for active peers (#2123) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary Describe the changes being made in 1-2 concise sentences. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ ] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR adjusts the cleanup interval for the Peer TTLMap in the sync engine from 36 hours to 25 hours. ### Detailed summary - Adjusted the cleanup interval for the Peer TTLMap from 36 hours to 25 hours > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/src/network/sync/syncEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/hubble/src/network/sync/syncEngine.ts b/apps/hubble/src/network/sync/syncEngine.ts index 2da9f68f8b..4fd3110a5a 100644 --- a/apps/hubble/src/network/sync/syncEngine.ts +++ b/apps/hubble/src/network/sync/syncEngine.ts @@ -59,7 +59,7 @@ import { peerIdFromString } from "@libp2p/peer-id"; // Time to live for peer contact info in the Peer TTLMap const PEER_TTL_MAP_EXPIRATION_TIME_MILLISECONDS = 1000 * 60 * 60 * 24; // 24 hours // Time interval to run cleanup on the Peer TTLMap -const PEER_TTL_MAP_CLEANUP_INTERVAL_MILLISECONDS = 1000 * 60 * 60 * 36; // 36 hours +const PEER_TTL_MAP_CLEANUP_INTERVAL_MILLISECONDS = 1000 * 60 * 60 * 25; // 25 hours // Number of seconds to wait for the network to "settle" before syncing. We will only // attempt to sync messages that are older than this time. From 644f81d6de969b56f97a836f5d5ec569e6a2aef8 Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Sat, 6 Jul 2024 14:24:34 -0500 Subject: [PATCH 08/24] chore: batch info and error logs for message bundles (#2127) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary ``` {"level":30,"time":1720293146285,"pid":90362,"hostname":"mbp.local","component":"Hub","0":"[event_id:453975843913728|farcaster_ts:110833945|fid:585630|hash:0xef173705c8f9c3843c4da43234e767582085e114|message_type:LINK_ADD|source:gossip]","1":"[event_id:453975843917824|farcaster_ts:110833945|fid:485354|hash:0xefba9d8b3f64469ffa5f3fd8c9907cf4f798d0cb|message_type:LINK_ADD|source:gossip]","2":"[event_id:453975843917825|farcaster_ts:110833945|fid:192373|hash:0x0ae9d1dce72f77f9228c0e39c70d4eac62a7442f|message_type:LINK_ADD|source:gossip]","3":"[event_id:453975843917826|farcaster_ts:110833945|fid:633096|hash:0xef823130e0b0e85821c942f518d5030f413cb1be|message_type:LINK_ADD|source:gossip]","4":"[event_id:453975843917827|farcaster_ts:110833945|fid:746504|hash:0xcc682a388d3ff1fa9f1c488afe34bc1eef5c0f7c|message_type:LINK_ADD|source:gossip]","5":"[event_id:453975843917829|farcaster_ts:110833945|fid:190012|hash:0x300a529843da3add0560c0c8be228b69d2628883|message_type:LINK_ADD|source:gossip]","6":"[event_id:453975843917828|farcaster_ts:110833945|fid:579583|hash:0xbf9ebf245dcf7173e7a3622f0a99f974688c3540|message_type:CAST_REMOVE|source:gossip]","7":"[event_id:453975843917830|farcaster_ts:110833945|fid:711341|hash:0x9ffd88aa81af70d6602f819f7a8b2ad76c17f165|message_type:CAST_ADD|source:gossip]","8":"[event_id:453975843921920|farcaster_ts:110833945|fid:270138|hash:0xd26ff734375c0a6a4804a14f042d06ae3fa0e241|message_type:CAST_ADD|source:gossip]","9":"[event_id:453975843921921|farcaster_ts:110833946|fid:305198|hash:0x50111086f8f27d4a7c080e48f20c287ba71a2dbb|message_type:CAST_ADD|source:gossip]","10":"[event_id:453975843921922|farcaster_ts:110833945|fid:3863|hash:0x60beb24804fe0df885fc251b88ac4f4d7e19e0d3|message_type:CAST_ADD|source:gossip]","11":"[event_id:453975843971072|farcaster_ts:110833945|fid:549607|hash:0xec82139fcad13947ef05c6d04d128897c5eefaf8|message_type:REACTION_ADD|source:gossip]","12":"[event_id:453975843971073|farcaster_ts:110833945|fid:265048|hash:0x2d872242ca0d99b8292e4ff7444440694f0afac9|message_type:REACTION_ADD|source:gossip]","13":"[event_id:453975843975168|farcaster_ts:110833945|fid:563445|hash:0xdedd26fb91e8cea03444386f204dadbf28fbbc20|message_type:REACTION_ADD|source:gossip]","14":"[event_id:453975843975169|farcaster_ts:110833945|fid:522942|hash:0x5ecea1832c45f0dea07a5e7db7ee9ababe065385|message_type:REACTION_ADD|source:gossip]","15":"[event_id:453975843975170|farcaster_ts:110833945|fid:520558|hash:0x987f67b9052d37bda525498d85ab2658e551a6b0|message_type:REACTION_ADD|source:gossip]","16":"[event_id:453975843975171|farcaster_ts:110833946|fid:651987|hash:0xe100174210580b70893107eadd1475e871b0ff27|message_type:REACTION_ADD|source:gossip]","17":"[event_id:453975843975172|farcaster_ts:110833945|fid:511684|hash:0x72561ecda54162bb31b7e234cdee3aafc065a067|message_type:REACTION_ADD|source:gossip]","18":"[event_id:453975843975173|farcaster_ts:110833945|fid:563445|hash:0x174b2b29e616c0dc351169a46d0d1a115ce0fe45|message_type:REACTION_ADD|source:gossip]","19":"[event_id:453975843975174|farcaster_ts:110833945|fid:370971|hash:0x7fbd427e27b8718c700466f437b994c04bd6bcc1|message_type:REACTION_ADD|source:gossip]","20":"[event_id:453975843979264|farcaster_ts:110833945|fid:306729|hash:0x9bca22a42956566cbdb7b2de7471ba91ee4a976d|message_type:REACTION_ADD|source:gossip]","21":"[event_id:453975843979265|farcaster_ts:110833945|fid:521421|hash:0x58855581f5a4a2f1a673af6c235fbb7a039616a3|message_type:REACTION_ADD|source:gossip]","22":"[event_id:453975843979266|farcaster_ts:110833945|fid:509854|hash:0x19f48b1c2113b400e37200be1383a92170b02909|message_type:REACTION_ADD|source:gossip]","23":"[event_id:453975843979267|farcaster_ts:110833945|fid:516301|hash:0xd4c626df0a29c2ea3e71872caa1c0f84dda5b62a|message_type:REACTION_ADD|source:gossip]","24":"[event_id:453975843979268|farcaster_ts:110833945|fid:337423|hash:0x9eb8d64713a79f6ed5983004fcdbb4667bf39a82|message_type:REACTION_ADD|source:gossip]","msg":"successful submit messages"} ``` ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ ] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview The focus of this PR is to enhance logging in the `Hub` class by capturing successful and failed message submissions. ### Detailed summary - Added `errorLogs` and `infoLogs` arrays for storing error and info logs respectively - Enhanced logging for successful and failed message submissions with detailed information - Added logging for successful and failed message submissions > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/src/hubble.ts | 52 +++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/apps/hubble/src/hubble.ts b/apps/hubble/src/hubble.ts index 7be20f7c0c..e6a6f65091 100644 --- a/apps/hubble/src/hubble.ts +++ b/apps/hubble/src/hubble.ts @@ -1792,6 +1792,8 @@ export class Hub implements HubInterface { // Merge the messages const mergeResults = await this.engine.mergeMessages(dedupedMessages.map((m) => m.message)); + const errorLogs: string[] = []; + const infoLogs: string[] = []; for (const [j, result] of mergeResults.entries()) { const message = dedupedMessages[j]?.message as Message; const type = messageTypeToName(message.data?.type); @@ -1800,32 +1802,28 @@ export class Hub implements HubInterface { result.match( (eventId) => { - if (this.options.logIndividualMessages) { - const logData = { - eventId, - fid: message.data?.fid, - type: type, - submittedMessage: messageToLog(message), - source, - }; - const msg = "submitMessage success"; - - if (source === "sync") { - log.debug(logData, msg); - } else { - log.info(logData, msg); - } - } else { - this.submitMessageLogger.log(source ?? "unknown-source"); - } + const parts = [ + `event_id:${eventId}`, + `farcaster_ts:${message.data?.timestamp ?? "no-timestamp"}`, + `fid:${message.data?.fid ?? "no-fid"}`, + `hash:${bytesToHexString(message.hash).unwrapOr("no-hash")}`, + `message_type:${type}`, + `source:${source}`, + ]; + infoLogs.push(`[${parts.join("|")}]`); }, (e) => { - // message is a reserved key in some logging systems, so we use submittedMessage instead - const logMessage = log.child({ - submittedMessage: messageToLog(message), - source, - }); - logMessage.warn({ errCode: e.errCode, source }, `submitMessage error: ${e.message}`); + const parts = [ + `farcaster_ts:${message.data?.timestamp ?? "no-timestamp"}`, + `fid:${message.data?.fid ?? "no-fid"}`, + `hash:${bytesToHexString(message.hash).unwrapOr("no-hash")}`, + `message_type:${type}`, + `source:${source ?? "unknown-source"}`, + `error:${e.message}`, + `error_code:${e.errCode}`, + ]; + errorLogs.push(`[${parts.join("|")}]`); + const tags: { [key: string]: string } = { error_code: e.errCode, message_type: type, @@ -1835,6 +1833,12 @@ export class Hub implements HubInterface { }, ); } + if (infoLogs.length > 0) { + log.info(infoLogs, "successful submit messages"); + } + if (errorLogs.length > 0) { + log.error(errorLogs, "failed submit messages"); + } // Convert the merge results to an Array of HubResults with the key const finalResults: HubResult[] = []; From 4794a1afa09d114279d969da8a896b7e35d0762e Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Mon, 8 Jul 2024 12:06:17 -0500 Subject: [PATCH 09/24] chore: update statsd metrics for diff sync (#2132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary Describe the changes being made in 1-2 concise sentences. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ ] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR focuses on adding statistics tracking for different types of sync messages in the `syncEngine.ts` file. ### Detailed summary - Added statistics tracking for on-chain events, FNames, and messages sync messages - Incremented success, error, and deferred counts for each type of sync message > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- apps/hubble/src/network/sync/syncEngine.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/hubble/src/network/sync/syncEngine.ts b/apps/hubble/src/network/sync/syncEngine.ts index 4fd3110a5a..9a1c65bc16 100644 --- a/apps/hubble/src/network/sync/syncEngine.ts +++ b/apps/hubble/src/network/sync/syncEngine.ts @@ -1567,16 +1567,25 @@ class SyncEngine extends TypedEmitter { const result = await this.validateAndMergeOnChainEvents( missingSyncIds.filter((id) => id.type() === SyncIdType.OnChainEvent), ); + statsd().increment("syncengine.sync_messages.onchain.success", result.successCount); + statsd().increment("syncengine.sync_messages.onchain.error", result.errCount); + statsd().increment("syncengine.sync_messages.onchain.deferred", result.deferredCount); // Then Fnames const fnameResult = await this.validateAndMergeFnames( missingSyncIds.filter((id) => id.type() === SyncIdType.FName), ); result.addResult(fnameResult); + statsd().increment("syncengine.sync_messages.fname.success", fnameResult.successCount); + statsd().increment("syncengine.sync_messages.fname.error", fnameResult.errCount); + statsd().increment("syncengine.sync_messages.fname.deferred", fnameResult.deferredCount); // And finally messages const messagesResult = await this.fetchAndMergeMessages(missingMessageIds, rpcClient); result.addResult(messagesResult); + statsd().increment("syncengine.sync_messages.message.success", messagesResult.successCount); + statsd().increment("syncengine.sync_messages.message.error", messagesResult.errCount); + statsd().increment("syncengine.sync_messages.message.deferred", messagesResult.deferredCount); this.curSync.fullResult.addResult(result); From a7fdaa16e9980b6f1d7a8e4e0e30727b6bd026b2 Mon Sep 17 00:00:00 2001 From: dhairya1008 <153992254+dhairya1008@users.noreply.github.com> Date: Tue, 9 Jul 2024 04:01:15 +0530 Subject: [PATCH 10/24] apps/hubble: fix docker-compose.yml warning (#2112) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "version is obsolelte" Docker has made this line obsolete, no need to nention docker compose version ## Motivation Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. ## Change Summary Describe the changes being made in 1-2 concise sentences. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [ ] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR updates the `docker-compose.yml` file for the `hubble` service to use version 3.9 and the latest image `farcasterxyz/hubble:latest`. ### Detailed summary - Updated `docker-compose.yml` to version 3.9 - Changed `hubble` service image to `farcasterxyz/hubble:latest` > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` Signed-off-by: dhairya1008 Co-authored-by: dhairya1899 <17itdhairya.parmar@gmail.com> --- apps/hubble/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/hubble/docker-compose.yml b/apps/hubble/docker-compose.yml index 95fb411dbe..ce6ab4b496 100644 --- a/apps/hubble/docker-compose.yml +++ b/apps/hubble/docker-compose.yml @@ -8,8 +8,6 @@ # # git fetch --tags --force && git checkout @latest && docker compose up -version: '3.9' - services: hubble: image: farcasterxyz/hubble:latest From 4518058412de5a1d1b7d58087cdcddbd0b0139ee Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Mon, 8 Jul 2024 16:04:24 -0700 Subject: [PATCH 11/24] fix(shuttle): Handle missing group key (#2133) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation We didn't handle the case where the group key wasn't already created. ## Change Summary Handle it. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) --- ## PR-Codex overview This PR focuses on gracefully handling "no such key" errors when querying a group on the first start in the `shuttle` package. ### Detailed summary - Added error handling for "no such key" scenario - Skips group creation if key doesn't exist - Improved robustness in handling ReplyError > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --------- Co-authored-by: Ken Goldfarb --- .changeset/calm-rings-lie.md | 5 +++++ packages/shuttle/src/shuttle/eventStream.ts | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .changeset/calm-rings-lie.md diff --git a/.changeset/calm-rings-lie.md b/.changeset/calm-rings-lie.md new file mode 100644 index 0000000000..8109da4593 --- /dev/null +++ b/.changeset/calm-rings-lie.md @@ -0,0 +1,5 @@ +--- +"@farcaster/shuttle": patch +--- + +Gracefully handle "no such key" when querying group on first start diff --git a/packages/shuttle/src/shuttle/eventStream.ts b/packages/shuttle/src/shuttle/eventStream.ts index 7cbc98eb53..1a86bfd766 100644 --- a/packages/shuttle/src/shuttle/eventStream.ts +++ b/packages/shuttle/src/shuttle/eventStream.ts @@ -42,11 +42,21 @@ export class EventStreamConnection { * Creates a consumer group for the given stream. */ async createGroup(key: string, consumerGroup: string) { + // Check if the group already exists try { - // Check if the group already exists const groups = (await this.client.xinfo("GROUPS", key)) as [string, string][]; if (groups.some(([_fieldName, groupName]) => groupName === consumerGroup)) return; + } catch (e: unknown) { + if (typeof e === "object" && e !== null && e instanceof ReplyError) { + if ("message" in e && (e.message as string).startsWith("ERR no such key")) { + // Ignore if the group hasn't been created yet + } else { + throw e; + } + } + } + try { // Otherwise create the group return await this.client.xgroup("CREATE", key, consumerGroup, "0", "MKSTREAM"); } catch (e: unknown) { From 795815afbe6b6e253e1419a5497e9c61aa56b7fa Mon Sep 17 00:00:00 2001 From: aquatic <95545650+aquaticone@users.noreply.github.com> Date: Mon, 8 Jul 2024 17:10:07 -0600 Subject: [PATCH 12/24] fix(hubble): hub operator fid is not set docker compose (#2129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation Fixes #2088 #2117 There is likely an underlying issue with the commander configuration but this is a quick change that gets this working for those having trouble getting the container up and running. ## Change Summary Reordered CLI arguments in docker-compose.yml. ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) ## Additional Context If this is a relatively large or complex change, provide more details here that will help reviewers --- ## PR-Codex overview This PR updates the `docker-compose.yml` file for the `@farcaster/hubble` app to fix an issue with CLI arguments order affecting the hub operator FID. ### Detailed summary - Added `--hub-operator-fid ${HUB_OPERATOR_FID:-0}` to fix unset hub operator FID in docker-compose.yml. - Rearranged CLI arguments to ensure correct order and parameter assignment. > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/new-kids-provide.md | 5 +++++ apps/hubble/docker-compose.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/new-kids-provide.md diff --git a/.changeset/new-kids-provide.md b/.changeset/new-kids-provide.md new file mode 100644 index 0000000000..3177f46c78 --- /dev/null +++ b/.changeset/new-kids-provide.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +fixed issue with cli arguments order in docker-compose.yml causing hub operator fid to be unset diff --git a/apps/hubble/docker-compose.yml b/apps/hubble/docker-compose.yml index ce6ab4b496..76b7358839 100644 --- a/apps/hubble/docker-compose.yml +++ b/apps/hubble/docker-compose.yml @@ -32,10 +32,10 @@ services: --eth-mainnet-rpc-url $ETH_MAINNET_RPC_URL --l2-rpc-url $OPTIMISM_L2_RPC_URL --network ${FC_NETWORK_ID:-1} + --hub-operator-fid ${HUB_OPERATOR_FID:-0} --rpc-subscribe-per-ip-limit ${RPC_SUBSCRIBE_PER_IP_LIMIT:-4} -b ${BOOTSTRAP_NODE:-/dns/nemes.farcaster.xyz/tcp/2282} --statsd-metrics-server $STATSD_METRICS_SERVER - --hub-operator-fid ${HUB_OPERATOR_FID:-0} --opt-out-diagnostics ${HUB_OPT_OUT_DIAGNOSTICS:-false} ${HUB_OPTIONS:-} ports: From 2bae6fb98bfd7359c2cf1c5701c6378662e66d30 Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Mon, 8 Jul 2024 16:16:00 -0700 Subject: [PATCH 13/24] chore: Update curve25519-dalek from 4.1.1 to 4.1.3 in Rust extension (#2134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation This addresses a vulnerability. ## Change Summary Update ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. - [x] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) --- ## PR-Codex overview This PR updates the `curve25519-dalek` package in the Rust extension of the `@farcaster/hubble` addon from version 4.1.1 to 4.1.3. ### Detailed summary - Updated `curve25519-dalek` package version from 4.1.1 to 4.1.3 - Removed `platforms` package - Updated checksum for `curve25519-dalek` package > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .changeset/silver-wombats-rule.md | 5 +++++ apps/hubble/src/addon/Cargo.lock | 11 ++--------- 2 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 .changeset/silver-wombats-rule.md diff --git a/.changeset/silver-wombats-rule.md b/.changeset/silver-wombats-rule.md new file mode 100644 index 0000000000..79a236ad81 --- /dev/null +++ b/.changeset/silver-wombats-rule.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +chore: Update curve25519-dalek from 4.1.1 to 4.1.3 in Rust extension diff --git a/apps/hubble/src/addon/Cargo.lock b/apps/hubble/src/addon/Cargo.lock index 2afa2fd9a5..f306dcf62b 100644 --- a/apps/hubble/src/addon/Cargo.lock +++ b/apps/hubble/src/addon/Cargo.lock @@ -433,16 +433,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.1" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", "digest", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -1221,12 +1220,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "platforms" -version = "3.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4503fa043bf02cee09a9582e9554b4c6403b2ef55e4612e96561d294419429f8" - [[package]] name = "powerfmt" version = "0.2.0" From d1500c007b0ff0e3914b95f2943cc71f77ae94ff Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Mon, 8 Jul 2024 16:27:50 -0700 Subject: [PATCH 14/24] chore: Update pull request template (#2135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? Our template is in need of a makeover. Made the following changes: - Change **Motivation** to **Why is this change needed?** to emphasize what we mean by Motivation. - Remove the **Change Summary** section since this is automatically summarized by PR-Codex bot. - Remove requirement to sign commits, since this doesn't provide any _real_ security and makes it slightly harder for contributors. Since GitHub doesn't prevent them from creating the PR in the first place if the HEAD commit isn't signed, we might as well allow it so we don't need to manually bypass. - Removed **Additional Context** section since this seemed superfluous. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [x] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR removes the `Change Summary` section from the pull request template and updates the checklist options. ### Detailed summary - Removed `Change Summary` section from PR template - Updated checklist options in PR template > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .github/PULL_REQUEST_TEMPLATE.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1b15e56d22..c61a658b2f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,7 @@ -## Motivation +## Why is this change needed? Describe why this issue should be fixed and link to any relevant design docs, issues or other relevant items. -## Change Summary - -Describe the changes being made in 1-2 concise sentences. - ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ @@ -14,8 +10,3 @@ _Choose all relevant options below by adding an `x` now or at any time before su - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. -- [ ] All [commits have been signed](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#22-signing-commits) - -## Additional Context - -If this is a relatively large or complex change, provide more details here that will help reviewers From fcb76cd9adcc2d9eff9f1a1832fc89c8bacff117 Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Mon, 8 Jul 2024 18:59:58 -0700 Subject: [PATCH 15/24] chore(shuttle) Release 0.4.3 (#2136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? Releases a bug fix that is affecting Shuttle users. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR updates the version of `@farcaster/shuttle` package to `0.4.3` and includes a patch to handle "no such key" error when querying group on first start. ### Detailed summary - Updated package version to `0.4.3` - Patched to handle "no such key" error when querying group on first start > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/calm-rings-lie.md | 5 ----- packages/shuttle/CHANGELOG.md | 6 ++++++ packages/shuttle/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/calm-rings-lie.md diff --git a/.changeset/calm-rings-lie.md b/.changeset/calm-rings-lie.md deleted file mode 100644 index 8109da4593..0000000000 --- a/.changeset/calm-rings-lie.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/shuttle": patch ---- - -Gracefully handle "no such key" when querying group on first start diff --git a/packages/shuttle/CHANGELOG.md b/packages/shuttle/CHANGELOG.md index 9f6a9239a3..3ec80d2a66 100644 --- a/packages/shuttle/CHANGELOG.md +++ b/packages/shuttle/CHANGELOG.md @@ -1,5 +1,11 @@ # @farcaster/hub-shuttle +## 0.4.3 + +### Patch Changes + +- 45180584: Gracefully handle "no such key" when querying group on first start + ## 0.4.2 ### Patch Changes diff --git a/packages/shuttle/package.json b/packages/shuttle/package.json index fb414a7a4b..01bf4097e6 100644 --- a/packages/shuttle/package.json +++ b/packages/shuttle/package.json @@ -1,6 +1,6 @@ { "name": "@farcaster/shuttle", - "version": "0.4.2", + "version": "0.4.3", "main": "./dist/index.js", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", From 864261b79810b98d7ecc2ee5636bfcb13d55bec7 Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Tue, 9 Jul 2024 14:21:06 -0700 Subject: [PATCH 16/24] feat(shuttle): Allow RedisClient's client to be a cluster instance (#2139) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? We were artificially limiting the accepted type to a `Redis` instance, when a `Cluster` instance would also work fine. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR adds support for Redis cluster instances in the `shuttle` package. ### Detailed summary - Added support for Redis cluster instances in `shuttle/redis.ts` - Updated `RedisClient` constructor to accept either `Redis` or `Cluster` client - Updated `create` method to handle Redis cluster instances > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/smart-garlics-sniff.md | 5 +++++ packages/shuttle/src/shuttle/redis.ts | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 .changeset/smart-garlics-sniff.md diff --git a/.changeset/smart-garlics-sniff.md b/.changeset/smart-garlics-sniff.md new file mode 100644 index 0000000000..f0266899c2 --- /dev/null +++ b/.changeset/smart-garlics-sniff.md @@ -0,0 +1,5 @@ +--- +"@farcaster/shuttle": patch +--- + +feat(shuttle) Allow Redis client to be a cluster instance diff --git a/packages/shuttle/src/shuttle/redis.ts b/packages/shuttle/src/shuttle/redis.ts index 357ae53ec4..67c6840737 100644 --- a/packages/shuttle/src/shuttle/redis.ts +++ b/packages/shuttle/src/shuttle/redis.ts @@ -1,4 +1,4 @@ -import { Redis, RedisOptions } from "ioredis"; +import { Redis, Cluster, RedisOptions } from "ioredis"; export const getRedisClient = (redisUrl: string, redisOpts?: RedisOptions) => { const client = new Redis(redisUrl, { @@ -10,8 +10,8 @@ export const getRedisClient = (redisUrl: string, redisOpts?: RedisOptions) => { }; export class RedisClient { - public client: Redis; - constructor(client: Redis) { + public client: Redis | Cluster; + constructor(client: Redis | Cluster) { this.client = client; } static create(redisUrl: string, redisOpts?: RedisOptions) { From 6b037f48c0c1491688ce59c01b37ed30e41193fc Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Tue, 9 Jul 2024 14:35:07 -0700 Subject: [PATCH 17/24] chore(shuttle): Release 0.4.4 (#2140) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? Includes a fix allowing Redis Clusters to be used with the `RedisClient` class. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR updates the version of `@farcaster/shuttle` package to `0.4.4` and introduces a new feature allowing the Redis client to be a cluster instance. ### Detailed summary - Updated `@farcaster/shuttle` package version to `0.4.4` - Added feature: Redis client can now be a cluster instance > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/smart-garlics-sniff.md | 5 ----- packages/shuttle/CHANGELOG.md | 6 ++++++ packages/shuttle/package.json | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) delete mode 100644 .changeset/smart-garlics-sniff.md diff --git a/.changeset/smart-garlics-sniff.md b/.changeset/smart-garlics-sniff.md deleted file mode 100644 index f0266899c2..0000000000 --- a/.changeset/smart-garlics-sniff.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/shuttle": patch ---- - -feat(shuttle) Allow Redis client to be a cluster instance diff --git a/packages/shuttle/CHANGELOG.md b/packages/shuttle/CHANGELOG.md index 3ec80d2a66..6561bbf29a 100644 --- a/packages/shuttle/CHANGELOG.md +++ b/packages/shuttle/CHANGELOG.md @@ -1,5 +1,11 @@ # @farcaster/hub-shuttle +## 0.4.4 + +### Patch Changes + +- 864261b7: feat(shuttle) Allow Redis client to be a cluster instance + ## 0.4.3 ### Patch Changes diff --git a/packages/shuttle/package.json b/packages/shuttle/package.json index 01bf4097e6..7273e20061 100644 --- a/packages/shuttle/package.json +++ b/packages/shuttle/package.json @@ -1,6 +1,6 @@ { "name": "@farcaster/shuttle", - "version": "0.4.3", + "version": "0.4.4", "main": "./dist/index.js", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", From b5ff774a9ca0854bb492ca3ada6febe92dcf0d4e Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Tue, 9 Jul 2024 18:50:56 -0500 Subject: [PATCH 18/24] feat: add hub service agreement - there will be no rewards for running a hub (#2142) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? ⚠️ THERE WILL BE NO REWARDS FOR RUNNING A HUB ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR adds a hub service agreement feature to the Hubble script, requiring hub operators to agree to terms before starting the hub. ### Detailed summary - Added function `prompt_for_hub_operator_agreement` to prompt hub operators to agree to terms - Prompt displays a warning about no rewards for running the hub - Operators must type "Yes" to proceed - Implemented input validation for agreement - Integration of agreement prompt before starting or upgrading the hub > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/poor-queens-lick.md | 5 +++++ scripts/hubble.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .changeset/poor-queens-lick.md diff --git a/.changeset/poor-queens-lick.md b/.changeset/poor-queens-lick.md new file mode 100644 index 0000000000..6e2f8ceab2 --- /dev/null +++ b/.changeset/poor-queens-lick.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +feat: add hub service agreement - there will be no rewards for running a hub diff --git a/scripts/hubble.sh b/scripts/hubble.sh index 6a110cfedd..3ae061431d 100755 --- a/scripts/hubble.sh +++ b/scripts/hubble.sh @@ -120,6 +120,26 @@ fetch_latest_docker_compose_and_dashboard() { fetch_file_from_repo "$GRAFANA_INI_PATH" "grafana/grafana.ini" } +prompt_for_hub_operator_agreement() { + # Check if stdin is a terminal + if [ -t 0 ]; then + while true; do + printf "⚠️ IMPORTANT: You will NOT get any rewards for running this hub\n" + printf "> Please type \"Yes\" to continue: " + read -r response + case $(printf "%s" "$response" | tr '[:upper:]' '[:lower:]') in + yes|y) + printf "✅ You have agreed to the terms of service. Proceeding with hub startup...\n" + return 0 + ;; + *) + printf "[i] Incorrect input. Please try again.\n" + ;; + esac + done + fi +} + validate_and_store() { local rpc_name=$1 local expected_chain_id=$2 @@ -498,6 +518,9 @@ if [ "$1" == "up" ]; then # Setup the docker-compose command set_compose_command + # Prompt for hub operator agreement + prompt_for_hub_operator_agreement + # Run docker compose up -d hubble $COMPOSE_CMD up -d hubble statsd grafana @@ -543,6 +566,9 @@ if [ "$1" == "upgrade" ]; then # Call the function to set the COMPOSE_CMD variable set_compose_command + # Prompt for hub operator agreement + prompt_for_hub_operator_agreement + # Update the env file if needed write_env_file From fa5eef40587e7bda5f4aafca506357889a56d5ea Mon Sep 17 00:00:00 2001 From: Sanjay Date: Tue, 9 Jul 2024 17:09:14 -0700 Subject: [PATCH 19/24] fix: Increase message threshold to reduce snapshot bandwidth usage (#2145) ## Why is this change needed? Snapshots are using too much bandwitdh, increase the threshold to about 10% of current messages (~470M) ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- .changeset/light-cobras-judge.md | 5 +++++ apps/hubble/src/defaultConfig.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/light-cobras-judge.md diff --git a/.changeset/light-cobras-judge.md b/.changeset/light-cobras-judge.md new file mode 100644 index 0000000000..736e5d9ca0 --- /dev/null +++ b/.changeset/light-cobras-judge.md @@ -0,0 +1,5 @@ +--- +"@farcaster/hubble": patch +--- + +fix: Increase message threshold to reduce snapshot bandwidth usage diff --git a/apps/hubble/src/defaultConfig.ts b/apps/hubble/src/defaultConfig.ts index e335c9c21a..60b3afe3fb 100644 --- a/apps/hubble/src/defaultConfig.ts +++ b/apps/hubble/src/defaultConfig.ts @@ -10,7 +10,7 @@ const DEFAULT_GOSSIP_PORT = 2282; const DEFAULT_RPC_PORT = 2283; const DEFAULT_HTTP_API_PORT = 2281; const DEFAULT_NETWORK = 3; // Farcaster Devnet -export const DEFAULT_CATCHUP_SYNC_SNAPSHOT_MESSAGE_LIMIT = 25_000_000; +export const DEFAULT_CATCHUP_SYNC_SNAPSHOT_MESSAGE_LIMIT = 50_000_000; export const Config = { /** Path to a PeerId file */ From 17a81b7aaa779862a409f2cbab31b192a6e7ca94 Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Tue, 9 Jul 2024 19:17:57 -0500 Subject: [PATCH 20/24] chore: release 1.13.6 (#2146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? Release 1.13.6 ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR updates the version of `@farcaster/hubble` to `1.13.6` and includes various patches and features related to message delays, snapshot bandwidth, hub service agreements, peer maps, clock skew validation, and Rust extension updates. ### Detailed summary - Added stats for bundle message delays and stale contact info - Increased message threshold to reduce snapshot bandwidth usage - Fixed cli arguments order issue in docker-compose.yml - Added hub service agreement: no rewards for running a hub - Added unique peer map to sync engine for active peers - Validated gossip message for clock skew - Updated Rust extension to curve25519-dalek 4.1.3 > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- .changeset/dull-dancers-float.md | 5 ----- .changeset/light-cobras-judge.md | 5 ----- .changeset/new-kids-provide.md | 5 ----- .changeset/poor-queens-lick.md | 5 ----- .changeset/popular-shoes-bathe.md | 5 ----- .changeset/silent-planes-promise.md | 5 ----- .changeset/silver-wombats-rule.md | 5 ----- apps/hubble/CHANGELOG.md | 12 ++++++++++++ apps/hubble/package.json | 2 +- 9 files changed, 13 insertions(+), 36 deletions(-) delete mode 100644 .changeset/dull-dancers-float.md delete mode 100644 .changeset/light-cobras-judge.md delete mode 100644 .changeset/new-kids-provide.md delete mode 100644 .changeset/poor-queens-lick.md delete mode 100644 .changeset/popular-shoes-bathe.md delete mode 100644 .changeset/silent-planes-promise.md delete mode 100644 .changeset/silver-wombats-rule.md diff --git a/.changeset/dull-dancers-float.md b/.changeset/dull-dancers-float.md deleted file mode 100644 index 440b5ba934..0000000000 --- a/.changeset/dull-dancers-float.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -chore: add stats for bundle message delays, stale contact info diff --git a/.changeset/light-cobras-judge.md b/.changeset/light-cobras-judge.md deleted file mode 100644 index 736e5d9ca0..0000000000 --- a/.changeset/light-cobras-judge.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -fix: Increase message threshold to reduce snapshot bandwidth usage diff --git a/.changeset/new-kids-provide.md b/.changeset/new-kids-provide.md deleted file mode 100644 index 3177f46c78..0000000000 --- a/.changeset/new-kids-provide.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -fixed issue with cli arguments order in docker-compose.yml causing hub operator fid to be unset diff --git a/.changeset/poor-queens-lick.md b/.changeset/poor-queens-lick.md deleted file mode 100644 index 6e2f8ceab2..0000000000 --- a/.changeset/poor-queens-lick.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -feat: add hub service agreement - there will be no rewards for running a hub diff --git a/.changeset/popular-shoes-bathe.md b/.changeset/popular-shoes-bathe.md deleted file mode 100644 index af1864f2aa..0000000000 --- a/.changeset/popular-shoes-bathe.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -feat: add unique peer map to sync engine to represent current active peers diff --git a/.changeset/silent-planes-promise.md b/.changeset/silent-planes-promise.md deleted file mode 100644 index 9442bd517f..0000000000 --- a/.changeset/silent-planes-promise.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -fix: validate gossip message for clock skew diff --git a/.changeset/silver-wombats-rule.md b/.changeset/silver-wombats-rule.md deleted file mode 100644 index 79a236ad81..0000000000 --- a/.changeset/silver-wombats-rule.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@farcaster/hubble": patch ---- - -chore: Update curve25519-dalek from 4.1.1 to 4.1.3 in Rust extension diff --git a/apps/hubble/CHANGELOG.md b/apps/hubble/CHANGELOG.md index 6c366c5564..dcc9017cd3 100644 --- a/apps/hubble/CHANGELOG.md +++ b/apps/hubble/CHANGELOG.md @@ -1,5 +1,17 @@ # @farcaster/hubble +## 1.13.6 + +### Patch Changes + +- fdcc3b52: chore: add stats for bundle message delays, stale contact info +- fa5eef40: fix: Increase message threshold to reduce snapshot bandwidth usage +- 795815af: fixed issue with cli arguments order in docker-compose.yml causing hub operator fid to be unset +- b5ff774a: feat: add hub service agreement - there will be no rewards for running a hub +- 2a82b3dc: feat: add unique peer map to sync engine to represent current active peers +- aa02a48d: fix: validate gossip message for clock skew +- 2bae6fb9: chore: Update curve25519-dalek from 4.1.1 to 4.1.3 in Rust extension + ## 1.13.5 ### Patch Changes diff --git a/apps/hubble/package.json b/apps/hubble/package.json index a9c1cab45e..b550777b43 100644 --- a/apps/hubble/package.json +++ b/apps/hubble/package.json @@ -1,6 +1,6 @@ { "name": "@farcaster/hubble", - "version": "1.13.5", + "version": "1.13.6", "description": "Farcaster Hub", "author": "", "license": "", From 1320371c70008d347d7d50fe01c2ffcb1dc9b999 Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Wed, 10 Jul 2024 00:34:38 -0700 Subject: [PATCH 21/24] feat(shuttle): Allow changing subscriber batch size or flush interval (#2147) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? The way we were writing events to the stream was resulting in a lot more separate requests to Redis since we weren't actually batching them together. Change this so we leverage the multi-argument version of `XADD` so that throughput can be increased at higher volumes. Accompanying this are the introduction of a few more configuration options that allow us to tweak the throughput of the `HubSubscriber`. While here, do a minor version bump since we're slightly changing how events are written to the stream. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [x] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR updates the `@farcaster/shuttle` package to version 0.5.0, introducing customization for event batch size and time between flushes. ### Detailed summary - Updated package version to 0.5.0 - Added customization for event batch size and time between flushes - Modified event handling logic in `EventStreamHubSubscriber` > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- packages/shuttle/CHANGELOG.md | 6 +++ packages/shuttle/package.json | 2 +- packages/shuttle/src/shuttle/hubSubscriber.ts | 42 +++++++++++++------ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/packages/shuttle/CHANGELOG.md b/packages/shuttle/CHANGELOG.md index 6561bbf29a..bf91d5f199 100644 --- a/packages/shuttle/CHANGELOG.md +++ b/packages/shuttle/CHANGELOG.md @@ -1,5 +1,11 @@ # @farcaster/hub-shuttle +## 0.5.0 + +### Minor Changes + +- Support customization of event batch size and time between flushes + ## 0.4.4 ### Patch Changes diff --git a/packages/shuttle/package.json b/packages/shuttle/package.json index 7273e20061..2e5e04158f 100644 --- a/packages/shuttle/package.json +++ b/packages/shuttle/package.json @@ -1,6 +1,6 @@ { "name": "@farcaster/shuttle", - "version": "0.4.4", + "version": "0.5.0", "main": "./dist/index.js", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", diff --git a/packages/shuttle/src/shuttle/hubSubscriber.ts b/packages/shuttle/src/shuttle/hubSubscriber.ts index 7cb0d87d20..b596d3fdcf 100644 --- a/packages/shuttle/src/shuttle/hubSubscriber.ts +++ b/packages/shuttle/src/shuttle/hubSubscriber.ts @@ -175,8 +175,12 @@ export class EventStreamHubSubscriber extends BaseHubSubscriber { private redis: RedisClient; public readonly streamKey: string; public readonly redisKey: string; - private eventsToAdd: HubEvent[]; - private eventBatchSize = 100; + private eventsToAdd: [HubEvent, Buffer][]; + public eventBatchSize = 100; + private eventBatchLastFlushedAt = 0; + public maxTimeBetweenBatchFlushes = 200; // Millis + public maxBatchBytesBeforeForceFlush = 2 ** 20; // 2 MiB + private eventBatchBytes = 0; constructor( label: string, @@ -208,17 +212,29 @@ export class EventStreamHubSubscriber extends BaseHubSubscriber { } public override async processHubEvent(event: HubEvent): Promise { - this.eventsToAdd.push(event); - if (this.eventsToAdd.length >= this.eventBatchSize) { - let lastEventId: number | undefined; - for (const evt of this.eventsToAdd) { - await this.eventStream.add(this.streamKey, Buffer.from(HubEvent.encode(evt).finish())); - lastEventId = evt.id; - } - if (lastEventId) { - await this.redis.setLastProcessedEvent(this.redisKey, lastEventId); - } - this.eventsToAdd = []; + const eventBytes = Buffer.from(HubEvent.encode(event).finish()); + this.eventBatchBytes += eventBytes.length; + this.eventsToAdd.push([event, eventBytes]); + if ( + this.eventsToAdd.length >= this.eventBatchSize || + this.eventBatchBytes >= this.maxBatchBytesBeforeForceFlush || + Date.now() - this.eventBatchLastFlushedAt > this.maxTimeBetweenBatchFlushes + ) { + // Empties the current batch + const eventBatch = this.eventsToAdd.splice(0, this.eventsToAdd.length); + + // Copies the removed events to the stream + await this.eventStream.add( + this.streamKey, + eventBatch.map(([_event, eventBytes]) => eventBytes), + ); + + this.eventBatchLastFlushedAt = Date.now(); + + // biome-ignore lint/style/noNonNullAssertion: batch always has at least one event + const [evt, eventBytes] = eventBatch[eventBatch.length - 1]!; + const lastEventId = evt.id; + await this.redis.setLastProcessedEvent(this.redisKey, lastEventId); } return true; From b1dc30a8146f9a8aa8b75553106d6cf75ebd8b7b Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Wed, 10 Jul 2024 08:23:30 -0700 Subject: [PATCH 22/24] fix(shuttle): Fix example app to work with Shuttle 0.5.0+ (#2148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? Our build process didn't catch this. Will need to investigate separately. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview The focus of this PR is to update dependencies and support clustering in the `getWorker` and `getQueue` functions. ### Detailed summary - Updated `@farcaster/shuttle` dependency to `^0.5.0` - Added support for `Cluster` in `getWorker` function - Added support for `Cluster` in `getQueue` function > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- packages/shuttle/src/example-app/package.json | 2 +- packages/shuttle/src/example-app/worker.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/shuttle/src/example-app/package.json b/packages/shuttle/src/example-app/package.json index 344bd42273..5820babc1c 100644 --- a/packages/shuttle/src/example-app/package.json +++ b/packages/shuttle/src/example-app/package.json @@ -4,7 +4,7 @@ "main": "./app.ts", "license": "MIT", "dependencies": { - "@farcaster/shuttle": "^0.2.0", + "@farcaster/shuttle": "^0.5.0", "@figma/hot-shots": "^9.0.0-figma.1", "commander": "^11.0.0", "ioredis": "^5.3.2", diff --git a/packages/shuttle/src/example-app/worker.ts b/packages/shuttle/src/example-app/worker.ts index ccecc262b8..d3deff5c0c 100644 --- a/packages/shuttle/src/example-app/worker.ts +++ b/packages/shuttle/src/example-app/worker.ts @@ -1,11 +1,11 @@ -import { Redis } from "ioredis"; +import { Cluster, Redis } from "ioredis"; import { Job, Queue, Worker } from "bullmq"; import { App } from "./app"; import { pino } from "pino"; const QUEUE_NAME = "default"; -export function getWorker(app: App, redis: Redis, log: pino.Logger, concurrency = 1) { +export function getWorker(app: App, redis: Redis | Cluster, log: pino.Logger, concurrency = 1) { const worker = new Worker( QUEUE_NAME, async (job: Job) => { @@ -38,7 +38,7 @@ export function getWorker(app: App, redis: Redis, log: pino.Logger, concurrency return worker; } -export function getQueue(redis: Redis) { +export function getQueue(redis: Redis | Cluster) { return new Queue("default", { connection: redis, defaultJobOptions: { attempts: 3, backoff: { delay: 1000, type: "exponential" } }, From 1365f1f88e6a675d1109d2f18db9f69df01fbc5c Mon Sep 17 00:00:00 2001 From: Shane da Silva Date: Wed, 10 Jul 2024 13:12:28 -0700 Subject: [PATCH 23/24] fix(shuttle): Fix resetting of batch flush logic when exceeding byte limit (#2151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? This results in the batch being flushed after each event since we don't reset the count on flush. ## Merge Checklist - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [x] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [ ] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview This PR updates the version to 0.5.1, fixing a bug related to the reset of the total batch bytes limit in `shuttle`. ### Detailed summary - Updated version to 0.5.1 - Fixed reset of limit for total batch bytes in `hubSubscriber.ts` > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- packages/shuttle/CHANGELOG.md | 6 ++++++ packages/shuttle/package.json | 2 +- packages/shuttle/src/shuttle/hubSubscriber.ts | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/shuttle/CHANGELOG.md b/packages/shuttle/CHANGELOG.md index bf91d5f199..019239cf87 100644 --- a/packages/shuttle/CHANGELOG.md +++ b/packages/shuttle/CHANGELOG.md @@ -1,5 +1,11 @@ # @farcaster/hub-shuttle +## 0.5.1 + +### Patch Changes + +- Fix reset of limit for total batch bytes + ## 0.5.0 ### Minor Changes diff --git a/packages/shuttle/package.json b/packages/shuttle/package.json index 2e5e04158f..c7f1801cf8 100644 --- a/packages/shuttle/package.json +++ b/packages/shuttle/package.json @@ -1,6 +1,6 @@ { "name": "@farcaster/shuttle", - "version": "0.5.0", + "version": "0.5.1", "main": "./dist/index.js", "module": "./dist/index.mjs", "types": "./dist/index.d.ts", diff --git a/packages/shuttle/src/shuttle/hubSubscriber.ts b/packages/shuttle/src/shuttle/hubSubscriber.ts index b596d3fdcf..bf42c65fb6 100644 --- a/packages/shuttle/src/shuttle/hubSubscriber.ts +++ b/packages/shuttle/src/shuttle/hubSubscriber.ts @@ -222,6 +222,7 @@ export class EventStreamHubSubscriber extends BaseHubSubscriber { ) { // Empties the current batch const eventBatch = this.eventsToAdd.splice(0, this.eventsToAdd.length); + this.eventBatchBytes = 0; // Copies the removed events to the stream await this.eventStream.add( From e304f0b2c0f8f4206997fd71b04236b373714055 Mon Sep 17 00:00:00 2001 From: Wasif Iqbal Date: Wed, 10 Jul 2024 17:44:54 -0500 Subject: [PATCH 24/24] fix: force acknowledgement of no rewards for existing hubs (#2154) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why is this change needed? - Force acknowledgement of no rewards for hubs that are already running ## Merge Checklist _Choose all relevant options below by adding an `x` now or at any time before submitting for review_ - [x] PR title adheres to the [conventional commits](https://www.conventionalcommits.org/en/v1.0.0/) standard - [ ] PR has a [changeset](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#35-adding-changesets) - [x] PR has been tagged with a change label(s) (i.e. documentation, feature, bugfix, or chore) - [ ] PR includes [documentation](https://github.com/farcasterxyz/hub-monorepo/blob/main/CONTRIBUTING.md#32-writing-docs) if necessary. --- ## PR-Codex overview The focus of this PR is to add a prompt for hub operator agreement before proceeding with hub startup. ### Detailed summary - Added a function `prompt_for_hub_operator_agreement()` to handle agreement prompt - Introduced `update_env_file()` to update or create `.env` file with agreement status - Improved user interaction with clear messages and input validation - Ensured hub operator agreement before executing hub-related commands > ✨ Ask PR-Codex anything about this PR by commenting with `/codex {your question}` --- scripts/hubble.sh | 96 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 23 deletions(-) diff --git a/scripts/hubble.sh b/scripts/hubble.sh index 3ae061431d..7aea8dfedf 100755 --- a/scripts/hubble.sh +++ b/scripts/hubble.sh @@ -120,24 +120,77 @@ fetch_latest_docker_compose_and_dashboard() { fetch_file_from_repo "$GRAFANA_INI_PATH" "grafana/grafana.ini" } +# Prompt for hub operator agreement prompt_for_hub_operator_agreement() { - # Check if stdin is a terminal - if [ -t 0 ]; then - while true; do - printf "⚠️ IMPORTANT: You will NOT get any rewards for running this hub\n" - printf "> Please type \"Yes\" to continue: " - read -r response - case $(printf "%s" "$response" | tr '[:upper:]' '[:lower:]') in - yes|y) - printf "✅ You have agreed to the terms of service. Proceeding with hub startup...\n" - return 0 - ;; - *) - printf "[i] Incorrect input. Please try again.\n" - ;; - esac - done - fi + ( + env_file=".env" + + update_env_file() { + key="AGREE_NO_REWARDS_FOR_ME" + value="true" + temp_file="${env_file}.tmp" + + if [ -f "$env_file" ]; then + # File exists, update or append + updated=0 + while IFS= read -r line || [ -n "$line" ]; do + if [ "${line%%=*}" = "$key" ]; then + echo "$key=$value" >>"$temp_file" + updated=1 + else + echo "$line" >>"$temp_file" + fi + done <"$env_file" + + if [ $updated -eq 0 ]; then + echo "$key=$value" >>"$temp_file" + fi + + mv "$temp_file" "$env_file" + else + # File doesn't exist, create it + echo "$key=$value" >"$env_file" + fi + } + + prompt_agreement() { + tried=0 + while true; do + printf "⚠️ IMPORTANT: You will NOT get any rewards for running this hub\n" + printf "> Please type \"Yes\" to continue: " + read -r response + case $(printf "%s" "$response" | tr '[:upper:]' '[:lower:]') in + yes | y) + printf "✅ You have agreed to the terms of service. Proceeding...\n" + update_env_file + return 0 + ;; + *) + tried=$((tried + 1)) + if [ $tried -gt 10 ]; then + printf "❌ You have not agreed to the terms of service. Please run script again manually to agree and continue.\n" + exit 1 + fi + printf "[i] Incorrect input. Please try again.\n" + ;; + esac + done + } + + if grep -q "AGREE_NO_REWARDS_FOR_ME=true" "$env_file"; then + printf "✅ You have agreed to the terms of service. Proceeding...\n" + return 0 + else + # Check if stdin is a terminal + if [ -t 0 ]; then + prompt_agreement + return $? + fi + + printf "❌ You have not agreed to the terms of service. Please run script again manually to agree and continue.\n" + return 1 + fi + ) } validate_and_store() { @@ -513,14 +566,14 @@ reexec_as_root_if_needed() { # Call the function at the beginning of your script reexec_as_root_if_needed "$@" +# Prompt for hub operator agreement +prompt_for_hub_operator_agreement || exit $? + # Check for the "up" command-line argument if [ "$1" == "up" ]; then # Setup the docker-compose command set_compose_command - # Prompt for hub operator agreement - prompt_for_hub_operator_agreement - # Run docker compose up -d hubble $COMPOSE_CMD up -d hubble statsd grafana @@ -566,9 +619,6 @@ if [ "$1" == "upgrade" ]; then # Call the function to set the COMPOSE_CMD variable set_compose_command - # Prompt for hub operator agreement - prompt_for_hub_operator_agreement - # Update the env file if needed write_env_file