initial commit

.gitignore

@@ -0,0 +1,5 @@
+/build/
+*.d
+*.o
+/strfry
+/strfry-db/*.mdb

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "golpe"]
+	path = golpe
+	url = https://github.com/hoytech/golpe.git

Makefile

@@ -0,0 +1,6 @@
+BIN = strfry
+OPT = -O3 -g
+
+include golpe/rules.mk
+
+LDLIBS += -lsecp256k1 -lb2

TODO

@@ -0,0 +1,25 @@
+features
+  finish syncing
+    * logging of bytes up/down
+    * up/both directions
+    * error handling and reporting
+    * way to close sync request
+    * limit on number of concurrent sync requests
+    * full-db scan limited by since/until
+  config for compression
+  less verbose default logging
+  nice new config "units" feature, is 1d instead of 86400
+  make it easier for a thread to setup a quadrable object
+  opt: PubkeyKind scans could be done index-only
+
+rate limits
+  slow-reader detection and back-pressure
+  max connections per ip (nginx?)
+  max bandwidth up/down (nginx?)
+  event writes per second per ip
+  max number of concurrent REQs per connection/ip
+  ? limit on total number of events from a DBScan, not just per filter
+
+misc
+  periodic reaping of disconnected sockets
+  ? websocket-level pings

fbs/nostr-index.fbs

@@ -0,0 +1,17 @@
+namespace NostrIndex;
+
+table Tag {
+    key: uint8;
+    val: [ubyte];
+}
+
+table Event {
+    id: [ubyte];
+    pubkey: [ubyte];
+    created_at: uint64;
+    kind: uint64;
+    tags: [Tag];
+}
+
+table Empty {}
+root_type Empty;

fbs/yesstr.fbs

@@ -0,0 +1,41 @@
+namespace Yesstr;
+
+
+
+// FilterSync
+
+table RequestSync {
+    filter: string; // only required for first in a sequence
+    reqsEncoded: [ubyte];
+}
+
+table ResponseSync {
+    respsEncoded: [ubyte];
+}
+
+
+
+// Request/Response wrappers
+
+union RequestPayload {
+    RequestSync,
+}
+
+union ResponsePayload {
+    ResponseSync,
+}
+
+table Request {
+    requestId: uint64;
+    payload: RequestPayload;
+}
+
+table Response {
+    requestId: uint64;
+    payload: ResponsePayload;
+}
+
+
+
+table Empty {}
+root_type Empty;

golpe.yaml

@@ -0,0 +1,110 @@
+appName: strfry
+
+quadrable: true
+
+flatBuffers: |
+    include "../fbs/nostr-index.fbs";
+
+tables:
+  Event:
+    tableId: 1
+
+    primaryKey: quadId
+
+    fields:
+      - name: quadId
+      - name: receivedAt # microseconds
+      - name: flat
+        type: ubytes
+        nestedFlat: NostrIndex.Event
+
+    indices:
+      created_at:
+        integer: true
+      id:
+        comparator: StringUint64
+      pubkey:
+        comparator: StringUint64
+      kind:
+        comparator: Uint64Uint64
+      pubkeyKind:
+        comparator: StringUint64Uint64
+      tag:
+        comparator: StringUint64
+        multi: true
+      deletion: # eventId, pubkey
+        multi: true
+
+    indexPrelude: |
+        auto *flat = v.flat_nested();
+        created_at = flat->created_at();
+        uint64_t indexTime = *created_at;
+
+        id = makeKey_StringUint64(sv(flat->id()), indexTime);
+        pubkey = makeKey_StringUint64(sv(flat->pubkey()), indexTime);
+        kind = makeKey_Uint64Uint64(flat->kind(), indexTime);
+        pubkeyKind = makeKey_StringUint64Uint64(sv(flat->pubkey()), flat->kind(), indexTime);
+
+        for (const auto &tagPair : *(flat->tags())) {
+            auto tagName = (char)tagPair->key();
+            auto tagVal = sv(tagPair->val());
+            tag.push_back(makeKey_StringUint64(std::string(1, tagName) + std::string(tagVal), indexTime));
+            if (flat->kind() == 5 && tagName == 'e') deletion.push_back(std::string(tagVal) + std::string(sv(flat->pubkey())));
+        }
+
+config:
+  - name: db
+    default: "./strfry-db/"
+    noReload: true
+
+  - name: relay__port
+    default: 7777
+    noReload: true
+  - name: relay__bind
+    default: "127.0.0.1"
+    noReload: true
+
+  - name: relay__info__name
+    default: "strfry default"
+  - name: relay__info__description
+    default: "This is a strfry instance."
+  - name: relay__info__pubkey
+    default: "unset"
+  - name: relay__info__contact
+    default: "unset"
+
+  - name: relay__numThreads__ingester
+    default: 3
+    noReload: true
+  - name: relay__numThreads__reqWorker
+    default: 3
+    noReload: true
+  - name: relay__numThreads__reqMonitor
+    default: 3
+    noReload: true
+  - name: relay__numThreads__yesstr
+    default: 1
+    noReload: true
+
+  - name: relay__maxWebsocketPayloadSize
+    default: 131072
+    noReload: true
+  - name: relay__queryTimesliceBudgetMicroseconds
+    default: 10000
+  - name: relay__maxFilterLimit
+    default: 500
+
+  - name: events__rejectEventsNewerThanSeconds
+    default: 900 # 15 mins
+  - name: events__rejectEventsOlderThanSeconds
+    default: 604800 # 1 week
+  - name: events__rejectEphemeralEventsOlderThanSeconds
+    default: 60
+  - name: events__ephemeralEventsLifetimeSeconds
+    default: 300
+  - name: events__maxEventSize
+    default: 65536
+  - name: events__maxNumTags
+    default: 250
+  - name: events__maxTagValSize
+    default: 128