Skip to content

Commit 3123476

Browse files
services: Ingest tss transactions (stellar#46)
* tables and interfaces for TSS * TSS tables and the channel interface * remove empty lines * update * adding semicolons adding semicolons * moving all migrations to one file * make hash primary key instead of xdr * missing , * remove the index on try_transaction_xdr and add column/index on try_transaction_hash ...also remove outgoing/incoming_status and have one status column * Squashed commit of the following: commit fb807aa Author: gouthamp-stellar <[email protected]> Date: Tue Sep 3 10:28:49 2024 -0700 remove the index on try_transaction_xdr and add column/index on try_transaction_hash ...also remove outgoing/incoming_status and have one status column commit 6fc0dc2 Author: gouthamp-stellar <[email protected]> Date: Tue Sep 3 08:58:46 2024 -0700 missing , commit a9cf4e3 Author: gouthamp-stellar <[email protected]> Date: Tue Sep 3 01:55:23 2024 -0700 make hash primary key instead of xdr commit c0f9d32 Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 15:18:27 2024 -0700 moving all migrations to one file commit 2de9898 Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 15:16:53 2024 -0700 adding semicolons adding semicolons commit 373c71a Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 15:12:24 2024 -0700 update commit 3f9f9f0 Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 15:12:00 2024 -0700 remove empty lines commit 9920f48 Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 15:06:40 2024 -0700 TSS tables and the channel interface commit a58d519 Author: gouthamp-stellar <[email protected]> Date: Fri Aug 30 10:55:22 2024 -0700 tables and interfaces for TSS * commit stellar#2 * changing from RpcIngestTxResponse -> RpcGetIngestTxResponse * latest changes * add tests for ValidateOptions * changes based on comments * latest changes based on changes to interface * string -> RPCTXStatus * adding a transaction_builder.go which takes a list of operation xdrs and builds a transaction out of it * moving transaction_service to the utils dir * upper case Channel methods * latest changes * p.txService.NetworkPassPhrase() * last commit before writing unit tests * Making the transaction service more injectible and adding fields to the Payload * typo * typo * Update 2024-08-28.0-tss_transactions.sql * lint errors * go mod tidy * test cases + other changes * remoce unused mocks * error handler service returns errorHandlerService * changes based on comments * lint deadcode error - suppress for now * removed deadcode * changes after comments on transaction service pr * TSS Error Handler Service * removing print statements * Update transaction_service.go * responding to comments * remove commented code * tx service changes * remove comment * remove comment * latest tx service changes * adding a router + utils file * removing println * changed function name * commit stellar#1 * Code() helper function on RPCTXCode * Code() * Delete .env.swp * adding a helpers file * removing BuildTestFeeBumpTransaction * adding to serve.go etc * casing * better test for Send * resolving merge conflicts * removing mockSleep * delete channels/mocks.go * sleep -> time.Sleep * incorporating Daniel's changes + comments * delete files * remove unused code * name change * remove commented code * moving the mocks file inside servicesmocks dir * name changes * refactor * changes to serve.go * checking error on route.Route * check error * merging main * fixing parsesendresp tests * commit changes * merging in changes from eror_handler_service * delete file * commit current changes before merging branch * rpc ingestor * delete file * fix lint errors * remove dead code * fix broken test + delete unused code * tidy mod * remove commented code * delete file * changes based on comments * removing test case that is not relevant anymore * changes based on prev pr comments * remove fmt.Println * merge latest error_handler_service branch + small changes * %s -> %w * U -> u * variable for channel name * account for NewStore returning an error * fix build * merge main * merged latest * typo in desc string * millisecond wait time * changes based on comments * test case for payment ingestion + RPC setup + removing current core setup
1 parent 6553fa9 commit 3123476

32 files changed

+1300
-825
lines changed

Dockerfile

+5-17
Original file line numberDiff line numberDiff line change
@@ -8,28 +8,16 @@ RUN go mod download
88
COPY . ./
99
RUN go build -o /bin/wallet-backend -ldflags "-X main.GitCommit=$GIT_COMMIT" .
1010

11+
# Use the official stellar/soroban-rpc image as the base
12+
FROM stellar/soroban-rpc
13+
14+
# Install bash or sh
15+
RUN apt-get update && apt-get install -y bash
1116

1217
# Step 2: Install Stellar Core and copy over app binary
1318
FROM ubuntu:jammy AS core-build
1419

15-
ARG STELLAR_CORE_VERSION
16-
17-
RUN apt-get update && \
18-
apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils gpg && \
19-
curl -sSL https://apt.stellar.org/SDF.asc | gpg --dearmor >/etc/apt/trusted.gpg.d/SDF.gpg && \
20-
echo "deb https://apt.stellar.org jammy stable" >/etc/apt/sources.list.d/SDF.list && \
21-
echo "deb https://apt.stellar.org jammy testing" >/etc/apt/sources.list.d/SDF-testing.list && \
22-
echo "deb https://apt.stellar.org jammy unstable" >/etc/apt/sources.list.d/SDF-unstable.list && \
23-
apt-get update && \
24-
apt-get install -y stellar-core=${STELLAR_CORE_VERSION} && \
25-
apt-get clean
26-
2720
COPY --from=api-build /bin/wallet-backend /app/
28-
COPY --from=api-build /src/wallet-backend/internal/ingest/config/stellar-core_pubnet.cfg /app/config/stellar-core_pubnet.cfg
29-
COPY --from=api-build /src/wallet-backend/internal/ingest/config/stellar-core_testnet.cfg /app/config/stellar-core_testnet.cfg
30-
31-
ENV CAPTIVE_CORE_BIN_PATH /usr/bin/stellar-core
32-
ENV CAPTIVE_CORE_CONFIG_DIR /app/config/
3321

3422
EXPOSE 8001
3523
WORKDIR /app

cmd/ingest.go

+19-27
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package cmd
33
import (
44
"fmt"
55
"go/types"
6+
"net/http"
7+
"time"
68

79
_ "github.com/lib/pq"
810
"github.com/spf13/cobra"
@@ -11,6 +13,7 @@ import (
1113
"github.com/stellar/wallet-backend/cmd/utils"
1214
"github.com/stellar/wallet-backend/internal/apptracker/sentry"
1315
"github.com/stellar/wallet-backend/internal/ingest"
16+
tsschannels "github.com/stellar/wallet-backend/internal/tss/channels"
1417
)
1518

1619
type ingestCmd struct{}
@@ -22,28 +25,15 @@ func (c *ingestCmd) Command() *cobra.Command {
2225
cfgOpts := config.ConfigOptions{
2326
utils.DatabaseURLOption(&cfg.DatabaseURL),
2427
utils.LogLevelOption(&cfg.LogLevel),
25-
utils.NetworkPassphraseOption(&cfg.NetworkPassphrase),
2628
utils.SentryDSNOption(&sentryDSN),
2729
utils.StellarEnvironmentOption(&stellarEnvironment),
2830
utils.RPCURLOption(&cfg.RPCURL),
29-
{
30-
Name: "captive-core-bin-path",
31-
Usage: "Path to Captive Core's binary file.",
32-
OptType: types.String,
33-
CustomSetValue: utils.SetConfigOptionCaptiveCoreBinPath,
34-
ConfigKey: &cfg.CaptiveCoreBinPath,
35-
FlagDefault: "/usr/local/bin/stellar-core",
36-
Required: true,
37-
},
38-
{
39-
Name: "captive-core-config-dir",
40-
Usage: "Path to Captive Core's configuration files directory.",
41-
OptType: types.String,
42-
CustomSetValue: utils.SetConfigOptionCaptiveCoreConfigDir,
43-
ConfigKey: &cfg.CaptiveCoreConfigDir,
44-
FlagDefault: "./internal/ingest/config",
45-
Required: true,
46-
},
31+
utils.StartLedgerOption(&cfg.StartLedger),
32+
utils.EndLedgerOption(&cfg.EndLedger),
33+
utils.WebhookHandlerServiceChannelMaxBufferSizeOption(&cfg.WebhookChannelMaxBufferSize),
34+
utils.WebhookHandlerServiceChannelMaxWorkersOptions(&cfg.WebhookChannelMaxWorkers),
35+
utils.WebhookHandlerServiceChannelMaxRetriesOption(&cfg.WebhookChannelMaxRetries),
36+
utils.WebhookHandlerServiceChannelMinWaitBtwnRetriesMSOption(&cfg.WebhookChannelWaitBtwnTriesMS),
4737
{
4838
Name: "ledger-cursor-name",
4939
Usage: "Name of last synced ledger cursor, used to keep track of the last ledger ingested by the service. When starting up, ingestion will resume from the ledger number stored in this record. It should be an unique name per container as different containers would overwrite the cursor value of its peers when using the same cursor name.",
@@ -60,14 +50,6 @@ func (c *ingestCmd) Command() *cobra.Command {
6050
FlagDefault: 0,
6151
Required: false,
6252
},
63-
{
64-
Name: "end",
65-
Usage: "Ledger number up to which ingestion should run. When not present, ingestion run indefinitely (live ingestion requires it to be empty).",
66-
OptType: types.Int,
67-
ConfigKey: &cfg.EndLedger,
68-
FlagDefault: 0,
69-
Required: false,
70-
},
7153
}
7254

7355
cmd := &cobra.Command{
@@ -85,11 +67,21 @@ func (c *ingestCmd) Command() *cobra.Command {
8567
return fmt.Errorf("initializing app tracker: %w", err)
8668
}
8769
cfg.AppTracker = appTracker
70+
cfg.WebhookChannel = tsschannels.NewWebhookChannel(tsschannels.WebhookChannelConfigs{
71+
HTTPClient: &http.Client{Timeout: 30 * time.Second},
72+
MaxBufferSize: cfg.WebhookChannelMaxBufferSize,
73+
MaxWorkers: cfg.WebhookChannelMaxWorkers,
74+
MaxRetries: cfg.WebhookChannelMaxRetries,
75+
MinWaitBtwnRetriesMS: cfg.WebhookChannelWaitBtwnTriesMS,
76+
})
8877
return nil
8978
},
9079
RunE: func(_ *cobra.Command, _ []string) error {
9180
return c.Run(cfg)
9281
},
82+
PersistentPostRun: func(_ *cobra.Command, _ []string) {
83+
cfg.WebhookChannel.Stop()
84+
},
9385
}
9486

9587
if err := cfgOpts.Init(cmd); err != nil {

cmd/utils/custom_set_value.go

-44
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ package utils
22

33
import (
44
"encoding/json"
5-
"errors"
65
"fmt"
7-
"os"
86
"strings"
97

108
"github.com/sirupsen/logrus"
@@ -83,48 +81,6 @@ func SetConfigOptionStellarPrivateKey(co *config.ConfigOption) error {
8381
return nil
8482
}
8583

86-
func SetConfigOptionCaptiveCoreBinPath(co *config.ConfigOption) error {
87-
binPath := viper.GetString(co.Name)
88-
89-
fileInfo, err := os.Stat(binPath)
90-
if errors.Is(err, os.ErrNotExist) {
91-
return fmt.Errorf("binary file %s does not exist", binPath)
92-
}
93-
94-
if fileInfo.IsDir() {
95-
return fmt.Errorf("binary file path %s is a directory, not a file", binPath)
96-
}
97-
98-
key, ok := co.ConfigKey.(*string)
99-
if !ok {
100-
return unexpectedTypeError(key, co)
101-
}
102-
*key = binPath
103-
104-
return nil
105-
}
106-
107-
func SetConfigOptionCaptiveCoreConfigDir(co *config.ConfigOption) error {
108-
dirPath := viper.GetString(co.Name)
109-
110-
fileInfo, err := os.Stat(dirPath)
111-
if errors.Is(err, os.ErrNotExist) {
112-
return fmt.Errorf("captive core configuration files dir %s does not exist", dirPath)
113-
}
114-
115-
if !fileInfo.IsDir() {
116-
return fmt.Errorf("captive core configuration files dir %s is not a directory", dirPath)
117-
}
118-
119-
key, ok := co.ConfigKey.(*string)
120-
if !ok {
121-
return unexpectedTypeError(key, co)
122-
}
123-
*key = dirPath
124-
125-
return nil
126-
}
127-
12884
func SetConfigOptionAssets(co *config.ConfigOption) error {
12985
assetsJSON := viper.GetString(co.Name)
13086

cmd/utils/custom_set_value_test.go

-91
Original file line numberDiff line numberDiff line change
@@ -220,97 +220,6 @@ func Test_SetConfigOptionLogLevel(t *testing.T) {
220220
}
221221
}
222222

223-
func TestSetConfigOptionCaptiveCoreBinPath(t *testing.T) {
224-
opts := struct{ binPath string }{}
225-
226-
co := config.ConfigOption{
227-
Name: "captive-core-bin-path",
228-
OptType: types.String,
229-
CustomSetValue: SetConfigOptionCaptiveCoreBinPath,
230-
ConfigKey: &opts.binPath,
231-
}
232-
233-
testCases := []customSetterTestCase[string]{
234-
{
235-
name: "returns an error if the file path is not set, should be caught by the Require() function",
236-
wantErrContains: "binary file does not exist",
237-
},
238-
{
239-
name: "returns an error if the path is invalid",
240-
args: []string{"--captive-core-bin-path", "/a/random/path/bin"},
241-
wantErrContains: "binary file /a/random/path/bin does not exist",
242-
},
243-
{
244-
name: "returns an error if the path format is invalid",
245-
args: []string{"--captive-core-bin-path", "^7JcrS8J4q@V0$c"},
246-
wantErrContains: "binary file ^7JcrS8J4q@V0$c does not exist",
247-
},
248-
{
249-
name: "returns an error if the path is a directory, not a file",
250-
args: []string{"--captive-core-bin-path", "./"},
251-
wantErrContains: "binary file path ./ is a directory, not a file",
252-
},
253-
{
254-
name: "sets to ENV var value",
255-
envValue: "./custom_set_value_test.go",
256-
wantResult: "./custom_set_value_test.go",
257-
},
258-
}
259-
260-
for _, tc := range testCases {
261-
t.Run(tc.name, func(t *testing.T) {
262-
opts.binPath = ""
263-
customSetterTester(t, tc, co)
264-
})
265-
}
266-
}
267-
268-
func TestSetConfigOptionCaptiveCoreConfigDir(t *testing.T) {
269-
opts := struct{ binPath string }{}
270-
271-
co := config.ConfigOption{
272-
Name: "captive-core-config-dir",
273-
OptType: types.String,
274-
CustomSetValue: SetConfigOptionCaptiveCoreConfigDir,
275-
ConfigKey: &opts.binPath,
276-
}
277-
278-
testCases := []customSetterTestCase[string]{
279-
{
280-
name: "returns an error if the file path is not set, should be caught by the Require() function",
281-
wantErrContains: "captive core configuration files dir does not exist",
282-
},
283-
{
284-
name: "returns an error if the path is invalid",
285-
envValue: "/a/random/path",
286-
wantErrContains: "captive core configuration files dir /a/random/path does not exist",
287-
},
288-
{
289-
name: "returns an error if the path format is invalid",
290-
envValue: "^7JcrS8J4q@V0$c",
291-
wantErrContains: "captive core configuration files dir ^7JcrS8J4q@V0$c does not exist",
292-
},
293-
294-
{
295-
name: "returns an error if the path is a file, not a directory",
296-
envValue: "./custom_set_value_test.go",
297-
wantErrContains: "captive core configuration files dir ./custom_set_value_test.go is not a directory",
298-
},
299-
{
300-
name: "sets to ENV var value",
301-
envValue: "../../internal/ingest/config",
302-
wantResult: "../../internal/ingest/config",
303-
},
304-
}
305-
306-
for _, tc := range testCases {
307-
t.Run(tc.name, func(t *testing.T) {
308-
opts.binPath = ""
309-
customSetterTester(t, tc, co)
310-
})
311-
}
312-
}
313-
314223
func TestSetConfigOptionAssets(t *testing.T) {
315224
opts := struct{ assets []entities.Asset }{}
316225

cmd/utils/global_options.go

+25-1
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func RPCURLOption(configKey *string) *config.ConfigOption {
7373
Usage: "The URL of the RPC Server.",
7474
OptType: types.String,
7575
ConfigKey: configKey,
76-
FlagDefault: "localhost:8080",
76+
FlagDefault: "http://soroban-rpc:8000",
7777
Required: true,
7878
}
7979
}
@@ -142,6 +142,30 @@ func DistributionAccountSignatureClientProviderOption(configKey *signing.Signatu
142142
}
143143
}
144144

145+
func StartLedgerOption(configKey *int) *config.ConfigOption {
146+
return &config.ConfigOption{
147+
Name: "start-ledger",
148+
Usage: "ledger number to start getting transactions from",
149+
OptType: types.Int,
150+
ConfigKey: configKey,
151+
FlagDefault: 0,
152+
Required: true,
153+
}
154+
155+
}
156+
157+
func EndLedgerOption(configKey *int) *config.ConfigOption {
158+
return &config.ConfigOption{
159+
Name: "end-ledger",
160+
Usage: "ledger number to end on",
161+
OptType: types.Int,
162+
ConfigKey: configKey,
163+
FlagDefault: 0,
164+
Required: true,
165+
}
166+
167+
}
168+
145169
func AWSOptions(awsRegionConfigKey *string, kmsKeyARN *string, required bool) config.ConfigOptions {
146170
awsOpts := config.ConfigOptions{
147171
{

0 commit comments

Comments
 (0)