diff --git a/.vscode/launch.json b/.vscode/launch.json index ab05dd0..06ff138 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,7 +13,7 @@ "program": "${workspaceFolder}/cmd/psweb/", "showLog": false, "envFile": "${workspaceFolder}/.env", - "args": ["-datadir", "/home/vlad/.peerswap2"] + //"args": ["-datadir", "/home/vlad/.peerswap2"] } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 573311f..1c288ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Versions +## 1.5.4 + +- Hide HTTPS option for Umbrel +- AF: apply HTLC Fail Bumps only when Local % <= Low Liq % +- AF: for HTLC Fails above Low Liq % allow increasing Low Liq % threshold +- AF: log full fee changes history, including inbound +- AF: reduce LND load when applying auto fees +- AF: add realized PPM chart for the channel to help decide AF parameters + ## 1.5.3 - Add automatic channel fees management diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 361b549..1540dfd 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -60,8 +60,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at -t.me/vladgoryachev. -All complaints will be reviewed and investigated promptly and fairly. +https://peerswap.dev. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. diff --git a/LICENSE b/LICENSE index 6af2943..5f55d71 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023-2024 Vlad Goryachev +Copyright (c) 2023-2024 Impa10r Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/SECURITY.md b/SECURITY.md index d66cdbf..028a42d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,4 +12,4 @@ Getblock runs a publicly available Bitcoin Core server. It is used as a fallback ## Reporting a Vulnerability -If you discover any vulnerability, please contact me directly at t.me/vladgoryachev +If you discover any vulnerability, please report it discretely to contributors. diff --git a/cmd/psweb/handlers.go b/cmd/psweb/handlers.go index 543b18c..339c23e 100644 --- a/cmd/psweb/handlers.go +++ b/cmd/psweb/handlers.go @@ -6,10 +6,16 @@ import ( "fmt" "io" "log" + "math" "net" "net/http" "os" "path/filepath" + "sort" + "strconv" + "strings" + "time" + "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" "peerswap-web/cmd/psweb/db" @@ -17,10 +23,6 @@ import ( "peerswap-web/cmd/psweb/liquid" "peerswap-web/cmd/psweb/ln" "peerswap-web/cmd/psweb/ps" - "sort" - "strconv" - "strings" - "time" "github.com/elementsproject/peerswap/peerswaprpc" "github.com/gorilla/sessions" @@ -375,7 +377,7 @@ func peerHandler(w http.ResponseWriter, r *http.Request) { rates = "*" + rates } - feeLog := ln.AutoFeeLog[ch.ChannelId] + feeLog := ln.LastAutoFeeLog(ch.ChannelId, false) if feeLog != nil { rates += ", last update " + timePassedAgo(time.Unix(feeLog.TimeStamp, 0)) rates += " from " + formatWithThousandSeparators(uint64(feeLog.OldRate)) @@ -831,19 +833,28 @@ func afHandler(w http.ResponseWriter, r *http.Request) { return } + // get fee rates for all channels + outboundFeeRates := make(map[uint64]int64) + inboundFeeRates := make(map[uint64]int64) + + ln.FeeReport(cl, outboundFeeRates, inboundFeeRates) + for _, peer := range res.GetPeers() { alias := getNodeAlias(peer.NodeId) for _, ch := range peer.Channels { - rates, custom := ln.AutoFeeRatesSummary(ch.ChannelId) + rule, custom := ln.AutoFeeRatesSummary(ch.ChannelId) + af, _ := ln.AutoFeeRule(ch.ChannelId) channelList = append(channelList, &ln.AutoFeeStatus{ - Enabled: ln.AutoFeeEnabled[ch.ChannelId], - LocalBalance: ch.LocalBalance, - RemoteBalance: ch.RemoteBalance, - Alias: alias, - LocalPct: ch.LocalBalance * 100 / (ch.LocalBalance + ch.RemoteBalance), - Rates: rates, - Custom: custom, - ChannelId: ch.ChannelId, + Enabled: ln.AutoFeeEnabled[ch.ChannelId], + Capacity: ch.LocalBalance + ch.RemoteBalance, + Alias: alias, + LocalPct: ch.LocalBalance * 100 / (ch.LocalBalance + ch.RemoteBalance), + Rule: rule, + Custom: custom, + AutoFee: af, + FeeRate: outboundFeeRates[ch.ChannelId], + InboundRate: inboundFeeRates[ch.ChannelId], + ChannelId: ch.ChannelId, }) if ch.ChannelId == channelId { @@ -858,7 +869,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { // sort by LocalPct descending sort.Slice(channelList, func(i, j int) bool { - return channelList[i].LocalPct > channelList[j].LocalPct + return channelList[i].LocalPct < channelList[j].LocalPct }) //check for error errorMessage to display errorMessage := "" @@ -874,6 +885,13 @@ func afHandler(w http.ResponseWriter, r *http.Request) { popupMessage = keys[0] } + chart := ln.PlotPPM(channelId) + // bubble square area reflects amount + for i, p := range *chart { + (*chart)[i].R = uint64(math.Sqrt(float64(p.Amount) / 10_000)) + (*chart)[i].Label = "Routed: " + formatWithThousandSeparators(p.Amount) + ", Fee: " + formatWithThousandSeparators(p.Fee) + ", PPM: " + formatWithThousandSeparators(p.PPM) + } + type Page struct { Authenticated bool ErrorMessage string @@ -890,6 +908,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { Enabled bool // for the displayed channel AnyEnabled bool // for any channel HasInboundFees bool + Chart *[]ln.DataPoint } data := Page{ @@ -908,6 +927,7 @@ func afHandler(w http.ResponseWriter, r *http.Request) { Enabled: ln.AutoFeeEnabled[channelId], AnyEnabled: anyEnabled, HasInboundFees: ln.HasInboundFees(), + Chart: chart, } // executing template named "af" @@ -1155,8 +1175,6 @@ func configHandler(w http.ResponseWriter, r *http.Request) { IsPossibleHTTPS: os.Getenv("NO_HTTPS") == "", } - log.Println(os.Getenv("NO_HTTPS")) - // executing template named "config" err := templates.ExecuteTemplate(w, "config", data) if err != nil { @@ -1424,6 +1442,12 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { return } + rule.FailedMoveThreshold, err = strconv.Atoi(r.FormValue("failedMoveThreshold")) + if err != nil { + redirectWithError(w, r, "/af?", err) + return + } + rule.LowLiqPct, err = strconv.Atoi(r.FormValue("lowLiqPct")) if err != nil { redirectWithError(w, r, "/af?", err) @@ -1502,9 +1526,6 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { } } - // apply the new fees - ln.ApplyAutoFeeAll() - // all done, display confirmation http.Redirect(w, r, "/af?id="+r.FormValue("channelId")+"&msg="+msg, http.StatusSeeOther) return @@ -1566,8 +1587,6 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { if isEnabled { msg += "Enabled" - // apply the new fees - ln.ApplyAutoFeeAll() } else { msg += "Disabled" } @@ -1612,12 +1631,15 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { } } - err = ln.SetFeeRate(r.FormValue("peerNodeId"), channelId, feeRate, inbound, false) + oldRate, err := ln.SetFeeRate(r.FormValue("peerNodeId"), channelId, feeRate, inbound, false) if err != nil { redirectWithError(w, r, nextPage, err) return } + // log change + ln.LogFee(channelId, oldRate, int(feeRate), inbound, true) + // all good, display confirmation msg := strings.Title(r.FormValue("direction")) + " fee rate updated to " + formatSigned(feeRate) http.Redirect(w, r, nextPage+"msg="+msg, http.StatusSeeOther) @@ -1659,7 +1681,7 @@ func submitHandler(w http.ResponseWriter, r *http.Request) { } } - err = ln.SetFeeRate(r.FormValue("peerNodeId"), channelId, feeBase, inbound, true) + _, err = ln.SetFeeRate(r.FormValue("peerNodeId"), channelId, feeBase, inbound, true) if err != nil { redirectWithError(w, r, nextPage, err) return @@ -1939,36 +1961,38 @@ func saveConfigHandler(w http.ResponseWriter, r *http.Request) { return } - secureConnection, err := strconv.ParseBool(r.FormValue("secureConnection")) - if err != nil { - redirectWithError(w, r, "/config?", err) - return - } + if os.Getenv("NO_HTTPS") != "true" { + secureConnection, err := strconv.ParseBool(r.FormValue("secureConnection")) + if err != nil { + redirectWithError(w, r, "/config?", err) + return + } - // display CA certificate installation instructions - if secureConnection && !config.Config.SecureConnection { - config.Config.ServerIPs = r.FormValue("serverIPs") - config.Save() - http.Redirect(w, r, "/ca", http.StatusSeeOther) - return - } + // display CA certificate installation instructions + if secureConnection && !config.Config.SecureConnection { + config.Config.ServerIPs = r.FormValue("serverIPs") + config.Save() + http.Redirect(w, r, "/ca", http.StatusSeeOther) + return + } - if r.FormValue("serverIPs") != config.Config.ServerIPs { - config.Config.ServerIPs = r.FormValue("serverIPs") - if secureConnection { - if err := config.GenerateServerCertificate(); err == nil { - restart(w, r, true, config.Config.Password) - } else { - log.Println("GenereateServerCertificate:", err) - redirectWithError(w, r, "/config?", err) - return + if r.FormValue("serverIPs") != config.Config.ServerIPs { + config.Config.ServerIPs = r.FormValue("serverIPs") + if secureConnection { + if err := config.GenerateServerCertificate(); err == nil { + restart(w, r, true, config.Config.Password) + } else { + log.Println("GenereateServerCertificate:", err) + redirectWithError(w, r, "/config?", err) + return + } } } - } - if !secureConnection && config.Config.SecureConnection { - // restart to listen on HTTP only - restart(w, r, false, "") + if !secureConnection && config.Config.SecureConnection { + // restart to listen on HTTP only + restart(w, r, false, "") + } } allowSwapRequests, err := strconv.ParseBool(r.FormValue("allowSwapRequests")) diff --git a/cmd/psweb/ln/cln.go b/cmd/psweb/ln/cln.go index 2fd125c..410440c 100644 --- a/cmd/psweb/ln/cln.go +++ b/cmd/psweb/ln/cln.go @@ -14,7 +14,6 @@ import ( "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" - "peerswap-web/cmd/psweb/db" "peerswap-web/cmd/psweb/internet" "github.com/btcsuite/btcd/chaincfg" @@ -1037,23 +1036,45 @@ func SetFeeRate(peerNodeId string, channelId uint64, feeRate int64, inbound bool, - isBase bool) error { + isBase bool) (int, error) { if inbound { - return errors.New("inbound rates are not implemented") + return 0, errors.New("inbound rates are not implemented") } client, cleanup, err := GetClient() if err != nil { log.Println("SetFeeRate:", err) - return err + return 0, err } defer cleanup() + clnChId := ConvertLndToClnChannelId(channelId) + + var response map[string]interface{} + err = client.Request(&ListPeerChannelsRequest{}, &response) + if err != nil { + return 0, err + } + + oldRate := 0 + + // Iterate over channels to get old fee + channels := response["channels"].([]interface{}) + for _, channel := range channels { + channelMap := channel.(map[string]interface{}) + if channelMap["short_channel_id"] != nil { + if clnChId == channelMap["short_channel_id"].(string) { + oldRate = int(channelMap["fee_proportional_millionths"].(float64)) + break + } + } + } + var req SetChannelRequest var res map[string]interface{} - req.Id = ConvertLndToClnChannelId(channelId) + req.Id = clnChId if isBase { req.BaseMsat = feeRate } else { @@ -1063,10 +1084,10 @@ func SetFeeRate(peerNodeId string, err = client.Request(&req, &res) if err != nil { log.Println("SetFeeRate:", err) - return err + return oldRate, err } - return nil + return oldRate, nil } // set min or max HTLC size (Msat!!!) for a channel @@ -1105,15 +1126,18 @@ func HasInboundFees() bool { return false } -func ApplyAutoFeeAll() { - if !AutoFeeEnabledAll { +func ApplyAutoFees() { + if !AutoFeeEnabledAll || autoFeeIsRunning { return } + autoFeeIsRunning = true + CacheForwards() client, cleanup, err := GetClient() if err != nil { + autoFeeIsRunning = false return } defer cleanup() @@ -1121,6 +1145,7 @@ func ApplyAutoFeeAll() { var response map[string]interface{} if client.Request(&ListPeerChannelsRequest{}, &response) != nil { + autoFeeIsRunning = false return } @@ -1150,39 +1175,38 @@ func ApplyAutoFeeAll() { oldFee := int(channelMap["fee_proportional_millionths"].(float64)) newFee := oldFee + liqPct := int(channelMap["to_us_msat"].(float64) * 100 / channelMap["total_msat"].(float64)) // check 10 minutes back to be sure - if params.FailedBumpPPM > 0 { - if failedForwardTS[channelId] > time.Now().Add(-time.Duration(10*time.Minute)).Unix() { + if failedForwardTS[channelId] > time.Now().Add(-time.Duration(10*time.Minute)).Unix() { + // forget failed HTLC to prevent duplicate action + failedForwardTS[channelId] = 0 + + if liqPct <= params.LowLiqPct { // bump fee newFee += params.FailedBumpPPM - // forget failed HTLC - failedForwardTS[channelId] = 0 + } else { + // move threshold or do nothing + moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + autoFeeIsRunning = false + return } } + // if no Fail Bump if newFee == oldFee { - liqPct := int(channelMap["to_us_msat"].(float64) * 100 / channelMap["total_msat"].(float64)) newFee = calculateAutoFee(channelId, params, liqPct, oldFee) } // set the new rate if newFee != oldFee { peerId := channelMap["peer_id"].(string) - if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { + if _, err = SetFeeRate(peerId, channelId, int64(newFee), false, false); err == nil { // log the last change - AutoFeeLog[channelId] = &AutoFeeEvent{ - TimeStamp: time.Now().Unix(), - OldRate: oldFee, - NewRate: newFee, - } - // persist to db - db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) + LogFee(channelId, oldFee, newFee, false, false) } } } -} -func ApplyAutoFee() { - // not implemented + autoFeeIsRunning = false } diff --git a/cmd/psweb/ln/common.go b/cmd/psweb/ln/common.go index 24d0950..fedae2c 100644 --- a/cmd/psweb/ln/common.go +++ b/cmd/psweb/ln/common.go @@ -1,6 +1,7 @@ package ln import ( + "reflect" "strconv" "strings" "time" @@ -74,19 +75,23 @@ type ChanneInfo struct { } type AutoFeeStatus struct { - Alias string - ChannelId uint64 - LocalBalance uint64 - LocalPct uint64 - RemoteBalance uint64 - Enabled bool - Rates string - Custom bool + Alias string + ChannelId uint64 + Capacity uint64 + LocalPct uint64 + Enabled bool + Rule string + AutoFee *AutoFeeParams + Custom bool + FeeRate int64 + InboundRate int64 } type AutoFeeParams struct { // fee rate ppm increase after each "Insufficient Balance" HTLC failure FailedBumpPPM int + // Move Low Liq % theshold after each 'Insufficient Balance' HTLC failure above it + FailedMoveThreshold int // low local balance threshold where fee rates stay high LowLiqPct int // ppm rate when liquidity is below LowLiqPct @@ -113,6 +118,18 @@ type AutoFeeEvent struct { TimeStamp int64 OldRate int NewRate int + IsInbound bool + IsManual bool +} + +// for chart plotting +type DataPoint struct { + TS uint64 + Amount uint64 + Fee uint64 + PPM uint64 + R uint64 + Label string } var ( @@ -124,7 +141,7 @@ var ( AutoFeeEnabledAll bool // maps to LND channel Id AutoFee = make(map[uint64]*AutoFeeParams) - AutoFeeLog = make(map[uint64]*AutoFeeEvent) + AutoFeeLog = make(map[uint64][]*AutoFeeEvent) AutoFeeEnabled = make(map[uint64]bool) AutoFeeDefaults = AutoFeeParams{ FailedBumpPPM: 10, @@ -136,12 +153,15 @@ var ( InactivityDays: 7, InactivityDropPPM: 5, InactivityDropPct: 5, - CoolOffHours: 12, + CoolOffHours: 24, LowLiqDiscount: 0, } // track timestamp of the last outbound forward per channel lastForwardTS = make(map[uint64]int64) + + // prevents starting another fee update while the first still running + autoFeeIsRunning = false ) func toSats(amount float64) int64 { @@ -224,7 +244,24 @@ func LoadAutoFees() { db.Load("AutoFees", "AutoFeeEnabled", &AutoFeeEnabled) db.Load("AutoFees", "AutoFee", &AutoFee) db.Load("AutoFees", "AutoFeeDefaults", &AutoFeeDefaults) - db.Load("AutoFees", "AutoFeeLog", &AutoFeeLog) + + // drop non-array legacy log + var log map[uint64]interface{} + db.Load("AutoFees", "AutoFeeLog", &log) + + if len(log) == 0 { + return + } + + // Use reflection to determine the type + for _, data := range log { + v := reflect.ValueOf(data) + if v.Kind() == reflect.Slice { + // Type is map[uint64][]*AutoFeeEvent, load again + db.Load("AutoFees", "AutoFeeLog", &AutoFeeLog) + return + } + } } func calculateAutoFee(channelId uint64, params *AutoFeeParams, liqPct int, oldFee int) int { @@ -232,8 +269,9 @@ func calculateAutoFee(channelId uint64, params *AutoFeeParams, liqPct int, oldFe if liqPct > params.LowLiqPct { // normal or high liquidity regime, check if fee can be dropped lastUpdate := int64(0) - if AutoFeeLog[channelId] != nil { - lastUpdate = AutoFeeLog[channelId].TimeStamp + lastLog := LastAutoFeeLog(channelId, false) + if lastLog != nil { + lastUpdate = lastLog.TimeStamp } if lastUpdate < time.Now().Add(-time.Duration(params.CoolOffHours)*time.Hour).Unix() { // check the last outbound timestamp @@ -267,3 +305,48 @@ func msatToSatUp(msat uint64) uint64 { } return sat } + +// returns last log entry +func LastAutoFeeLog(channelId uint64, isInbound bool) *AutoFeeEvent { + // Loop backwards through the array + for i := len(AutoFeeLog[channelId]) - 1; i >= 0; i-- { + if AutoFeeLog[channelId][i].IsInbound == isInbound { + return AutoFeeLog[channelId][i] + } + } + + return nil +} + +func LogFee(channelId uint64, oldRate int, newRate int, isInbound bool, isManual bool) { + AutoFeeLog[channelId] = append(AutoFeeLog[channelId], &AutoFeeEvent{ + TimeStamp: time.Now().Unix(), + OldRate: oldRate, + NewRate: newRate, + IsInbound: isInbound, + IsManual: isManual, + }) + // persist to db + db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) +} + +func moveLowLiqThreshold(channelId uint64, bump int) { + if bump == 0 { + return + } + + if AutoFee[channelId] == nil { + // add custom parameters + AutoFee[channelId] = new(AutoFeeParams) + // clone default values + *AutoFee[channelId] = AutoFeeDefaults + } + + // do not alow exeeding high liquidity threshold + if AutoFee[channelId].LowLiqPct+bump < AutoFee[channelId].ExcessPct { + AutoFee[channelId].LowLiqPct += bump + // persist to db + db.Save("AutoFees", "AutoFee", AutoFee) + } + +} diff --git a/cmd/psweb/ln/lnd.go b/cmd/psweb/ln/lnd.go index f49ecbb..9b6afa2 100644 --- a/cmd/psweb/ln/lnd.go +++ b/cmd/psweb/ln/lnd.go @@ -19,7 +19,6 @@ import ( "peerswap-web/cmd/psweb/bitcoin" "peerswap-web/cmd/psweb/config" - "peerswap-web/cmd/psweb/db" "github.com/elementsproject/peerswap/peerswaprpc" @@ -74,8 +73,9 @@ var ( lastInvoiceSettleIndex uint64 // last timestamps for downloads - lastforwardCreationTs uint64 + lastForwardCreationTs uint64 lastPaymentCreationTs int64 + lastInvoiceCreationTs int64 // default lock id used by LND internalLockId = []byte{ @@ -694,14 +694,66 @@ func GetMyAlias() string { return myNodeAlias } +func downloadInvoices(client lnrpc.LightningClient) error { + // only go back 6 months for itinial download + start := uint64(time.Now().AddDate(0, -6, 0).Unix()) + offset := uint64(0) + totalInvoices := uint64(0) + + // incremental download + if lastInvoiceCreationTs > 0 { + start = uint64(lastInvoiceCreationTs) + 1 + } + + for { + res, err := client.ListInvoices(context.Background(), &lnrpc.ListInvoiceRequest{ + CreationDateStart: start, + Reversed: false, + IndexOffset: offset, + NumMaxInvoices: 100, // bolt11 fields can be long + }) + if err != nil { + if !strings.HasPrefix(fmt.Sprint(err), "rpc error: code = Unknown desc = waiting to start") { + log.Println("ListInvoices:", err) + } + return err + } + + for _, invoice := range res.Invoices { + appendInvoice(invoice) + } + + n := len(res.Invoices) + totalInvoices += uint64(n) + + if n > 0 { + // settle index for subscription + lastInvoiceSettleIndex = res.Invoices[n-1].SettleIndex + } + if n < 100 { + // all invoices retrieved + break + } + + // next pull start from the next index + offset = res.LastIndexOffset + } + + if totalInvoices > 0 { + log.Printf("Cached %d invoices", totalInvoices) + } + + return nil +} + func downloadForwards(client lnrpc.LightningClient) { // only go back 6 months start := uint64(time.Now().AddDate(0, -6, 0).Unix()) // incremental download if substription was interrupted - if lastforwardCreationTs > 0 { + if lastForwardCreationTs > 0 { // continue from the last timestamp in seconds - start = lastforwardCreationTs + 1 + start = lastForwardCreationTs + 1 } // download forwards @@ -734,7 +786,7 @@ func downloadForwards(client lnrpc.LightningClient) { if n > 0 { // store the last timestamp - lastforwardCreationTs = res.ForwardingEvents[n-1].TimestampNs / 1_000_000_000 + lastForwardCreationTs = res.ForwardingEvents[n-1].TimestampNs / 1_000_000_000 } if n < 50000 { // all events retrieved @@ -920,7 +972,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error } defer cleanup() - ApplyAutoFee(client, htlcEvent.OutgoingChannelId, true) + applyAutoFee(client, htlcEvent.OutgoingChannelId, true) } case *routerrpc.HtlcEvent_SettleEvent: @@ -932,7 +984,7 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error // settled htlcEvent has no Outgoing info, take from queue forwardsOut[htlc.OutgoingChannelId] = append(forwardsOut[htlc.OutgoingChannelId], htlc.forwardingEvent) // store the last timestamp - lastforwardCreationTs = htlc.forwardingEvent.TimestampNs / 1_000_000_000 + lastForwardCreationTs = htlc.forwardingEvent.TimestampNs / 1_000_000_000 // delete from queue removeInflightHTLC(htlcEvent.IncomingChannelId, htlcEvent.IncomingHtlcId) @@ -948,7 +1000,8 @@ func subscribeForwards(ctx context.Context, client routerrpc.RouterClient) error } defer cleanup() - ApplyAutoFee(client, htlc.forwardingEvent.ChanIdOut, false) + // calculate with new balance + applyAutoFee(client, htlc.forwardingEvent.ChanIdOut, false) break } } @@ -1000,72 +1053,36 @@ func SubscribeAll() { client := lnrpc.NewLightningClient(conn) ctx := context.Background() - // only go back 6 months for itinial download - start := uint64(time.Now().AddDate(0, -6, 0).Unix()) - offset := uint64(0) - totalInvoices := uint64(0) - - for { - res, err := client.ListInvoices(ctx, &lnrpc.ListInvoiceRequest{ - CreationDateStart: start, - Reversed: false, - IndexOffset: offset, - NumMaxInvoices: 100, // bolt11 fields can be long - }) - if err != nil { - if !strings.HasPrefix(fmt.Sprint(err), "rpc error: code = Unknown desc = waiting to start") { - log.Println("ListInvoices:", err) - } - return - } - - for _, invoice := range res.Invoices { - appendInvoice(invoice) - } - - n := len(res.Invoices) - totalInvoices += uint64(n) - - if n > 0 { - // settle index for subscription - lastInvoiceSettleIndex = res.Invoices[n-1].SettleIndex - } - if n < 100 { - // all invoices retrieved - break - } - - // next pull start from the next index - offset = res.LastIndexOffset - } - - if totalInvoices > 0 { - log.Printf("Cached %d invoices", totalInvoices) + // initial download + if downloadInvoices(client) != nil { + return } routerClient := routerrpc.NewRouterClient(conn) go func() { - // initial or incremental download forwards + // initial download forwards downloadForwards(client) // subscribe to Forwards for { - if err := subscribeForwards(ctx, routerClient); err != nil { - log.Println("Forwards subscription error:", err) + if subscribeForwards(ctx, routerClient) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadForwards(client) } } }() go func() { - // initial or incremental download payments + // initial download payments downloadPayments(client) // subscribe to Payments for { - if err := subscribePayments(ctx, routerClient); err != nil { - log.Println("Payments subscription error:", err) + if subscribePayments(ctx, routerClient) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadPayments(client) } } }() @@ -1074,9 +1091,10 @@ func SubscribeAll() { // subscribe to Invoices for { - if err := subscribeInvoices(ctx, client); err != nil { - log.Println("Invoices subscription error:", err) + if subscribeInvoices(ctx, client) != nil { time.Sleep(60 * time.Second) + // incremental download after error + downloadInvoices(client) } } } @@ -1086,6 +1104,10 @@ func appendInvoice(invoice *lnrpc.Invoice) { // precaution return } + + // save for incremental downloads + lastInvoiceCreationTs = invoice.CreationDate + // only append settled htlcs if invoice.State == lnrpc.Invoice_SETTLED { if parts := strings.Split(invoice.Memo, " "); len(parts) > 4 { @@ -1485,17 +1507,17 @@ func FeeReport(client lnrpc.LightningClient, outboundFeeRates map[uint64]int64, return nil } -// set fee rate for a channel +// set fee rate for a channel, return old rate func SetFeeRate(peerNodeId string, channelId uint64, feeRate int64, inbound bool, - isBase bool) error { + isBase bool) (int, error) { client, cleanup, err := GetClient() if err != nil { log.Println("SetFeeRate:", err) - return err + return 0, err } defer cleanup() @@ -1505,7 +1527,7 @@ func SetFeeRate(peerNodeId string, }) if err != nil { log.Println("SetFeeRate:", err) - return err + return 0, err } policy := r.Node1Policy @@ -1518,7 +1540,7 @@ func SetFeeRate(peerNodeId string, outputIndex, err := strconv.ParseInt(parts[1], 10, 64) if err != nil { log.Println("SetFeeRate:", err) - return err + return 0, err } var req lnrpc.PolicyUpdateRequest @@ -1543,17 +1565,23 @@ func SetFeeRate(peerNodeId string, BaseFeeMsat: policy.InboundFeeBaseMsat, } + oldRate := 0 + // change what's new if isBase { if inbound { + oldRate = int(policy.InboundFeeBaseMsat) req.InboundFee.BaseFeeMsat = int32(feeRate) } else { + oldRate = int(policy.FeeBaseMsat) req.BaseFeeMsat = feeRate } } else { if inbound { + oldRate = int(policy.InboundFeeRateMilliMsat) req.InboundFee.FeeRatePpm = int32(feeRate) } else { + oldRate = int(policy.FeeRateMilliMsat) req.FeeRatePpm = uint32(feeRate) } } @@ -1561,10 +1589,10 @@ func SetFeeRate(peerNodeId string, _, err = client.UpdateChannelPolicy(context.Background(), &req) if err != nil { log.Println("SetFeeRate:", err) - return err + return oldRate, err } - return nil + return oldRate, nil } // set min or max HTLC size (Msat!!!) for a channel @@ -1648,12 +1676,15 @@ func SetHtlcSize(peerNodeId string, return nil } -func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC bool) { +// for failed HTLC only +func applyAutoFee(client lnrpc.LightningClient, channelId uint64, htlcFail bool) { - if !AutoFeeEnabledAll || !AutoFeeEnabled[channelId] { + if !AutoFeeEnabledAll || !AutoFeeEnabled[channelId] || autoFeeIsRunning { return } + autoFeeIsRunning = true + params := &AutoFeeDefaults if AutoFee[channelId] != nil { // channel has custom parameters @@ -1665,6 +1696,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo // get my node id res, err := client.GetInfo(ctx, &lnrpc.GetInfoRequest{}) if err != nil { + autoFeeIsRunning = false return } myNodeId = res.GetIdentityPubkey() @@ -1673,6 +1705,7 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo ChanId: channelId, }) if err != nil { + autoFeeIsRunning = false return } @@ -1687,82 +1720,179 @@ func ApplyAutoFee(client lnrpc.LightningClient, channelId uint64, failedHTLC boo oldFee := int(policy.FeeRateMilliMsat) newFee := oldFee - if failedHTLC { - // increase fee to help prevent further failed HTLCs - newFee += params.FailedBumpPPM - } else { - // get balances - bytePeer, err := hex.DecodeString(peerId) - if err != nil { - return - } + // get balances + bytePeer, err := hex.DecodeString(peerId) + if err != nil { + autoFeeIsRunning = false + return + } - res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ - PublicOnly: true, - Peer: bytePeer, - }) - if err != nil { - return - } + res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ + Peer: bytePeer, + }) + if err != nil { + autoFeeIsRunning = false + return + } - localBalance := int64(0) - for _, ch := range res.Channels { - if ch.ChanId == channelId { - if ch.UnsettledBalance != 0 { - // skip af while htlcs are pending - return - } - localBalance = ch.LocalBalance - break + localBalance := int64(0) + for _, ch := range res.Channels { + if ch.ChanId == channelId { + if ch.UnsettledBalance != 0 { + // skip af while htlcs are pending + autoFeeIsRunning = false + return } + localBalance = ch.LocalBalance + break } + } - liqPct := int(localBalance * 100 / r.Capacity) - - if HasInboundFees() && liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { - // set discount - SetFeeRate(peerId, channelId, int64(params.LowLiqDiscount), true, false) + liqPct := int(localBalance * 100 / r.Capacity) + if htlcFail { + if liqPct <= params.LowLiqPct { + // increase fee to help prevent further failed HTLCs + newFee += params.FailedBumpPPM + } else { + // move threshold or do nothing + moveLowLiqThreshold(channelId, params.FailedMoveThreshold) + autoFeeIsRunning = false + return } - + } else { newFee = calculateAutoFee(channelId, params, liqPct, oldFee) } // set the new rate if newFee != oldFee { - if SetFeeRate(peerId, channelId, int64(newFee), false, false) == nil { + if old, err := SetFeeRate(peerId, channelId, int64(newFee), false, false); err == nil { // log the last change - AutoFeeLog[channelId] = &AutoFeeEvent{ - TimeStamp: time.Now().Unix(), - OldRate: oldFee, - NewRate: newFee, - } - // persist to db - db.Save("AutoFees", "AutoFeeLog", AutoFeeLog) + LogFee(channelId, old, newFee, false, false) } } + + autoFeeIsRunning = false } -func ApplyAutoFeeAll() { +func ApplyAutoFees() { - if !AutoFeeEnabledAll { + if !AutoFeeEnabledAll || autoFeeIsRunning { return } + autoFeeIsRunning = true + client, cleanup, err := GetClient() if err != nil { + autoFeeIsRunning = false return } defer cleanup() ctx := context.Background() + if myNodeId == "" { + // get my node id + res, err := client.GetInfo(ctx, &lnrpc.GetInfoRequest{}) + if err != nil { + autoFeeIsRunning = false + return + } + myNodeId = res.GetIdentityPubkey() + } + res, err := client.ListChannels(ctx, &lnrpc.ListChannelsRequest{ ActiveOnly: true, }) if err != nil { + autoFeeIsRunning = false return } for _, ch := range res.Channels { - ApplyAutoFee(client, ch.ChanId, false) + if ch.UnsettledBalance != 0 || !AutoFeeEnabled[ch.ChanId] { + // skip af while htlcs are pending + continue + } + + params := &AutoFeeDefaults + if AutoFee[ch.ChanId] != nil { + // channel has custom parameters + params = AutoFee[ch.ChanId] + } + + r, err := client.GetChanInfo(ctx, &lnrpc.ChanInfoRequest{ + ChanId: ch.ChanId, + }) + if err != nil { + autoFeeIsRunning = false + return + } + + policy := r.Node1Policy + peerId := r.Node2Pub + if r.Node1Pub != myNodeId { + // the first policy is not ours, use the second + policy = r.Node2Policy + peerId = r.Node1Pub + } + + oldFee := int(policy.FeeRateMilliMsat) + newFee := oldFee + liqPct := int(ch.LocalBalance * 100 / r.Capacity) + + newFee = calculateAutoFee(ch.ChanId, params, liqPct, oldFee) + + // set the new rate + if newFee != oldFee { + _, err := SetFeeRate(peerId, ch.ChanId, int64(newFee), false, false) + if err == nil { + // log the last change + LogFee(ch.ChanId, oldFee, newFee, false, false) + } + } + + if HasInboundFees() { + toSet := false + discountRate := int64(0) + + if liqPct < params.LowLiqPct && policy.InboundFeeRateMilliMsat != int32(params.LowLiqDiscount) { + // set inbound fee discount + discountRate = int64(params.LowLiqDiscount) + toSet = true + } else if liqPct >= params.LowLiqPct && policy.InboundFeeRateMilliMsat < 0 { + // remove discount unless it was manually set + if !LastAutoFeeLog(ch.ChanId, true).IsManual { + toSet = true + } + } + + if toSet { + oldRate, err := SetFeeRate(peerId, ch.ChanId, discountRate, true, false) + if err == nil { + // log the last change + LogFee(ch.ChanId, oldRate, int(discountRate), true, false) + } + } + } + } + + autoFeeIsRunning = false +} + +func PlotPPM(channelId uint64) *[]DataPoint { + var plot []DataPoint + + for _, e := range forwardsOut[channelId] { + // ignore small forwards + if e.AmtOut > 1000 { + plot = append(plot, DataPoint{ + TS: e.TimestampNs / 1_000_000_000, + Amount: e.AmtOut, + Fee: e.Fee, + PPM: e.FeeMsat * 1_000_000 / e.AmtOutMsat, + }) + } } + + return &plot } diff --git a/cmd/psweb/main.go b/cmd/psweb/main.go index 16bb01b..b33d894 100644 --- a/cmd/psweb/main.go +++ b/cmd/psweb/main.go @@ -34,7 +34,7 @@ import ( const ( // App version tag - version = "v1.5.3" + version = "v1.5.4" ) type SwapParams struct { @@ -133,6 +133,7 @@ func main() { "fmt": formatWithThousandSeparators, "fs": formatSigned, "m": toMil, + "last": last, }). ParseFS(tplFolder, templateNames...)) @@ -348,7 +349,7 @@ func onTimer() { go ln.SubscribeAll() // execute auto fee - ln.ApplyAutoFeeAll() + go ln.ApplyAutoFees() } func liquidBackup(force bool) { @@ -1459,14 +1460,27 @@ func feeInputField(peerNodeId string, channelId uint64, direction string, feePer rates = "*" + rates } - feeLog := ln.AutoFeeLog[channelId] + change := " " + feeLog := ln.LastAutoFeeLog(channelId, direction == "inbound") if feeLog != nil { rates += "\nLast update " + timePassedAgo(time.Unix(feeLog.TimeStamp, 0)) - rates += "\nFrom " + formatWithThousandSeparators(uint64(feeLog.OldRate)) - rates += " to " + formatWithThousandSeparators(uint64(feeLog.NewRate)) + rates += "\nFrom " + formatSigned(int64(feeLog.OldRate)) + rates += " to " + formatSigned(int64(feeLog.NewRate)) + if feeLog.TimeStamp > time.Now().Add(-24*time.Hour).Unix() { + if feeLog.NewRate > feeLog.OldRate { + if config.Config.ColorScheme == "dark" { + change = `⬆` + } else { + change = `⬆` + } + + } else { + change = `⬇` + } + } } - t += "" + formatSigned(feePerMil) + "" + t += "" + formatSigned(feePerMil) + "" + change } else { t += `
` t += `` @@ -1483,3 +1497,8 @@ func feeInputField(peerNodeId string, channelId uint64, direction string, feePer return t } + +// Template function to check if the element is the last one in the slice +func last(x int, a interface{}) bool { + return x == len(*(a.(*[]ln.DataPoint)))-1 +} diff --git a/cmd/psweb/templates/af.gohtml b/cmd/psweb/templates/af.gohtml index 619ce89..b88e8d0 100644 --- a/cmd/psweb/templates/af.gohtml +++ b/cmd/psweb/templates/af.gohtml @@ -127,7 +127,19 @@
- + +
+ + +
+ +
+ + + + +
+
@@ -135,6 +147,16 @@ + +
+ +
+ + +
+ +
+ @@ -180,20 +202,6 @@ - {{if .HasInboundFees}} - - -
- -
- - -
- -
- - - {{end}}
@@ -203,8 +211,16 @@ {{end}}
+
+ + {{if ne .ChannelId 0}} +
+

Realized Routing PPM

+
+ +

- + {{end}}
@@ -232,11 +248,12 @@ Alias - Local - % - Remote + Cap + % + Rate + In Rules +* indicates custom rule" style="text-align: center;">AF Rules ☑ @@ -244,10 +261,17 @@ {{range .ChannelList}} {{.Alias}} - {{m .LocalBalance}} - {{fmt .LocalPct}} - {{m .RemoteBalance}} - {{if .Custom}}*{{end}}{{.Rates}} + {{m .Capacity}} + {{fmt .LocalPct}} + {{fs .FeeRate}} + {{fs .InboundRate}} + {{if .Enabled}}{{if .Custom}}*{{end}}{{.Rule}}{{end}}
@@ -280,6 +304,80 @@ } } } + + // Get the context of the canvas element we want to select + var ctx = document.getElementById('myScatterChart').getContext('2d'); + + // Create the chart + var myScatterChart = new Chart(ctx, { + type: 'bubble', // The type of chart we want to create + data: { + datasets: [{ + data: [ + {{range $index, $element := .Chart}} + { x: new Date({{$element.TS}} * 1000), y: {{$element.PPM}}, r: {{$element.R}}, label: `{{$element.Label}}` }{{if not (last $index $.Chart)}},{{end}} + {{end}} + ], + backgroundColor: 'rgba(54, 162, 235, 0.8)', + borderColor: 'rgba(75, 192, 192, 1)', + borderWidth: 1 + }] + }, + options: { + scales: { + x: { + type: 'time', + time: { + unit: 'month' + }, + position: 'bottom', + {{if eq .ColorScheme "dark"}} + grid: { + color: 'rgba(255, 255, 255, 0.1)', // Set grid line color (white with reduced opacity) + borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) + borderWidth: 1 // Grid border width + }, + ticks: { + color: 'white' // Set x-axis font color to white + }, + {{end}} + title: { + display: false, + } + }, + y: { + beginAtZero: false, + {{if eq .ColorScheme "dark"}} + grid: { + color: 'rgba(255, 255, 255, 0.1)', // Set grid line color (white with reduced opacity) + borderColor: 'rgba(255, 255, 255, 0.2)', // Set grid border color (white with slightly higher opacity) + borderWidth: 1 // Grid border width + }, + ticks: { + color: 'white' // Set x-axis font color to white + }, + {{end}} + title: { + display: false, + } + } + }, + plugins: { + tooltip: { + callbacks: { + label: function(context) { + let label = context.raw.label || ''; + return label; + }, + }, + }, + legend: { + display: false + }, + } + } + }); +
{{template "footer" .}} diff --git a/cmd/psweb/templates/config.gohtml b/cmd/psweb/templates/config.gohtml index ba30fa9..540f2a3 100644 --- a/cmd/psweb/templates/config.gohtml +++ b/cmd/psweb/templates/config.gohtml @@ -117,22 +117,39 @@
-
-
-
- -
-
-
- +
+ {{if .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
-
+ {{end}}
+ {{if not .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
+
+
+ {{end}}
-
-
- -
-
-
- + {{if .IsPossibleHTTPS}} +
+
+ +
+
+
+ +
-
-
-
- -
-
- +
+
+ +
+
+ +
-
+ {{end}}
diff --git a/cmd/psweb/templates/reusable.gohtml b/cmd/psweb/templates/reusable.gohtml index 940ec06..dad8a96 100644 --- a/cmd/psweb/templates/reusable.gohtml +++ b/cmd/psweb/templates/reusable.gohtml @@ -94,6 +94,10 @@ } } + + + +
{{if eq .ErrorMessage "welcome"}} @@ -122,7 +126,7 @@

- PeerSwap Web UI by Vlad Goryachev + PeerSwap Web UI by Impa10r