Add support for option_payment_metadata

Add support for lightning/bolts#912

Whenever we find a payment metadata field in an invoice, we send it in
the onion payload for the final recipient.

We include a payment metadata in every invoice we generate. This lets us
see whether our payers support it or not, which is important data to have
before we make it mandatory and use it for storage-less invoices.

Author: t-bast

src/commonMain/kotlin/fr/acinq/lightning/payment/IncomingPaymentHandler.kt

@@ -4,6 +4,7 @@ import fr.acinq.bitcoin.ByteVector32
 import fr.acinq.bitcoin.Crypto
 import fr.acinq.bitcoin.PrivateKey
 import fr.acinq.lightning.CltvExpiry
+import fr.acinq.lightning.Lightning.randomBytes
 import fr.acinq.lightning.Lightning.randomBytes32
 import fr.acinq.lightning.MilliSatoshi
 import fr.acinq.lightning.NodeParams
@@ -88,6 +89,8 @@ class IncomingPaymentHandler(val nodeParams: NodeParams, val walletParams: Walle
             PaymentRequest.DEFAULT_MIN_FINAL_EXPIRY_DELTA,
             nodeParams.features,
             randomBytes32(),
+            // We always include a payment metadata in our invoices, which lets us test whether senders support it
+            randomBytes(64).toByteVector(),
             expirySeconds,
             extraHops,
             timestampSeconds
@@ -248,7 +251,10 @@ class IncomingPaymentHandler(val nodeParams: NodeParams, val walletParams: Walle
                         }
                         else -> {
                             // We have received all the payment parts.
-                            logger.info { "h:${paymentPart.paymentHash} payment received (${payment.amountReceived})" }
+                            when (val paymentMetadata = paymentPart.finalPayload.paymentMetadata) {
+                                null -> logger.info { "h:${paymentPart.paymentHash} payment received (${payment.amountReceived}) without payment metadata" }
+                                else -> logger.info { "h:${paymentPart.paymentHash} payment received (${payment.amountReceived}) with payment metadata ($paymentMetadata)" }
+                            }
                             val (actions, receivedWith) = payment.parts.map { part ->
                                 when (part) {
                                     is HtlcPart -> {

src/commonMain/kotlin/fr/acinq/lightning/payment/IncomingPaymentPacket.kt

@@ -74,7 +74,7 @@ object IncomingPaymentPacket {
             else -> {
                 // We merge contents from the outer and inner payloads.
                 // We must use the inner payload's total amount and payment secret because the payment may be split between multiple trampoline payments (#reckless).
-                Either.Right(PaymentOnion.FinalPayload.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret))
+                Either.Right(PaymentOnion.FinalPayload.createMultiPartPayload(outerPayload.amount, innerPayload.totalAmount, outerPayload.expiry, innerPayload.paymentSecret, innerPayload.paymentMetadata))
             }
         }
     }

src/commonMain/kotlin/fr/acinq/lightning/payment/OutgoingPaymentHandler.kt

@@ -312,7 +312,7 @@ class OutgoingPaymentHandler(val nodeId: PublicKey, val walletParams: WalletPara
 
         val finalExpiryDelta = request.details.paymentRequest.minFinalExpiryDelta ?: Channel.MIN_CLTV_EXPIRY_DELTA
         val finalExpiry = finalExpiryDelta.toCltvExpiry(currentBlockHeight.toLong())
-        val finalPayload = PaymentOnion.FinalPayload.createSinglePartPayload(request.amount, finalExpiry, request.details.paymentRequest.paymentSecret)
+        val finalPayload = PaymentOnion.FinalPayload.createSinglePartPayload(request.amount, finalExpiry, request.details.paymentRequest.paymentSecret, request.details.paymentRequest.paymentMetadata)
 
         val invoiceFeatures = Features(request.details.paymentRequest.features)
         val (trampolineAmount, trampolineExpiry, trampolineOnion) = if (invoiceFeatures.hasFeature(Feature.TrampolinePayment)) {

src/commonMain/kotlin/fr/acinq/lightning/payment/OutgoingPaymentPacket.kt

@@ -76,7 +76,10 @@ object OutgoingPaymentPacket {
      *  - the trampoline onion to include in final payload of a normal onion
      */
     fun buildTrampolineToLegacyPacket(invoice: PaymentRequest, hops: List<NodeHop>, finalPayload: PaymentOnion.FinalPayload): Triple<MilliSatoshi, CltvExpiry, PacketAndSecrets> {
-        val (firstAmount, firstExpiry, payloads) = hops.drop(1).reversed().fold(Triple(finalPayload.amount, finalPayload.expiry, listOf<PaymentOnion.PerHopPayload>(finalPayload))) { triple, hop ->
+        // NB: the final payload will never reach the recipient, since the next-to-last trampoline hop will convert that to a legacy payment
+        // We use the smallest final payload possible, otherwise we may overflow the trampoline onion size.
+        val dummyFinalPayload = PaymentOnion.FinalPayload.createSinglePartPayload(finalPayload.amount, finalPayload.expiry, finalPayload.paymentSecret, null)
+        val (firstAmount, firstExpiry, payloads) = hops.drop(1).reversed().fold(Triple(finalPayload.amount, finalPayload.expiry, listOf<PaymentOnion.PerHopPayload>(dummyFinalPayload))) { triple, hop ->
             val (amount, expiry, payloads) = triple
             val payload = when (payloads.size) {
                 // The next-to-last trampoline hop must include invoice data to indicate the conversion to a legacy payment.

src/commonMain/kotlin/fr/acinq/lightning/payment/PaymentRequest.kt

@@ -28,6 +28,9 @@ data class PaymentRequest(
     @Transient
     val paymentSecret: ByteVector32 = tags.find { it is TaggedField.PaymentSecret }!!.run { (this as TaggedField.PaymentSecret).secret }
 
+    @Transient
+    val paymentMetadata: ByteVector? = tags.find { it is TaggedField.PaymentMetadata }?.run { (this as TaggedField.PaymentMetadata).data }
+
     @Transient
     val description: String? = tags.find { it is TaggedField.Description }?.run { (this as TaggedField.Description).description }
 
@@ -147,6 +150,7 @@ data class PaymentRequest(
             minFinalCltvExpiryDelta: CltvExpiryDelta,
             features: Features,
             paymentSecret: ByteVector32 = randomBytes32(),
+            paymentMetadata: ByteVector? = null,
             expirySeconds: Long? = null,
             extraHops: List<List<TaggedField.ExtraHop>> = listOf(),
             timestampSeconds: Long = currentTimestampSeconds()
@@ -159,9 +163,8 @@ data class PaymentRequest(
                 TaggedField.PaymentSecret(paymentSecret),
                 TaggedField.Features(features.invoiceFeatures().toByteArray().toByteVector())
             )
-            if (expirySeconds != null) {
-                tags.add(TaggedField.Expiry(expirySeconds))
-            }
+            paymentMetadata?.let { tags.add(TaggedField.PaymentMetadata(it)) }
+            expirySeconds?.let { tags.add(TaggedField.Expiry(it)) }
             if (extraHops.isNotEmpty()) {
                 extraHops.forEach { tags.add(TaggedField.RoutingInfo(it)) }
             }
@@ -210,6 +213,7 @@ data class PaymentRequest(
                     when (tag) {
                         TaggedField.PaymentHash.tag -> tags.add(kotlin.runCatching { TaggedField.PaymentHash.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
                         TaggedField.PaymentSecret.tag -> tags.add(kotlin.runCatching { TaggedField.PaymentSecret.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
+                        TaggedField.PaymentMetadata.tag -> tags.add(kotlin.runCatching { TaggedField.PaymentMetadata.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
                         TaggedField.Description.tag -> tags.add(kotlin.runCatching { TaggedField.Description.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
                         TaggedField.DescriptionHash.tag -> tags.add(kotlin.runCatching { TaggedField.DescriptionHash.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
                         TaggedField.Expiry.tag -> tags.add(kotlin.runCatching { TaggedField.Expiry.decode(value) }.getOrDefault(TaggedField.InvalidTag(tag, value)))
@@ -339,6 +343,17 @@ data class PaymentRequest(
             }
         }
 
+        @Serializable
+        data class PaymentMetadata(@Contextual val data: ByteVector) : TaggedField() {
+            override val tag: Int5 = PaymentMetadata.tag
+            override fun encode(): List<Int5> = Bech32.eight2five(data.toByteArray()).toList()
+
+            companion object {
+                const val tag: Int5 = 27
+                fun decode(input: List<Int5>): PaymentMetadata = PaymentMetadata(Bech32.five2eight(input.toTypedArray(), 0).toByteVector())
+            }
+        }
+
         /** @param expirySeconds payment expiry (in seconds) */
         @Serializable
         data class Expiry(val expirySeconds: Long) : TaggedField() {

src/commonMain/kotlin/fr/acinq/lightning/serialization/v1/Serialization.kt

@@ -49,6 +49,7 @@ object Serialization {
             subclass(OnionPaymentPayloadTlv.OutgoingCltv.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingChannelId.serializer())
             subclass(OnionPaymentPayloadTlv.PaymentData.serializer())
+            subclass(OnionPaymentPayloadTlv.PaymentMetadata.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceFeatures.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingNodeId.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceRoutingInfo.serializer())

src/commonMain/kotlin/fr/acinq/lightning/serialization/v2/Serialization.kt

@@ -59,6 +59,7 @@ object Serialization {
             subclass(OnionPaymentPayloadTlv.OutgoingCltv.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingChannelId.serializer())
             subclass(OnionPaymentPayloadTlv.PaymentData.serializer())
+            subclass(OnionPaymentPayloadTlv.PaymentMetadata.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceFeatures.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingNodeId.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceRoutingInfo.serializer())

src/commonMain/kotlin/fr/acinq/lightning/serialization/v3/Serialization.kt

@@ -59,6 +59,7 @@ object Serialization {
             subclass(OnionPaymentPayloadTlv.OutgoingCltv.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingChannelId.serializer())
             subclass(OnionPaymentPayloadTlv.PaymentData.serializer())
+            subclass(OnionPaymentPayloadTlv.PaymentMetadata.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceFeatures.serializer())
             subclass(OnionPaymentPayloadTlv.OutgoingNodeId.serializer())
             subclass(OnionPaymentPayloadTlv.InvoiceRoutingInfo.serializer())

src/commonMain/kotlin/fr/acinq/lightning/wire/PaymentOnion.kt

@@ -75,6 +75,21 @@ sealed class OnionPaymentPayloadTlv : Tlv {
         }
     }
 
+    /**
+     * When payment metadata is included in a Bolt 9 invoice, we should send it as-is to the recipient.
+     * This lets recipients generate invoices without having to store anything on their side until the invoice is paid.
+     */
+    @Serializable
+    data class PaymentMetadata(@Contextual val data: ByteVector) : OnionPaymentPayloadTlv() {
+        override val tag: Long get() = PaymentMetadata.tag
+        override fun write(out: Output) = LightningCodecs.writeBytes(data, out)
+
+        companion object : TlvValueReader<PaymentMetadata> {
+            const val tag: Long = 16
+            override fun read(input: Input): PaymentMetadata = PaymentMetadata(ByteVector(LightningCodecs.bytes(input, input.availableBytes)))
+        }
+    }
+
     /**
      * Invoice feature bits. Only included for intermediate trampoline nodes when they should convert to a legacy payment
      * because the final recipient doesn't support trampoline.
@@ -176,6 +191,7 @@ object PaymentOnion {
                     OnionPaymentPayloadTlv.OutgoingCltv.tag to OnionPaymentPayloadTlv.OutgoingCltv.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
                     OnionPaymentPayloadTlv.OutgoingChannelId.tag to OnionPaymentPayloadTlv.OutgoingChannelId.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
                     OnionPaymentPayloadTlv.PaymentData.tag to OnionPaymentPayloadTlv.PaymentData.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
+                    OnionPaymentPayloadTlv.PaymentMetadata.tag to OnionPaymentPayloadTlv.PaymentMetadata.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
                     OnionPaymentPayloadTlv.InvoiceFeatures.tag to OnionPaymentPayloadTlv.InvoiceFeatures.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
                     OnionPaymentPayloadTlv.OutgoingNodeId.tag to OnionPaymentPayloadTlv.OutgoingNodeId.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
                     OnionPaymentPayloadTlv.InvoiceRoutingInfo.tag to OnionPaymentPayloadTlv.InvoiceRoutingInfo.Companion as TlvValueReader<OnionPaymentPayloadTlv>,
@@ -198,30 +214,54 @@ object PaymentOnion {
             val total = records.get<OnionPaymentPayloadTlv.PaymentData>()!!.totalAmount
             if (total > 0.msat) total else amount
         }
+        val paymentMetadata = records.get<OnionPaymentPayloadTlv.PaymentMetadata>()?.data
 
         override fun write(out: Output) = tlvSerializer.write(records, out)
 
         companion object : PerHopPayloadReader<FinalPayload> {
             override fun read(input: Input): FinalPayload = FinalPayload(tlvSerializer.read(input))
 
             /** Create a single-part payment (total amount sent at once). */
-            fun createSinglePartPayload(amount: MilliSatoshi, expiry: CltvExpiry, paymentSecret: ByteVector32, userCustomTlvs: List<GenericTlv> = listOf()): FinalPayload =
-                FinalPayload(TlvStream(listOf(OnionPaymentPayloadTlv.AmountToForward(amount), OnionPaymentPayloadTlv.OutgoingCltv(expiry), OnionPaymentPayloadTlv.PaymentData(paymentSecret, amount)), userCustomTlvs))
+            fun createSinglePartPayload(amount: MilliSatoshi, expiry: CltvExpiry, paymentSecret: ByteVector32, paymentMetadata: ByteVector?, userCustomTlvs: List<GenericTlv> = listOf()): FinalPayload {
+                val tlvs = buildList {
+                    add(OnionPaymentPayloadTlv.AmountToForward(amount))
+                    add(OnionPaymentPayloadTlv.OutgoingCltv(expiry))
+                    add(OnionPaymentPayloadTlv.PaymentData(paymentSecret, amount))
+                    paymentMetadata?.let { add(OnionPaymentPayloadTlv.PaymentMetadata(it)) }
+                }
+                return FinalPayload(TlvStream(tlvs, userCustomTlvs))
+            }
 
             /** Create a partial payment (total amount split between multiple payments). */
             fun createMultiPartPayload(
-                amount: MilliSatoshi, totalAmount: MilliSatoshi, expiry: CltvExpiry, paymentSecret: ByteVector32, additionalTlvs: List<OnionPaymentPayloadTlv> = listOf(), userCustomTlvs: List<GenericTlv> = listOf()
-            ): FinalPayload =
-                FinalPayload(TlvStream(listOf(OnionPaymentPayloadTlv.AmountToForward(amount), OnionPaymentPayloadTlv.OutgoingCltv(expiry), OnionPaymentPayloadTlv.PaymentData(paymentSecret, totalAmount)) + additionalTlvs, userCustomTlvs))
+                amount: MilliSatoshi,
+                totalAmount: MilliSatoshi,
+                expiry: CltvExpiry,
+                paymentSecret: ByteVector32,
+                paymentMetadata: ByteVector?,
+                additionalTlvs: List<OnionPaymentPayloadTlv> = listOf(),
+                userCustomTlvs: List<GenericTlv> = listOf()
+            ): FinalPayload {
+                val tlvs = buildList {
+                    add(OnionPaymentPayloadTlv.AmountToForward(amount))
+                    add(OnionPaymentPayloadTlv.OutgoingCltv(expiry))
+                    add(OnionPaymentPayloadTlv.PaymentData(paymentSecret, totalAmount))
+                    paymentMetadata?.let { add(OnionPaymentPayloadTlv.PaymentMetadata(it)) }
+                    addAll(additionalTlvs)
+                }
+                return FinalPayload(TlvStream(tlvs, userCustomTlvs))
+            }
 
             /** Create a trampoline outer payload. */
-            fun createTrampolinePayload(amount: MilliSatoshi, totalAmount: MilliSatoshi, expiry: CltvExpiry, paymentSecret: ByteVector32, trampolinePacket: OnionRoutingPacket): FinalPayload = FinalPayload(
-                TlvStream(
-                    listOf(
-                        OnionPaymentPayloadTlv.AmountToForward(amount), OnionPaymentPayloadTlv.OutgoingCltv(expiry), OnionPaymentPayloadTlv.PaymentData(paymentSecret, totalAmount), OnionPaymentPayloadTlv.TrampolineOnion(trampolinePacket)
-                    )
-                )
-            )
+            fun createTrampolinePayload(amount: MilliSatoshi, totalAmount: MilliSatoshi, expiry: CltvExpiry, paymentSecret: ByteVector32, trampolinePacket: OnionRoutingPacket): FinalPayload {
+                val tlvs = buildList {
+                    add(OnionPaymentPayloadTlv.AmountToForward(amount))
+                    add(OnionPaymentPayloadTlv.OutgoingCltv(expiry))
+                    add(OnionPaymentPayloadTlv.PaymentData(paymentSecret, totalAmount))
+                    add(OnionPaymentPayloadTlv.TrampolineOnion(trampolinePacket))
+                }
+                return FinalPayload(TlvStream(tlvs))
+            }
         }
     }
 
@@ -255,6 +295,7 @@ object PaymentOnion {
 
         // NB: the following fields are only included in the trampoline-to-legacy case.
         val paymentSecret = records.get<OnionPaymentPayloadTlv.PaymentData>()?.secret
+        val paymentMetadata = records.get<OnionPaymentPayloadTlv.PaymentMetadata>()?.data
         val invoiceFeatures = records.get<OnionPaymentPayloadTlv.InvoiceFeatures>()?.features
         val invoiceRoutingInfo = records.get<OnionPaymentPayloadTlv.InvoiceRoutingInfo>()?.extraHops
 
@@ -279,14 +320,15 @@ object PaymentOnion {
                 }.map { it.hints }
                 return NodeRelayPayload(
                     TlvStream(
-                        listOf(
-                            OnionPaymentPayloadTlv.AmountToForward(amount),
-                            OnionPaymentPayloadTlv.OutgoingCltv(expiry),
-                            OnionPaymentPayloadTlv.OutgoingNodeId(targetNodeId),
-                            OnionPaymentPayloadTlv.PaymentData(invoice.paymentSecret, totalAmount),
-                            OnionPaymentPayloadTlv.InvoiceFeatures(invoice.features),
-                            OnionPaymentPayloadTlv.InvoiceRoutingInfo(prunedRoutingHints)
-                        )
+                        buildList {
+                            add(OnionPaymentPayloadTlv.AmountToForward(amount))
+                            add(OnionPaymentPayloadTlv.OutgoingCltv(expiry))
+                            add(OnionPaymentPayloadTlv.OutgoingNodeId(targetNodeId))
+                            add(OnionPaymentPayloadTlv.PaymentData(invoice.paymentSecret, totalAmount))
+                            invoice.paymentMetadata?.let { add(OnionPaymentPayloadTlv.PaymentMetadata(it)) }
+                            add(OnionPaymentPayloadTlv.InvoiceFeatures(invoice.features))
+                            add(OnionPaymentPayloadTlv.InvoiceRoutingInfo(prunedRoutingHints))
+                        }
                     )
                 )
             }

src/commonTest/kotlin/fr/acinq/lightning/channel/TestsHelper.kt

@@ -320,7 +320,7 @@ object TestsHelper {
         val expiry = CltvExpiryDelta(144).toCltvExpiry(currentBlockHeight)
         val dummyKey = PrivateKey(ByteVector32("0101010101010101010101010101010101010101010101010101010101010101")).publicKey()
         val dummyUpdate = ChannelUpdate(ByteVector64.Zeroes, ByteVector32.Zeroes, ShortChannelId(144, 0, 0), 0, 0, 0, CltvExpiryDelta(1), 0.msat, 0.msat, 0, null)
-        val cmd = OutgoingPaymentPacket.buildCommand(paymentId, paymentHash, listOf(ChannelHop(dummyKey, destination, dummyUpdate)), PaymentOnion.FinalPayload.createSinglePartPayload(amount, expiry, randomBytes32())).first.copy(commit = false)
+        val cmd = OutgoingPaymentPacket.buildCommand(paymentId, paymentHash, listOf(ChannelHop(dummyKey, destination, dummyUpdate)), PaymentOnion.FinalPayload.createSinglePartPayload(amount, expiry, randomBytes32(), null)).first.copy(commit = false)
         return Pair(paymentPreimage, cmd)
     }