Implement nokia NAT logging. See #533

Author: phaag

man/nfdump.1

@@ -1853,6 +1853,14 @@ NAT pool block end
 NAT pool block step
 .It Cm %pbsize
 NAT pool block size
+.It Cm %flid
+Flow ID
+.It Cm %isid
+Nokia NAT in service ID
+.It Cm %osid
+Nokia NAT out service ID
+.It Cm %nats
+Nokia NAT string
 .Pp
 .It Nprobe formats
 .It Cm %cl

src/inline/nffile_inline.c

@@ -73,8 +73,8 @@ static inline int MapRecordHandle(recordHandle_t *handle, recordHeaderV3_t *reco
         if (nselCommon) {
             genericFlow->msecFirst = nselCommon->msecEvent;
         } else {
-            EXnelCommon_t *nelCommon = (EXnelCommon_t *)handle->extensionList[EXnelCommonID];
-            if (nelCommon) genericFlow->msecFirst = nelCommon->msecEvent;
+            EXnatCommon_t *natCommon = (EXnatCommon_t *)handle->extensionList[EXnatCommonID];
+            if (natCommon) genericFlow->msecFirst = natCommon->msecEvent;
         }
     }
     return 1;

src/libnfdump/filter/grammar.y

@@ -1170,7 +1170,7 @@ static int AddNATString(char *event, char *natStr) {
 			natEventInfo();
 			return -1;
 		}
-		return NewElement(EXnelCommonID, OFFnatEvent, SIZEnatEvent, eventNum, CMP_EQ, FUNC_NONE, NULLPtr);
+		return NewElement(EXnatCommonID, OFFnatEvent, SIZEnatEvent, eventNum, CMP_EQ, FUNC_NONE, NULLPtr);
 	} 
 
 	yyerror("Invalid NAT type: %s", event);
@@ -1185,7 +1185,7 @@ static int AddNAT(char *event, uint16_t comp, uint64_t number) {
 			yyerror("NAT event: %llu out of range\n", number);
 			return -1;
 		}
-		return NewElement(EXnelCommonID, OFFnatEvent, SIZEnatEvent, number, comp, FUNC_NONE, NULLPtr);
+		return NewElement(EXnatCommonID, OFFnatEvent, SIZEnatEvent, number, comp, FUNC_NONE, NULLPtr);
 	} 
 
 	return -1;

src/libnffile/conf/nfdump.conf.dist

@@ -22,7 +22,12 @@
 # define any new format
 # fmt.newformat = "%ts ...."
 fmt.geolong = "%ts %td %pr %sc %gsap -> %dc %gdap %flg %pkt %byt %fl"
-fmt.pfline = "fmt:%ts %pfact %pfrea  %pfdir on %pfifn %pfrule  %pr %sap -> %dap %pkt %byt"
+
+# Nokia NAT
+fmt.nokia = "%ts %te %flid %pr %sap -> %dap %isid %osid %nats"
+
+# OpenBSD pf logs
+fmt.pflog = "%ts %pfact %pfrea  %pfdir on %pfifn %pfrule  %pr %sap -> %dap %pkt %byt"
 
 # geodb
 # if you use maxmind DB to geo-locate IPs - see geolookup(1)

src/libnffile/nfxV3.h

@@ -481,35 +481,22 @@ typedef struct EXnselUser_s {
 } EXnselUser_t;
 #define EXnselUserSize (sizeof(EXnselUser_t) + sizeof(elementHeader_t))
 
-// NEL
-typedef struct EXnelCommon_s {
-#define EXnelCommonID 25
+// NAT event logging
+typedef struct EXnatCommon_s {
+#define EXnatCommonID 25
     uint64_t msecEvent;  // NF_F_EVENT_TIME_MSEC(323)
     uint32_t natPoolID;  // NF_N_NATPOOL_ID(283)
     uint8_t natEvent;    // NAT_EVENT(230)
     uint8_t fill1;
     uint16_t fill2;
-#define OFFnelMsecEvent offsetof(EXnelCommon_t, msecEvent)
-#define SIZEnelMsecEvent MemberSize(EXnelCommon_t, msecEvent)
-#define OFFnatPoolID offsetof(EXnelCommon_t, natPoolID)
-#define SIZEnatPoolID MemberSize(EXnelCommon_t, natPoolID)
-#define OFFnatEvent offsetof(EXnelCommon_t, natEvent)
-#define SIZEnatEvent MemberSize(EXnelCommon_t, natEvent)
-} EXnelCommon_t;
-#define EXnelCommonSize (sizeof(EXnelCommon_t) + sizeof(elementHeader_t))
-
-// comapt record includes vrf fields
-// no longer used, but old data may exist
-typedef struct EXnelCommonCompat_s {
-    uint64_t msecEvent;   // NF_F_EVENT_TIME_MSEC(323)
-    uint32_t egressVrf;   // NF_N_EGRESS_VRFID(235)
-    uint32_t ingressVrf;  // NF_N_INGRESS_VRFID(234)
-    uint32_t natPoolID;   // NF_N_NATPOOL_ID(283)
-    uint8_t natEvent;     // NAT_EVENT(230)
-    uint8_t fill1;
-    uint16_t fill2;
-} EXnelCommonCompat_t;
-#define EXnelCommonCompatSize (sizeof(EXnelCommonCompat_t) + sizeof(elementHeader_t))
+#define OFFnelMsecEvent offsetof(EXnatCommon_t, msecEvent)
+#define SIZEnelMsecEvent MemberSize(EXnatCommon_t, msecEvent)
+#define OFFnatPoolID offsetof(EXnatCommon_t, natPoolID)
+#define SIZEnatPoolID MemberSize(EXnatCommon_t, natPoolID)
+#define OFFnatEvent offsetof(EXnatCommon_t, natEvent)
+#define SIZEnatEvent MemberSize(EXnatCommon_t, natEvent)
+} EXnatCommon_t;
+#define EXnatCommonSize (sizeof(EXnatCommon_t) + sizeof(elementHeader_t))
 
 typedef struct EXnatPortBlock_s {
 #define EXnatPortBlockID 26
@@ -672,8 +659,35 @@ typedef struct EXlayer2_s {
 } EXlayer2_t;
 #define EXlayer2Size (sizeof(EXlayer2_t) + sizeof(elementHeader_t))
 
+typedef struct EXflowId_s {
+#define EXflowIdID 39
+    uint64_t flowId;  // IPFIX_flowId
+#define OFFflowId offsetof(EXflowId_t, flowId)
+#define SIZEflowId MemberSize(EXflowId_t, flowId)
+} EXflowId_t;
+#define EXflowIdSize (sizeof(EXflowId_t) + sizeof(elementHeader_t))
+
+typedef struct EXnokiaNat_s {
+#define EXnokiaNatID 40
+    uint16_t inServiceID;
+    uint16_t outServiceID;
+#define OFFinServiceID offsetof(EXnokiaNat_t, inServiceID)
+#define SIZEinServiceID MemberSize(EXnokiaNat_t, inServiceID)
+#define OFFoutServiceID offsetof(EXnokiaNat_t, outServiceID)
+#define SIZEoutServiceID MemberSize(EXnokiaNat_t, outServiceID)
+} EXnokiaNat_t;
+#define EXnokiaNatSize (sizeof(EXnokiaNat_t) + sizeof(elementHeader_t))
+
+typedef struct EXnokiaNatString_s {
+#define EXnokiaNatStringID 41
+    char natSubString[4];
+#define OFFnatSubString offsetof(EXnokiaNatString_t, natSubString)
+#define SIZEnatSubString VARLENGTH
+} EXnokiaNatString_t;
+#define EXnokiaNatStringSize (sizeof(EXnokiaNatString_t) + sizeof(elementHeader_t))
+
 // max possible elements
-#define MAXEXTENSIONS 39
+#define MAXEXTENSIONS 42
 
 // push a fixed length extension to the v3 record
 // h v3 record header
@@ -734,14 +748,15 @@ static const struct extensionTable_s {
     uint32_t size;  // number of bytes incl. header, 0xFFFF for dyn length
     char *name;     // name of extension
 } extensionTable[] = {
-    {0, 0, "ExNull"},          EXTENSION(EXgenericFlow),  EXTENSION(EXipv4Flow),     EXTENSION(EXipv6Flow),     EXTENSION(EXflowMisc),
-    EXTENSION(EXcntFlow),      EXTENSION(EXvLan),         EXTENSION(EXasRouting),    EXTENSION(EXbgpNextHopV4), EXTENSION(EXbgpNextHopV6),
-    EXTENSION(EXipNextHopV4),  EXTENSION(EXipNextHopV6),  EXTENSION(EXipReceivedV4), EXTENSION(EXipReceivedV6), EXTENSION(EXmplsLabel),
-    EXTENSION(EXmacAddr),      EXTENSION(EXasAdjacent),   EXTENSION(EXlatency),      EXTENSION(EXsamplerInfo),  EXTENSION(EXnselCommon),
-    EXTENSION(EXnatXlateIPv4), EXTENSION(EXnatXlateIPv6), EXTENSION(EXnatXlatePort), EXTENSION(EXnselAcl),      EXTENSION(EXnselUser),
-    EXTENSION(EXnelCommon),    EXTENSION(EXnatPortBlock), EXTENSION(EXnbarApp),      EXTENSION(EXlabel),        EXTENSION(EXinPayload),
-    EXTENSION(EXoutPayload),   EXTENSION(EXtunIPv4),      EXTENSION(EXtunIPv6),      EXTENSION(EXobservation),  EXTENSION(EXinmonMeta),
-    EXTENSION(EXinmonFrame),   EXTENSION(EXvrf),          EXTENSION(EXpfinfo),       EXTENSION(EXlayer2)};
+    {0, 0, "EXnull"},          EXTENSION(EXgenericFlow),   EXTENSION(EXipv4Flow),     EXTENSION(EXipv6Flow),     EXTENSION(EXflowMisc),
+    EXTENSION(EXcntFlow),      EXTENSION(EXvLan),          EXTENSION(EXasRouting),    EXTENSION(EXbgpNextHopV4), EXTENSION(EXbgpNextHopV6),
+    EXTENSION(EXipNextHopV4),  EXTENSION(EXipNextHopV6),   EXTENSION(EXipReceivedV4), EXTENSION(EXipReceivedV6), EXTENSION(EXmplsLabel),
+    EXTENSION(EXmacAddr),      EXTENSION(EXasAdjacent),    EXTENSION(EXlatency),      EXTENSION(EXsamplerInfo),  EXTENSION(EXnselCommon),
+    EXTENSION(EXnatXlateIPv4), EXTENSION(EXnatXlateIPv6),  EXTENSION(EXnatXlatePort), EXTENSION(EXnselAcl),      EXTENSION(EXnselUser),
+    EXTENSION(EXnatCommon),    EXTENSION(EXnatPortBlock),  EXTENSION(EXnbarApp),      EXTENSION(EXlabel),        EXTENSION(EXinPayload),
+    EXTENSION(EXoutPayload),   EXTENSION(EXtunIPv4),       EXTENSION(EXtunIPv6),      EXTENSION(EXobservation),  EXTENSION(EXinmonMeta),
+    EXTENSION(EXinmonFrame),   EXTENSION(EXvrf),           EXTENSION(EXpfinfo),       EXTENSION(EXlayer2),       EXTENSION(EXflowId),
+    EXTENSION(EXnokiaNat),     EXTENSION(EXnokiaNatString)};
 
 typedef struct record_map_s {
     recordHeaderV3_t *recordHeader;

src/libnffile/vcs_track.h

@@ -2,6 +2,6 @@
 #define __VCS_TRACK_H__
 //THIS FILE IS AUTO GENERATED
 //DO NOT TRACK THIS FILE WITH THE VCS
-#define VCS_TRACK_DATE "2024-06-01 15:41:05 +0200"
-#define VCS_TRACK_HASH "1619a60"
+#define VCS_TRACK_DATE "2024-06-02 11:22:32 +0200"
+#define VCS_TRACK_HASH "f0a5f6f"
 #endif

src/netflow/ipfix.c

@@ -221,14 +221,15 @@ static const struct ipfixTranslationMap_s {
     {IPFIX_INGRESS_VRFID, SIZEingressVrf, NumberCopy, EXvrfID, OFFingressVrf, STACK_NONE, "ingress VRF ID"},
     {IPFIX_EGRESS_VRFID, SIZEegressVrf, NumberCopy, EXvrfID, OFFegressVrf, STACK_NONE, "egress VRF ID"},
     // NAT
-    {IPFIX_observationTimeMilliseconds, SIZEmsecEvent, NumberCopy, EXnelCommonID, OFFmsecEvent, STACK_MSEC, "msec time event"},
-    {IPFIX_natEvent, SIZEnatEvent, NumberCopy, EXnelCommonID, OFFnatEvent, STACK_NONE, "NAT event"},
+    {IPFIX_observationTimeMilliseconds, SIZEmsecEvent, NumberCopy, EXnatCommonID, OFFmsecEvent, STACK_MSEC, "msec time event"},
+    {IPFIX_natEvent, SIZEnatEvent, NumberCopy, EXnatCommonID, OFFnatEvent, STACK_NONE, "NAT event"},
     {IPFIX_postNATSourceIPv4Address, SIZExlateSrc4Addr, NumberCopy, EXnatXlateIPv4ID, OFFxlateSrc4Addr, STACK_NONE, "xlate src addr"},
     {IPFIX_postNATDestinationIPv4Address, SIZExlateDst4Addr, NumberCopy, EXnatXlateIPv4ID, OFFxlateDst4Addr, STACK_NONE, "xlate dst addr"},
     {IPFIX_postNAPTSourceTransportPort, SIZExlateSrcPort, NumberCopy, EXnatXlatePortID, OFFxlateSrcPort, STACK_NONE, "xlate src port"},
     {IPFIX_postNAPTDestinationTransportPort, SIZExlateDstPort, NumberCopy, EXnatXlatePortID, OFFxlateDstPort, STACK_NONE, "xlate dst port"},
+    {IPFIX_flowId, SIZEflowId, NumberCopy, EXflowIdID, OFFflowId, STACK_NONE, "flow ID"},
     // cgNAT
-    {IPFIX_NATPOOL_ID, SIZEnatPoolID, NumberCopy, EXnelCommonID, OFFnatPoolID, STACK_NONE, "nat pool ID"},
+    {IPFIX_NATPOOL_ID, SIZEnatPoolID, NumberCopy, EXnatCommonID, OFFnatPoolID, STACK_NONE, "nat pool ID"},
     {IPFIX_PORT_BLOCK_START, SIZEnelblockStart, NumberCopy, EXnatPortBlockID, OFFnelblockStart, STACK_NONE, "NAT block start"},
     {IPFIX_PORT_BLOCK_END, SIZEnelblockEnd, NumberCopy, EXnatPortBlockID, OFFnelblockEnd, STACK_NONE, "NAT block end"},
     {IPFIX_PORT_BLOCK_STEP, SIZEnelblockStep, NumberCopy, EXnatPortBlockID, OFFnelblockStep, STACK_NONE, "NAT block step"},
@@ -239,8 +240,13 @@ static const struct ipfixTranslationMap_s {
     {IPFIX_dataLinkFrameSection, SIZEpacket, ByteCopy, EXinmonFrameID, OFFpacket, STACK_NONE, "inmon packet content"},
 
     // payload
-    {LOCAL_inPayload, VARLENGTH, NumberCopy, EXinPayloadID, 0, STACK_NONE, "in payload"},
-    {LOCAL_outPayload, VARLENGTH, NumberCopy, EXoutPayloadID, 0, STACK_NONE, "out payload"},
+    {LOCAL_inPayload, VARLENGTH, ByteCopy, EXinPayloadID, 0, STACK_NONE, "in payload"},
+    {LOCAL_outPayload, VARLENGTH, ByteCopy, EXoutPayloadID, 0, STACK_NONE, "out payload"},
+
+    // Nokia
+    {NOKIA_InsideServiceId, SIZEinServiceID, NumberCopy, EXnokiaNatID, OFFinServiceID, STACK_NONE, "Nokia inside service ID"},
+    {NOKIA_OutsideServiceId, SIZEoutServiceID, NumberCopy, EXnokiaNatID, OFFoutServiceID, STACK_NONE, "Nokia outside service ID"},
+    {NOKIA_NatSubString, SIZEnatSubString, ByteCopy, EXnokiaNatStringID, OFFnatSubString, STACK_NONE, "Nokia nat substring"},
 
     // End of table
     {0, 0, 0, 0, 0, STACK_NONE, NULL},

src/netflow/ipfix.h

@@ -277,6 +277,7 @@ typedef struct ipfix_template_elements_e_s {
 #define IPFIX_flowEndReason 136
 #define IPFIX_observationPointId 138
 #define IPFIX_icmpTypeCodeIPv6 139
+#define IPFIX_flowId 148
 #define IPFIX_observationDomainId 149
 #define IPFIX_flowStartSeconds 150
 #define IPFIX_flowEndSeconds 151

src/netflow/netflow_v9.c

@@ -235,8 +235,8 @@ static const struct v9TranslationMap_s {
     {NF_N_EGRESS_VRFID, SIZEegressVrf, NumberCopy, EXvrfID, OFFegressVrf, STACK_NONE, "egress VRF ID"},
 
     // NEL
-    {NF_N_NAT_EVENT, SIZEnatEvent, NumberCopy, EXnelCommonID, OFFnatEvent, STACK_NONE, "NAT event"},
-    {NF_N_NATPOOL_ID, SIZEnatPoolID, NumberCopy, EXnelCommonID, OFFnatPoolID, STACK_NONE, "nat pool ID"},
+    {NF_N_NAT_EVENT, SIZEnatEvent, NumberCopy, EXnatCommonID, OFFnatEvent, STACK_NONE, "NAT event"},
+    {NF_N_NATPOOL_ID, SIZEnatPoolID, NumberCopy, EXnatCommonID, OFFnatPoolID, STACK_NONE, "nat pool ID"},
     {NF_F_XLATE_PORT_BLOCK_START, SIZEnelblockStart, NumberCopy, EXnatPortBlockID, OFFnelblockStart, STACK_NONE, "NAT block start"},
     {NF_F_XLATE_PORT_BLOCK_END, SIZEnelblockEnd, NumberCopy, EXnatPortBlockID, OFFnelblockEnd, STACK_NONE, "NAT block end"},
     {NF_F_XLATE_PORT_BLOCK_STEP, SIZEnelblockStep, NumberCopy, EXnatPortBlockID, OFFnelblockStep, STACK_NONE, "NAT block step"},
@@ -1267,9 +1267,9 @@ static inline void Process_v9_data(exporterDomain_t *exporter, void *data_flowse
             }
             SetFlag(recordHeaderV3->flags, V3_FLAG_EVENT);
         }
-        EXnelCommon_t *nelCommon = sequencer->offsetCache[EXnelCommonID];
-        if (nelCommon) {
-            nelCommon->msecEvent = stack[STACK_MSEC];
+        EXnatCommon_t *natCommon = sequencer->offsetCache[EXnatCommonID];
+        if (natCommon) {
+            natCommon->msecEvent = stack[STACK_MSEC];
             if (genericFlow) {
                 genericFlow->msecFirst = stack[STACK_MSEC];
                 genericFlow->msecLast = stack[STACK_MSEC];

src/nfdump/compat_1_6_x/convert.c

@@ -386,8 +386,8 @@ static inline int ConvertRecordV2(common_record_t *commonRecord, dataBlock_t *da
             } break;
             case EX_NEL_COMMON: {
                 tpl_ext_46_t *tpl = (tpl_ext_46_t *)p;
-                PushExtension(recordHeader, EXnelCommon, nelCommon);
-                nelCommon->natEvent = tpl->nat_event;
+                PushExtension(recordHeader, EXnatCommon, natCommon);
+                natCommon->natEvent = tpl->nat_event;
                 PushExtension(recordHeader, EXvrf, vrf);
                 vrf->egressVrf = tpl->egress_vrfid;
                 vrf->ingressVrf = tpl->ingress_vrfid;

src/nfdump/nfstat.c

@@ -197,7 +197,7 @@ static struct StatParameter_s {
     {"opid", "Obs PointID", {EXobservationID, OFFpointID, SIZEpointID, 0}, IS_HEXNUMBER, NULL},
     {"event", " Event", {EXnselCommonID, OFFfwEvent, SIZEfwEvent, 0}, IS_EVENT, NULL},
     {"xevent", " Event", {EXnselCommonID, OFFfwXevent, SIZEfwXevent, 0}, IS_NUMBER, NULL},
-    {"nat", "NAT Event", {EXnelCommonID, OFFnatEvent, SIZEnatEvent, 0}, IS_EVENT, NULL},
+    {"nat", "NAT Event", {EXnatCommonID, OFFnatEvent, SIZEnatEvent, 0}, IS_EVENT, NULL},
     {"xsrcip", "X-Src IP Addr", {EXnatXlateIPv4ID, OFFxlateSrc4Addr, SIZExlateSrc4Addr, AF_INET}, IS_IPADDR, NULL},
     {"xsrcip", NULL, {EXnatXlateIPv6ID, OFFxlateSrc6Addr, SIZExlateSrc6Addr, AF_INET6}, IS_IPADDR, NULL},
     {"xdstip", "X-Dst IP Addr", {EXnatXlateIPv4ID, OFFxlateDst4Addr, SIZExlateDst4Addr, AF_INET}, IS_IPADDR, NULL},