Skip to content

Enhancement: Queue rate counter support #4052

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Enhancement: Queue rate counter support #4052

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c49cc4b
Add queue rate counter support
ssithaia-ebay Aug 26, 2025
e9a1cca
23910 - quee counter port stats removed
ssithaia-ebay Sep 5, 2025
5b0d175
test file changes
ssithaia-ebay Sep 5, 2025
07a666a
queue_counter rate and netstat test changes
ssithaia-ebay Sep 5, 2025
207bd2c
netstat test changes
ssithaia-ebay Sep 5, 2025
bda0605
netstat test changes
ssithaia-ebay Sep 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 79 additions & 25 deletions scripts/queuestat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,23 +57,31 @@ VoqStats = namedtuple(

std_header = [
'Port', 'TxQ',
'Counter/pkts', 'Counter/bytes', 'Drop/pkts', 'Drop/bytes'
'Counter/pkts', 'Counter/bytes', 'Drop/pkts', 'Drop/bytes',
'Pkts/s', 'Bytes/s', 'Bits/s'
]
all_header = [
'Port', 'TxQ',
'Counter/pkts', 'Counter/bytes', 'Drop/pkts', 'Drop/bytes',
'Trim/pkts', 'TrimSent/pkts', 'TrimDrop/pkts'
'Trim/pkts', 'TrimSent/pkts', 'TrimDrop/pkts',
'Pkts/s', 'Bytes/s', 'Bits/s'
]
trim_header = [
'Port', 'TxQ',
'Trim/pkts', 'TrimSent/pkts', 'TrimDrop/pkts'
'Trim/pkts', 'TrimSent/pkts', 'TrimDrop/pkts',
'Pkts/s', 'Bytes/s', 'Bits/s'
]
voq_header = [
'Port', 'Voq',
'Counter/pkts', 'Counter/bytes', 'Drop/pkts', 'Drop/bytes',
'Credit-WD-Del/pkts'
'Credit-WD-Del/pkts',
'Pkts/s', 'Bytes/s', 'Bits/s'
]

rates_key_list = [ 'Q_PPS', 'Q_BPS', 'Q_bPS']
ratestat = [ 'qpktsps', 'qbytesps', 'qbitsps']
RateStats = namedtuple("RateStats", ratestat)

counter_bucket_dict = {
'SAI_QUEUE_STAT_PACKETS': 2,
'SAI_QUEUE_STAT_BYTES': 3,
Expand Down Expand Up @@ -101,6 +109,7 @@ SAI_QUEUE_TYPE_UNICAST = "SAI_QUEUE_TYPE_UNICAST"
SAI_QUEUE_TYPE_UNICAST_VOQ = "SAI_QUEUE_TYPE_UNICAST_VOQ"
SAI_QUEUE_TYPE_ALL = "SAI_QUEUE_TYPE_ALL"

RATES_TABLE_PREFIX = "RATES:"
COUNTER_TABLE_PREFIX = "COUNTERS:"
COUNTERS_PORT_NAME_MAP = "COUNTERS_PORT_NAME_MAP"
COUNTERS_SYSTEM_PORT_NAME_MAP = "COUNTERS_SYSTEM_PORT_NAME_MAP"
Expand Down Expand Up @@ -374,16 +383,33 @@ class Queuestat(object):
cntr = QueueStats._make(fields)._asdict()
return cntr

def get_rates(table_id):
"""
Get the rates from specific table.
"""
fields = ["0", "0", "0"]
for pos, name in enumerate(rates_key_list):
full_table_id = RATES_TABLE_PREFIX + table_id
counter_data = self.db.get(self.db.COUNTERS_DB, full_table_id, name)
if counter_data is None:
fields[pos] = STATUS_NA
elif fields[pos] != STATUS_NA:
fields[pos] = float(counter_data)
cntr = RateStats._make(fields)
return cntr

# Build a dictionary of the stats
cnstat_dict = OrderedDict()
cnstat_dict['time'] = datetime.datetime.now()
ratestat_dict = OrderedDict()
if queue_map is None:
return cnstat_dict
for queue in natsorted(queue_map):
cnstat_dict[queue] = get_counters(queue_map[queue])
return cnstat_dict
ratestat_dict[queue] = get_rates(queue_map[queue])
return cnstat_dict,ratestat_dict

def cnstat_print(self, port, cnstat_dict, json_opt, non_zero):
def cnstat_print(self, port, cnstat_dict, json_opt, non_zero,ratestat_dict):
"""
Print the cnstat. If JSON option is True, return data in
JSON format.
Expand All @@ -396,12 +422,23 @@ class Queuestat(object):
if json_opt:
json_output[port][key] = data
continue

qpktsps = qbytesps = qbitsps = STATUS_NA
rates = ratestat_dict.get(key, RateStats._make([STATUS_NA] * len(ratestat)))
if rates.qpktsps != STATUS_NA:
qpktsps = "{:,}".format(int(float(rates.qpktsps)))
if rates.qbytesps != STATUS_NA:
qbytesps = "{:,}".format(int(float(rates.qbytesps)))
if rates.qbitsps != STATUS_NA:
qbitsps = "{:,}".format(int(float(rates.qbitsps)))

if self.voq:
if not non_zero or data['totalpacket'] != '0' or data['totalbytes'] != '0' or \
data['droppacket'] != '0' or data['dropbytes'] != '0' or data['creditWDpkts'] != '0':
table.append((port, data['queuetype'] + str(data['queueindex']),
data['totalpacket'], data['totalbytes'],
data['droppacket'], data['dropbytes'], data['creditWDpkts']))
data['droppacket'], data['dropbytes'], data['creditWDpkts'],
qpktsps, qbytesps, qbitsps))
else:
queuetag = data['queuetype'] + str(data['queueindex'])

Expand All @@ -414,14 +451,16 @@ class Queuestat(object):
port, queuetag,
data['totalpacket'], data['totalbytes'],
data['droppacket'], data['dropbytes'],
data['trimpkt'], data['trimsentpkt'], data['trimdroppkt']
data['trimpkt'], data['trimsentpkt'], data['trimdroppkt'],
qpktsps, qbytesps, qbitsps
))
elif self.trim: # Packet Trimming related statistics
if not non_zero or \
data['trimpkt'] != '0' or data['trimsentpkt'] != '0' or data['trimdroppkt'] != '0':
table.append((
port, queuetag,
data['trimpkt'], data['trimsentpkt'], data['trimdroppkt']
data['trimpkt'], data['trimsentpkt'], data['trimdroppkt'],
qpktsps, qbytesps, qbitsps
))
else: # Generic statistics
if not non_zero or \
Expand All @@ -430,7 +469,8 @@ class Queuestat(object):
table.append((
port, queuetag,
data['totalpacket'], data['totalbytes'],
data['droppacket'], data['dropbytes']
data['droppacket'], data['dropbytes'],
qpktsps, qbytesps, qbitsps
))

if json_opt:
Expand All @@ -452,7 +492,7 @@ class Queuestat(object):
print(tabulate(table, hdr, tablefmt='simple', stralign='right'))
print()

def cnstat_diff_print(self, port, cnstat_new_dict, cnstat_old_dict, json_opt, non_zero):
def cnstat_diff_print(self, port, cnstat_new_dict, cnstat_old_dict, json_opt, non_zero,ratestat_dict):
"""
Print the difference between two cnstat results. If JSON
option is True, return data in JSON format.
Expand All @@ -468,6 +508,16 @@ class Queuestat(object):
old_cntr = None
if key in cnstat_old_dict:
old_cntr = cnstat_old_dict.get(key)

qpktsps = qbytesps = qbitsps = STATUS_NA
rates = ratestat_dict.get(key, RateStats._make([STATUS_NA] * len(ratestat)))
if rates.qpktsps != STATUS_NA:
qpktsps = "{:,}".format(int(float(rates.qpktsps)))
if rates.qbytesps != STATUS_NA:
qbytesps = "{:,}".format(int(float(rates.qbytesps)))
if rates.qbitsps != STATUS_NA:
qbitsps = "{:,}".format(int(float(rates.qbitsps)))

if old_cntr is not None:
if self.voq:
if not non_zero or ns_diff(cntr['totalpacket'], old_cntr['totalpacket']) != '0' or \
Expand All @@ -480,7 +530,8 @@ class Queuestat(object):
ns_diff(cntr['totalbytes'], old_cntr['totalbytes']),
ns_diff(cntr['droppacket'], old_cntr['droppacket']),
ns_diff(cntr['dropbytes'], old_cntr['dropbytes']),
ns_diff(cntr['creditWDpkts'], old_cntr['creditWDpkts'])))
ns_diff(cntr['creditWDpkts'], old_cntr['creditWDpkts']),
qpktsps, qbytesps, qbitsps))
else:
queuetag = cntr['queuetype'] + str(cntr['queueindex'])

Expand All @@ -501,7 +552,8 @@ class Queuestat(object):
port, queuetag,
totalpacket, totalbytes,
droppacket, dropbytes,
trimpkt, trimsentpkt, trimdroppkt
trimpkt, trimsentpkt, trimdroppkt,
qpktsps, qbytesps, qbitsps
))
elif self.trim: # Packet Trimming related statistics
trimpkt = ns_diff(cntr['trimpkt'], old_cntr['trimpkt'])
Expand All @@ -512,7 +564,8 @@ class Queuestat(object):
trimpkt != '0' or trimsentpkt != '0' or trimdroppkt != '0':
table.append((
port, queuetag,
trimpkt, trimsentpkt, trimdroppkt
trimpkt, trimsentpkt, trimdroppkt,
qpktsps, qbytesps, qbitsps
))
else: # Generic statistics
totalpacket = ns_diff(cntr['totalpacket'], old_cntr['totalpacket'])
Expand All @@ -526,7 +579,8 @@ class Queuestat(object):
table.append((
port, queuetag,
totalpacket, totalbytes,
droppacket, dropbytes
droppacket, dropbytes,
qpktsps, qbytesps, qbitsps
))
if json_opt:
json_output[port].update(build_json(port, table, self.all, self.trim, self.voq))
Expand Down Expand Up @@ -569,16 +623,16 @@ class Queuestat(object):
cnstat_cached_dict = json.load(open(cnstat_fqn_file_name, 'r'))
if json_opt:
json_output[port].update({"cached_time":cnstat_cached_dict.get('time')})
json_output.update(self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero))
json_output.update(self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero,ratestat_dict))
else:
self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero)
self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero,ratestat_dict)
except IOError as e:
print(e.errno, e)
else:
if json_opt:
json_output.update(self.cnstat_print(port, cnstat_dict, json_opt, non_zero))
json_output.update(self.cnstat_print(port, cnstat_dict, json_opt, non_zero,ratestat_dict))
else:
self.cnstat_print(port, cnstat_dict, json_opt, non_zero)
self.cnstat_print(port, cnstat_dict, json_opt, non_zero,ratestat_dict)

if json_opt:
print(json_dump(json_output))
Expand All @@ -597,7 +651,7 @@ class Queuestat(object):
if self.voq and device_info.is_supervisor():
cnstat_dict = self.get_aggregate_port_stats(port)
else:
cnstat_dict = self.get_cnstat(self.port_queues_map[port])
cnstat_dict,ratestat_dict = self.get_cnstat(self.port_queues_map[port])
cache_ns = ''
if self.voq and self.namespace is not None:
cache_ns = '-' + self.namespace + '-'
Expand All @@ -609,17 +663,17 @@ class Queuestat(object):
cnstat_cached_dict = json.load(open(cnstat_fqn_file_name, 'r'))
if json_opt:
json_output[port].update({"cached_time":cnstat_cached_dict.get('time')})
json_output.update(self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero))
json_output.update(self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero,ratestat_dict))
else:
print(f"Last cached time{self.namespace_str} was " + str(cnstat_cached_dict.get('time')))
self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero)
self.cnstat_diff_print(port, cnstat_dict, cnstat_cached_dict, json_opt, non_zero,ratestat_dict)
except IOError as e:
print(e.errno, e)
else:
if json_opt:
json_output.update(self.cnstat_print(port, cnstat_dict, json_opt, non_zero))
json_output.update(self.cnstat_print(port, cnstat_dict, json_opt, non_zero,ratestat_dict))
else:
self.cnstat_print(port, cnstat_dict, json_opt, non_zero)
self.cnstat_print(port, cnstat_dict, json_opt, non_zero,ratestat_dict)

if json_opt:
print(json_dump(json_output))
Expand All @@ -630,7 +684,7 @@ class Queuestat(object):
if self.voq and self.namespace is not None:
cache_ns = '-' + self.namespace + '-'
for port in natsorted(self.counter_port_name_map):
cnstat_dict = self.get_cnstat(self.port_queues_map[port])
cnstat_dict,ratestat_dict = self.get_cnstat(self.port_queues_map[port])
try:
json.dump(cnstat_dict, open(cnstat_fqn_file + cache_ns + port, 'w'), default=json_serial)
except IOError as e:
Expand Down
48 changes: 48 additions & 0 deletions tests/netstat_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""Tests for utilities_common.netstat formatting using 1024-based units."""

import sys
from pathlib import Path
import importlib


# Try normal import first
try:
from utilities_common.netstat import (
format_brate,
format_util,
STATUS_NA,
)
except ModuleNotFoundError:
# Fallback: allow running this file directly by adding repo root to sys.path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
netstat = importlib.import_module("utilities_common.netstat")
format_brate = netstat.format_brate
format_util = netstat.format_util
STATUS_NA = netstat.STATUS_NA


def test_format_brate_uses_1024_units():
# > 10,000,000 bytes/s → MB path (now divides by 1024*1024)
assert format_brate(11 * 1024 * 1024) == "11.00 MB/s"

# > 10,000 bytes/s → KB path (now divides by 1024)
assert format_brate(20 * 1024) == "20.00 KB/s"

# <= 10,000 bytes/s → B path
assert format_brate(9_999) == "9999.00 B/s"


def test_format_util_uses_1024_conversion():
"""
util = brate / (port_rate * 1024 * 1024 / 8) * 100
Choose brate so utilization is exactly 50.00% at port_rate=1000 (Mb/s).
"""
port_rate_mbps = 1000
bytes_per_sec_at_line_rate = (port_rate_mbps * 1024 * 1024) / 8.0
brate = 0.5 * bytes_per_sec_at_line_rate
assert format_util(brate, port_rate_mbps) == "50.00%"


def test_format_util_status_na_passthrough():
assert format_util(STATUS_NA, 1000) == STATUS_NA
assert format_util(12345, STATUS_NA) == STATUS_NA
Loading
Loading