iostatparse.py

#!/usr/local/bin/python3
#
# iostatparse.py
#
# Usage: iostatparse.py [-h] file [file ...]
#
# Parse the output of zpool iostat to generate summarized performance statistics
#
# Positional arguments:
#  file 		Text file(s) containing the zpool iostat output to parse
#
# Details:
# Outputs 50th, 90th, 95th, 99th percentile, max value, avg, and stddev for performance statistics generated by
# zpool iostat command. Performance statistics include R/W/Combined IOPS and throughput, I/O size, total R/W latency,
# and disk R/W latency. Also calculates total time of zpool iostat run, R/W mix, total data written across iostat
# interval, and average total written per day.
#
# Copyright 2021 Jason Rose <jason@jro.io>
# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
# following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
#    disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
#    following disclaimer in the documentation and/or other materials provided with the distribution
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import argparse
import re
import math
import statistics

def IOStatParse(files):
	# Track line count
	ln = 0

	# Initialize arrays for storing each type of data
	read_iops_raw = []
	write_iops_raw = []
	both_iops_raw = []
	read_tp_raw = []
	write_tp_raw = []
	both_tp_raw = []
	read_io_size = []
	write_io_size = []

	# Initialize latency arrays
	read_tot_wait_raw = []
	write_tot_wait_raw = []
	read_disk_wait_raw = []
	write_disk_wait_raw = []

	# We will attempt to pull latency values from file if they exist. If they don't exist, this flag
	# gets disabled, we stop trying to track latency, and latency reporting gets disabled
	track_latency = True

	# Initialize array to hold all the raw data
	iostat = []

	# RegExp for removing header rows
	h1 = '\s+capacity\s+operations\s+bandwidth'
	h2 = 'pool\s+alloc\s+free'

	# Remove empty lines and header lines from output file, transfer file to array
	for file in files:
		f = open(file,"r")
		for line in f:
			if (not line.isspace()) and ("---" not in line) and not re.search(h1,line) and not re.search(h2,line):
				iostat.append(line)

		# Close iostat file, all data is in array
		f.close()

	# Walk through all data in output array
	for line in iostat:
		#  Split the data
		data = line.split()

		# Pull out read ops data. Multiply by 1000 if "K" in value
		read = data[3]
		if "K" in read:
			read = int(float(read[:-1])*1000)
		read_iops_raw.append(int(read))

		# Pull out write ops data. Multiply by 1000 if "K" in value
		write = data[4]
		if "K" in write:
			write = int(float(write[:-1])*1000)
		write_iops_raw.append(int(write))

		# Add read and write value to give R+W value
		both_iops_raw.append(int(read) + int(write))

		# Pull out read TP data. Multiply by 1024 if "K", 1024^2 if "M", 1024^3 if "G" in value
		readtp = data[5]
		if "G" in readtp:
			readtp = int(float(readtp[:-1])*1024**3)
		elif "M" in readtp:
			readtp = int(float(readtp[:-1])*1024**2)
		elif "K" in readtp:
			readtp = int(float(readtp[:-1])*1024)
		read_tp_raw.append(int(readtp))

		# Pull out write TP data. Multiply by 1024 if "K", 1024^2 if "M", 1024^3 if "G" in value
		writetp = data[6]
		if "G" in writetp:
			writetp = int(float(writetp[:-1])*1024**3)
		elif "M" in writetp:
			writetp = int(float(writetp[:-1])*1024**2)
		elif "K" in writetp:
			writetp = int(float(writetp[:-1])*1024)
		write_tp_raw.append(int(writetp))

		# Add read and write value to give R+W value
		both_tp_raw.append(int(readtp) + int(writetp))

		# Determine avg IO size (throughput/num. ops)
		if int(read) != 0:
			read_io_size.append(int(readtp)/int(read))

		if int(write) != 0:
			write_io_size.append(int(writetp)/int(write))

		# Gather latency data if -l flag is passed
		if track_latency:
			try:
				# Pull total read wait, convert to nSec
				read_tot_wait = data[7]
				if "-" not in read_tot_wait:
					if "ms" in read_tot_wait:
						read_tot_wait = int(read_tot_wait[:-2])*1000000
					elif "us" in read_tot_wait:
						read_tot_wait = int(read_tot_wait[:-2])*1000
					elif "ns" in read_tot_wait:
						read_tot_wait = int(read_tot_wait[:-2])
					else:
						read_tot_wait = int(read_tot_wait)
					read_tot_wait_raw.append(read_tot_wait)

				# Pull total write wait, convert to nSec
				write_tot_wait = data[8]
				if "-" not in write_tot_wait:
					if "ms" in write_tot_wait:
						write_tot_wait = int(write_tot_wait[:-2])*1000000
					elif "us" in write_tot_wait:
						write_tot_wait = int(write_tot_wait[:-2])*1000
					elif "ns" in write_tot_wait:
						write_tot_wait = int(write_tot_wait[:-2])
					else:
						write_tot_wait = int(write_tot_wait)
					write_tot_wait_raw.append(write_tot_wait)

				# Pull disk read wait, convert to nSec
				read_disk_wait = data[8]
				if "-" not in read_disk_wait:
					if "ms" in read_disk_wait:
						read_disk_wait = int(read_disk_wait[:-2])*1000000
					elif "us" in read_disk_wait:
						read_disk_wait = int(read_disk_wait[:-2])*1000
					elif "ns" in read_disk_wait:
						read_disk_wait = int(read_disk_wait[:-2])
					else:
						read_disk_wait = int(read_disk_wait)
					read_disk_wait_raw.append(read_disk_wait)

				# Pull disk write wait, convert to nSec
				write_disk_wait = data[9]
				if "-" not in write_disk_wait:
					if "ms" in write_disk_wait:
						write_disk_wait = int(write_disk_wait[:-2])*1000000
					elif "us" in write_disk_wait:
						write_disk_wait = int(write_disk_wait[:-2])*1000
					elif "ns" in write_disk_wait:
						write_disk_wait = int(write_disk_wait[:-2])
					else:
						write_disk_wait = int(write_disk_wait)
					write_disk_wait_raw.append(write_disk_wait)

			# If -l was passed but the extra latency columns are not present, bail on the
			# rest of the latency tracking/stats calculations
			except:
				track_latency = False

		# Increment line number
		ln+=1

		# Print parse status every 100K lines (longer files can take a while to parse)
		if (ln) % 100000 == 0:
			print("   Parsing Line " + format(ln,",d") + "...")

	# Percentile function requires data lists to be sorted
	read_iops_raw = sorted(read_iops_raw)
	write_iops_raw = sorted(write_iops_raw)
	both_iops_raw = sorted(both_iops_raw)
	read_tp_raw = sorted(read_tp_raw)
	write_tp_raw = sorted(write_tp_raw)
	both_tp_raw = sorted(both_tp_raw)
	read_io_size = sorted(read_io_size)
	write_io_size = sorted(write_io_size)

	# Sort latency values if we're
	if track_latency:
		read_tot_wait_raw = sorted(read_tot_wait_raw)
		write_tot_wait_raw = sorted(write_tot_wait_raw)
		read_disk_wait_raw = sorted(read_disk_wait_raw)
		write_disk_wait_raw = sorted(write_disk_wait_raw)

	# Calculate 50P, 90P, 95P, 99P max, avg, stdev for all states, put them in an array along with labels for printing
	try:
		ReadIOPS_stdev = str(round(statistics.stdev(read_iops_raw),2))
	except:
		ReadIOPS_stdev = "-"
	ReadIOPS = ["Read ","IOPS","Ops/Sec",
		str(int(percentile(read_iops_raw,.50))),
		str(int(percentile(read_iops_raw,.90))),
		str(int(percentile(read_iops_raw,.95))),
		str(int(percentile(read_iops_raw,.99))),
		str(max(read_iops_raw)),
		str(round(avg(read_iops_raw),2)),
		ReadIOPS_stdev]
	try:
		WriteIOPS_stdev = str(round(statistics.stdev(write_iops_raw),2))
	except:
		WriteIOPS_stdev = "-"
	WriteIOPS = ["Write ","IOPS","Ops/Sec",
		str(int(percentile(write_iops_raw,.50))),
		str(int(percentile(write_iops_raw,.90))),
		str(int(percentile(write_iops_raw,.95))),
		str(int(percentile(write_iops_raw,.99))),
		str(max(write_iops_raw)),
		str(round(avg(write_iops_raw),2)),
		WriteIOPS_stdev]
	try:
		BothIOPS_stdev = str(round(statistics.stdev(both_iops_raw),2))
	except:
		BothIOPS_stdev = "-"
	BothIOPS = ["R+W ","IOPS","Ops/Sec",
		str(int(percentile(both_iops_raw,.50))),
		str(int(percentile(both_iops_raw,.90))),
		str(int(percentile(both_iops_raw,.95))),
		str(int(percentile(both_iops_raw,.99))),
		str(max(both_iops_raw)),
		str(round(avg(both_iops_raw),2)),
		BothIOPS_stdev]

	# Multiply all throughput values by 1024^2
	try:
		ReadTP_stdev = str(round(statistics.stdev(read_tp_raw),2))
	except:
		ReadTP_stdev = "-"
	ReadTP = ["Read ","Throughput","MiB/sec",
		str(round(int(percentile(read_tp_raw,.50))/1024**2,2)),
		str(round(int(percentile(read_tp_raw,.90))/1024**2,2)),
		str(round(int(percentile(read_tp_raw,.95))/1024**2,2)),
		str(round(int(percentile(read_tp_raw,.99))/1024**2,2)),
		str(round(max(read_tp_raw)/1024**2,2)),
		str(round(avg(read_tp_raw)/1024**2,2)),
		ReadTP_stdev]
	try:
		WriteTP_stdev = str(round(statistics.stdev(write_tp_raw),2))
	except:
		WriteTP_stdev = "-"
	WriteTP = ["Write ","Throughput","MiB/sec",
		str(round(int(percentile(write_tp_raw,.50))/1024**2,2)),
		str(round(int(percentile(write_tp_raw,.90))/1024**2,2)),
		str(round(int(percentile(write_tp_raw,.95))/1024**2,2)),
		str(round(int(percentile(write_tp_raw,.99))/1024**2,2)),
		str(round(max(write_tp_raw)/1024**2,2)),
		str(round(avg(write_tp_raw)/1024**2,2)),
		WriteTP_stdev]
	try:
		BothTP_stdev = str(round(statistics.stdev(both_tp_raw),2))
	except:
		BothTP_stdev = "-"
	BothTP = ["R+W ","Throughput","MiB/sec",
		str(round(int(percentile(both_tp_raw,.50))/1024**2,2)),
		str(round(int(percentile(both_tp_raw,.90))/1024**2,2)),
		str(round(int(percentile(both_tp_raw,.95))/1024**2,2)),
		str(round(int(percentile(both_tp_raw,.99))/1024**2,2)),
		str(round(max(both_tp_raw)/1024**2,2)),
		str(round(avg(both_tp_raw)/1024**2,2)),
		BothTP_stdev]

	# Multiply all IO size values by 1024 to convert to KiB
	try:
		ReadIOSize_stdev = str(round(statistics.stdev(read_io_size_raw),2))
	except:
		ReadIOSize_stdev = "-"
	ReadIOSize =["Read ","IO Size","KiB",
		str(round(int(percentile(read_io_size,.50))/1024,2)),
		str(round(int(percentile(read_io_size,.90))/1024,2)),
		str(round(int(percentile(read_io_size,.95))/1024,2)),
		str(round(int(percentile(read_io_size,.99))/1024,2)),
		str(round(max(read_io_size)/1024,2)),
		str(round(avg(read_io_size)/1024,2)),
		ReadIOSize_stdev]
	try:
		WriteIOSize_stdev = str(round(statistics.stdev(write_io_size_raw),2))
	except:
		WriteIOSize_stdev = "-"
	WriteIOSize = ["Write ","IO Size","KiB",
		str(round(int(percentile(write_io_size,.50))/1024,2)),
		str(round(int(percentile(write_io_size,.90))/1024,2)),
		str(round(int(percentile(write_io_size,.95))/1024,2)),
		str(round(int(percentile(write_io_size,.99))/1024,2)),
		str(round(max(write_io_size)/1024,2)),
		str(round(avg(write_io_size)/1024,2)),
		WriteIOSize_stdev]

	# All latency values are in nSec, divide by 1M to convert to mS
	if track_latency:
		try:
			ReadTotWait_stdev = str(round(statistics.stdev(read_tot_wait_raw),2))
		except:
			ReadTotWait_stdev = "-"
		ReadTotWait = ["Read ","Tot. Lat.","mSec",
			str(round(percentile(read_tot_wait_raw,.50)/1000000,2)),
			str(round(percentile(read_tot_wait_raw,.90)/1000000,2)),
			str(round(percentile(read_tot_wait_raw,.95)/1000000,2)),
			str(round(percentile(read_tot_wait_raw,.99)/1000000,2)),
			str(round(max(read_tot_wait_raw)/1000000,2)),
			str(round(avg(read_tot_wait_raw)/1000000,2)),
			ReadTotWait_stdev]
		try:
			WriteTotWait_stdev = str(round(statistics.stdev(write_tot_wait_raw),2))
		except:
			WriteTotWait_stdev = "-"
		WriteTotWait = ["Write ","Tot. Lat.","mSec",
			str(round(percentile(write_tot_wait_raw,.50)/1000000,2)),
			str(round(percentile(write_tot_wait_raw,.90)/1000000,2)),
			str(round(percentile(write_tot_wait_raw,.95)/1000000,2)),
			str(round(percentile(write_tot_wait_raw,.99)/1000000,2)),
			str(round(max(write_tot_wait_raw)/1000000,2)),
			str(round(avg(write_tot_wait_raw)/1000000,2)),
			WriteTotWait_stdev]

		try:
			ReadDiskWait_stdev = str(round(statistics.stdev(read_disk_wait_raw),2))
		except:
			ReadDiskWait_stdev = "-"
		ReadDiskWait = ["Read ","Disk Lat.","mSec",
			str(round(percentile(read_disk_wait_raw,.50)/1000000,2)),
			str(round(percentile(read_disk_wait_raw,.90)/1000000,2)),
			str(round(percentile(read_disk_wait_raw,.95)/1000000,2)),
			str(round(percentile(read_disk_wait_raw,.99)/1000000,2)),
			str(round(max(read_disk_wait_raw)/1000000,2)),
			str(round(avg(read_disk_wait_raw)/1000000,2)),
			ReadDiskWait_stdev]
		try:
			WriteDiskWait_stdev = str(round(statistics.stdev(write_disk_wait_raw),2))
		except:
			WriteDiskWait_stdev = "-"
		WriteDiskWait = ["Write ","Disk Lat.","mSec",
			str(round(percentile(write_disk_wait_raw,.50)/1000000,2)),
			str(round(percentile(write_disk_wait_raw,.90)/1000000,2)),
			str(round(percentile(write_disk_wait_raw,.95)/1000000,2)),
			str(round(percentile(write_disk_wait_raw,.99)/1000000,2)),
			str(round(max(write_disk_wait_raw)/1000000,2)),
			str(round(avg(write_disk_wait_raw)/1000000,2)),
			WriteDiskWait_stdev]

	# Formatting for stats table
	headers = ['','',' Unit',' 50p',' 90p',' 95p',' 99p',' Max',' Avg','  σ']
	h2 = [''] * 2 + ['------'] + ['-----'] * 7
	row_format = "{:>7}" + "{:<12}" + "{:<12}" + "{:<15}" * 7

	# Print the stats table
	print()
	print(row_format.format(*headers))
	print(row_format.format(*h2))
	print(row_format.format(*ReadIOPS))
	print(row_format.format(*WriteIOPS))
	print(row_format.format(*BothIOPS))
	print(row_format.format(*ReadTP))
	print(row_format.format(*WriteTP))
	print(row_format.format(*BothTP))
	print(row_format.format(*ReadIOSize))
	print(row_format.format(*WriteIOSize))

	# Print latency stats if l flag is passed
	if track_latency:
		print(row_format.format(*ReadTotWait))
		print(row_format.format(*WriteTotWait))
		print(row_format.format(*ReadDiskWait))
		print(row_format.format(*WriteDiskWait))

	# Calculate and print other misc. stats
	tot_io = sum(read_iops_raw) + sum(write_iops_raw)
	tot_tp = sum(read_tp_raw) + sum(write_tp_raw)
	print()
	print("   Total Samples Parsed: " + format(ln,",d") + " (" + conv_time(ln) + ")")
	print("   Read Mix (by Op Count): " + str(int(sum(read_iops_raw)/tot_io*100)) + " %")
	print("   Read Mix (by Size): " + str(int(sum(read_tp_raw)/tot_tp*100)) + " %")
	print("   Total Written: " + str(round(sum(write_tp_raw)/(1024**4),2)) + " TiB")
	print("   Total Written per Day: " + str(round(sum(write_tp_raw)/(1024**4)/((ln)/86400),2)) + " TiB\n")

def percentile(N, percent, key=lambda x:x):
	# Find the percentile of a list of values
 	# N is a list of values (N must already sorted)
	# percent is a float value from 0.0 to 1.0
	# parameter key - optional key function to compute value from each element of N

	if not N:
		return None
	k = (len(N)-1) * percent
	f = math.floor(k)
	c = math.ceil(k)
	if f == c:
		return key(N[int(k)])
	d0 = key(N[int(f)]) * (c-k)
	d1 = key(N[int(c)]) * (k-f)
	return d0+d1

def avg(N):
	# Return average value of a list N
	return sum(N) / len(N)

def conv_time(sec):
	# Convert seconds
	[d,h,m,s] = [0,0,0,0]
	m, s = divmod(sec, 60)
	h, m = divmod(m, 60)
	d, h = divmod(h, 24)
	# Format seconds output with two decimal places so it aligns with row above
	if d != 0:
		t = str(round(d)) + "d " + "{:02}".format(round(h)) + "h " + "{:02}".format(round(m)) + "m " \
			+ "{:02}".format(s) + "s"
	elif h != 0:
		t = str(round(h)) + "h " + "{:02}".format(round(m)) + "m " + "{:02}".format(s) + "s"
	elif m != 0:
		t = str(round(m)) + "m " + "{:02}".format(s) + "s"
	else:
		t = "{:2}".format(s) + "s"
	return t

if __name__ == '__main__':
	parser = argparse.ArgumentParser(
		description="Parse the output of zpool iostat to generate summarized performance statistics")
	parser.add_argument("file", nargs="+",help="Text file(s) containing the zpool iostat output to parse")
	args = parser.parse_args()

	IOStatParse(args.file)