Please provide a little more info, like the exact OS version, whether you use lsm, stipes, what connectivity, which standard tools you tried, and why they did not solve your problem.
The first standard tool is plain old 'iostat'. Give it a lit of devices and it will spit out IO rate and volume.
Personally I like 'monitor' for a quick overview.
And then there is 'collect' for longer term tiemline views.
For instant, specific, detailed, data I dig deep and use: '/sbin/hwmgr -get attribute'
All the gory counters are there for you to interpret. Hardest things to understand:
- the multiple fibr path having individual cell... but you want that.
- the transfer size in counted in 64 byte words (need 16384 to make a Megabyte).
I use a helper script for that which I will include below as well as attach.
Instead of measuring rate like to use 'dd' to enforced a rate & size and measure resulting timing.
Maybe your stripe chunksizes are too low? (lsm or storage). Maybe you need more alternative paths?
Hope this helps some,
Hein.
#!/bin/perl
#
# io_eval based on os_stat.
# os_stat, Hein van den Heuvel, July 1999.
# July-2000 clean up parsing.
#
# The module is either called directly by SAP benchrun for the evalute
# function or indirectly through the os_start or os_stop scripts.
# Look at argument 2 to find out what to do: start, stop, or eval if empty.
#
# start: an new temporary file with counters is created.
# stop: append new counter values to temporary file.
# eval: read temporary, calculate differences, print summaries, append input.
#
($function, $argument) = @ARGV;
$directory="";
$io_out="io.out";
$io_temp="io.tmp";
$append_or_create = ($function eq "start") ? "" : ">"; # create new files ?
if ($function =~ /^st/) { # both stop or start
# print "os_stat start-or-stop - $argument - $function\n" ;
system ("date $append_or_create> $directory$io_temp");
open (TMP,">> $directory$io_temp");
foreach $_ (`/sbin/hwmgr -get attribute`) {
if (/^ path_state_(\d+) = (\d+)/) {
$bus = int($1);
$state = int($2);
break;
}
if (/^ path_wds_(\d+) = (\d+)/) {
$b=int($1);
$wds=int($2);
if ($b==$bus) {
print TMP "b${b}w${wds}\n" if ($state==1);
} else {
print "found bus $b when expecting $bus for $dsk\n";
}
break;
}
if (/^ dev_base_name = dsk(\d+)/) {
$dsk = "dsk$1";
$state = 0;
print TMP "dsk$1\n";
break;
}
if (/^ write_bytes = (\d+)/) {
print TMP "r${read}w$1\n";
break;
}
if (/^ read_bytes = (\d+)/) {
$read = $1;
break;
}
}
exit 0;
}
if ($function ne "eval") {
print "Usage: os_stat [start | stop | eval] argument\n";
exit 0;
}
#--------------- This section when executing eval ----------------------.
sub sort_by_disk {
int($a) <=> int($b);
}
sub seconds {
@time=(0, split(/[:.]/,@_[0]), 0);
shift(@time) unless (@_[0]=~/\./);
(($time[0]*60) + $time[1])*60 + $time[2] + $time[3]/100;
}
sub mega {
my $x;
$x = @_[0];
if ( $x > 10000000000 ) {
$x = int($x/1000000000) . "G";
} elsif ( $x > 10000000 ) {
$x = int($x/1000000) . "M";
} elsif ( $x > 10000 ) {
$x = int($x/1000) . "K";
}
$x;
}
$d=-1;
$end=-1;
$maxwds=0;
# Loop through the input file.
unless (open (TMP,$directory . $io_temp)){
die "os_stat - failed to open $io_temp\n";
}
unless (open (IO,">".$io_out)){
die "os_stat - failed to open $io_out\n";
}
while () {
if ($d >= 0) {
if (/^b(\d+)w(\d+)/) {
$b = int($1);
$w = $2;
$maxbus = $b if ($b > $maxbus);
if ($end) {
$x = 0.0 + $w - $wds[$d][$b];
$maxwds = $x if ($x > $maxwds);
$wds[$d][$b] = $x;
$bus[$b] += $x;
$maxwdsbus = $bus[$b] if ($bus[$b] > $maxwdsbus);
} else {
$wds[$d][$b] = $w;
}
break;
}
}
if (/^dsk(\d+)/) {
$dsk = $1;
$d = int($dsk);
break;
}
if (/^r(\d+)w(\d+)/) {
if ($end) {
$read{$dsk} += $1;
$write{$dsk}+= $2;
} else {
$read{$dsk} = -$1;
$write{$dsk}= -$2;
}
$d = -1;
}
# Recognise (re)start of IO output from date and time. Sample input:
#Mon Jul 26 12:41:52 MET DST 1999
if (/^\w{3} (\w{3}\s+\d+) (\d+:\d+:\d+).*( \d{4})/) {
$elapsed[++$end]=seconds($2); # Time saved in regular expresion.
if ($end) {
$interval = $elapsed[1] -$elapsed[0];
$interval += 86400 if ($interval < 0);
$procent = $cpu * $interval / 100;
print IO "\n$1 $3 ";
}
}
}
if (($maxwds < 1000) || ($interval < 1)) {
print "Not enough data. No done?\n";
exit 1;
}
$u = 16384 * $interval;
$u = $maxwds / 32;
$maxbus++;
#x print "maxbus = $maxbus, maxwds = $maxwds.\n";
$h1 = sprintf ("%-16s BUS...", $interval . " seconds.");
$h2 = " ";
$h3 = "disk MB/sec Rd%";
$h4 = "----------------";
for ($i = 0; $i < $maxbus; $i++) {
$h2 .= " " . int($i/10);
$h3 .= " " . $i%10;
$h4 .= "--";
}
$code=" .--====++++++++****************################################";
for ($i=0; $i<6; $i++) {
$l1 .= substr ($code,2**$i,1) ;
$l2 .= sprintf ("%4.2f ", ($u * 2**$i / $interval ) / 16384);
}
print IO " BUS MB/sec legend: $l1 = $l2\n$h1\n$h2\n$h3\n$h4\n";
foreach $dsk (sort sort_by_disk keys %read) {
$sum = $read{$dsk} + $write{$dsk};
$total += $sum;
$d = int($dsk);
if ($sum > 0) {
$line = sprintf ("dsk%-3d%6.2f%4d ", $d,
(($sum/(1024*1024)) / $interval), int(100*($read{$dsk}/$sum)) );
for ($b=0; $b<$maxbus; $b++) {
$line .= substr($code,$wds[$d][$b]/$u, 1) . " ";
}
print IO $line, "\n";
}
}
$u = $maxwdsbus / 32;
$line = $h4 . sprintf ("\nTOTAL%7.2f ", ($total/(1024*1024)) / $interval);
for ($b=0; $b<$maxbus; $b++) {
$line .= substr($code,$bus[$b]/$u, 1) . " ";
}
print IO $line, "\n\nBUS MB/Sec\n------------\n";
for ($b=0; $b<$maxbus; $b++) {
printf IO ("%-3d%9.2f\n", $b, ($bus[$b]/16384)/$interval);
}
#unless (unlink ($io_temp, $net_temp)){
# die "os_stat - failed to unlink $io_temp or $net_temp\n";
