[ Isilon ] PERFORMANCE METRICS HINTS

# Start
 cd /ifs/data/Isilon_Support
 screen -L

# health of the Clusters and nodes
 Capacity:
  isi status --quiet
  isi storagepool list
  isi status -p

 CPU:
  isi_for_array -s -X 'top -n –S 10'

 Memory:
  isi statistics query current --nodes all --degraded --stats node.memory.used
  isi statistics query current --nodes all --degraded --stats node.memory.free
  isi_for_array -s -X 'sysctl kern.malloc_pigs' | less

 Load average:
  isi statistics query current --nodes all --degraded --stats node.cpu.count
  isi_for_array -s 'uptime'

 disk operations across Balance across nodes
  isi statistics query current --nodes=all --stats=node.disk.xfers.rate.sum

# SmartConnect and load balance
 isi network interfaces list
 isi status

isi_for_array -sX 'isi devices|grep -vi healthy'

# SmartConnect Suspend check
 isi networks list pools --verbose
 isi networks modify pool --sc-suspend-node=5 --name=172_28_30_x:Service_Net
 isi networks modify pool --sc-resume-node=5 --name=172_28_30_x:Service_Net

# File Server Protocol Operational Rates
isi statistics protocol –-nodes all –-top –-orderby=timeavg

# Disk Hardware Latency
isi statistics drive –-nodes all –-top –-long –-orderby timeavg

# Node CPU Utilization
isi statistics system –-nodes --top

# isi statistics client
 isi statistics client --orderby=ops -nall --top --numeric output
 isi statistics client --orderby=ops -nall --top --numeric
 isi statistics client --long -i 5


# isi statistics system
 isi statistics system --timestamp --nodes -i 5
 isi statistics system --nodes --interval=1 --top

# isi statistics query
 isi statistics query --stats=node.disk.xfers.rate.5 -i 10

# SMB open files list
 isi smb session list
 isi_for_array -X 'isi smb openfiles list -v --format=csv --no-header --no-footer'

 # isi statistics pstat (<--SMB)
 isi statistics pstat --protocol smb2

# NFS locks list
 isi_for_array -X isi nfs nlm locks list -v --format=csv --no-header

# Users by protocol
isi statistics client --protocols external --no-footer | awk '{print $1 " "$6 " " $8}' | head -22

# Busiest files and paths
 isi statistics heat --nodes all --totalby path | awk '{print $1 " " $5}' | sort -n -r | head -20
 isi statistics heat --nodes 1 | awk '{print $1 " " $4 " " $5}' | sort -n -r | head -20

# Blocked, contended, and deadlocked events
 isi statistics heat --totalby event,lin,path --limit 50

# Disk activity
Time in Queue: isi statistics drive --nodes=all --degraded --no-header --no-footer | awk ' /SATA/{sum+=$8; max=0; min=1000} {if ($8>max) max=$8; if ($8<min) min=$8} END {print "Min =",min; print "Max = ",max; print "Average = ",sum/NR}'
Number in Queue: isi statistics drive --nodes=all --degraded --no-header --no-footer | awk ' /SATA/{sum+=$9; max=0; min=1000} {if ($9>max) max=$9; if ($9<min) min=$9} END {print "Min =",min; print "Max = ",max; print "Average = ",sum/NR}'

# Network factors

Latency and hops:
 traceroute –q 5 <IP > Issue from client to a node
 traceroute –q 5 <IP > Issue from node to another node

 Latency and packet loss:
  ping -i .1 -c 250 <IP > Issue from client to a node
  ping -i .1 -c 250 <IP > Issue from node to another node

 Bandwidth:
  iperf -s On target node
  iperf -c <IP of Target Host> On source host

 Hostcache.list:
 isi_for_array -X sysctl net.inet.tcp.hostcache.list

 Cache response times and effectiveness:
  isi_cbind show dns
  isi_cbind show cluster


# Protocol operations
 Protocol operations most used:
  isi statistics protocol list --sort Ops --degraded

 Protocol operations taking the most time
  isi statistics protocol list --sort TimeAvg --degraded

 Clients most-to-least demanding:
  isi statistics client list --sort Ops --degraded

 Connection distribution:
  isi statistics query current --nodes all --degraded --stats node.clientstats.connected.nfs,node.clientstats.active.nfs
  isi statistics query current --nodes all --degraded --stats node.clientstats.connected.smb,node.clientstats.active.smb2

 Slow authentication:
  isi statistics protocol list --protocols lsass_out --degraded

 Protocol latency:
  isi statistics protocol list --totalby class






# L3 Cache
 isi_cache_stats
 isi_cache_stats –c  (*reset command)
 isi_cache_stats –v
 isi_for_array –s isi_cache_stats –v | grep –A3 l3

# SMB Config
 isi_for_array sysctl isi.stats.client.smb2.max_clients

# Netstat
SMB: isi_for_array -s "netstat -an |grep 445 | grep -i ESTABLISHED |wc -l"

# tcpdump
 tcpdump -i vlan0 -s 0 -w /ifs/data/nasssip30.pcap host 172.28.30.15

# IP move
 isi networks support sc_put_ip --sc-iface=5:10gige-agg-1 --sc-ip=172.28.31.16