# Start
cd /ifs/data/Isilon_Support
screen -L
# health of the Clusters and nodes
Capacity:
isi status --quiet
isi storagepool list
isi status -p
CPU:
isi_for_array -s -X 'top -n –S 10'
Memory:
isi statistics query current --nodes all --degraded --stats node.memory.used
isi statistics query current --nodes all --degraded --stats node.memory.free
isi_for_array -s -X 'sysctl kern.malloc_pigs' | less
Load average:
isi statistics query current --nodes all --degraded --stats node.cpu.count
isi_for_array -s 'uptime'
disk operations across Balance across nodes
isi statistics query current --nodes=all --stats=node.disk.xfers.rate.sum
# SmartConnect and load balance
isi network interfaces list
isi status
isi_for_array -sX 'isi devices|grep -vi healthy'
# SmartConnect Suspend check
isi networks list pools --verbose
isi networks modify pool --sc-suspend-node=5 --name=172_28_30_x:Service_Net
isi networks modify pool --sc-resume-node=5 --name=172_28_30_x:Service_Net
# File Server Protocol Operational Rates
isi statistics protocol –-nodes all –-top –-orderby=timeavg
# Disk Hardware Latency
isi statistics drive –-nodes all –-top –-long –-orderby timeavg
# Node CPU Utilization
isi statistics system –-nodes --top
# isi statistics client
isi statistics client --orderby=ops -nall --top --numeric output
isi statistics client --orderby=ops -nall --top --numeric
isi statistics client --long -i 5
# isi statistics system
isi statistics system --timestamp --nodes -i 5
isi statistics system --nodes --interval=1 --top
# isi statistics query
isi statistics query --stats=node.disk.xfers.rate.5 -i 10
# SMB open files list
isi smb session list
isi_for_array -X 'isi smb openfiles list -v --format=csv --no-header --no-footer'
# isi statistics pstat (<--SMB)
isi statistics pstat --protocol smb2
# NFS locks list
isi_for_array -X isi nfs nlm locks list -v --format=csv --no-header
# Users by protocol
isi statistics client --protocols external --no-footer | awk '{print $1 " "$6 " " $8}' | head -22
# Busiest files and paths
isi statistics heat --nodes all --totalby path | awk '{print $1 " " $5}' | sort -n -r | head -20
isi statistics heat --nodes 1 | awk '{print $1 " " $4 " " $5}' | sort -n -r | head -20
# Blocked, contended, and deadlocked events
isi statistics heat --totalby event,lin,path --limit 50
# Disk activity
Time in Queue: isi statistics drive --nodes=all --degraded --no-header --no-footer | awk ' /SATA/{sum+=$8; max=0; min=1000} {if ($8>max) max=$8; if ($8<min) min=$8} END {print "Min =",min; print "Max = ",max; print "Average = ",sum/NR}'
Number in Queue: isi statistics drive --nodes=all --degraded --no-header --no-footer | awk ' /SATA/{sum+=$9; max=0; min=1000} {if ($9>max) max=$9; if ($9<min) min=$9} END {print "Min =",min; print "Max = ",max; print "Average = ",sum/NR}'
# Network factors
Latency and hops:
traceroute –q 5 <IP > Issue from client to a node
traceroute –q 5 <IP > Issue from node to another node
Latency and packet loss:
ping -i .1 -c 250 <IP > Issue from client to a node
ping -i .1 -c 250 <IP > Issue from node to another node
Bandwidth:
iperf -s On target node
iperf -c <IP of Target Host> On source host
Hostcache.list:
isi_for_array -X sysctl net.inet.tcp.hostcache.list
Cache response times and effectiveness:
isi_cbind show dns
isi_cbind show cluster
# Protocol operations
Protocol operations most used:
isi statistics protocol list --sort Ops --degraded
Protocol operations taking the most time
isi statistics protocol list --sort TimeAvg --degraded
Clients most-to-least demanding:
isi statistics client list --sort Ops --degraded
Connection distribution:
isi statistics query current --nodes all --degraded --stats node.clientstats.connected.nfs,node.clientstats.active.nfs
isi statistics query current --nodes all --degraded --stats node.clientstats.connected.smb,node.clientstats.active.smb2
Slow authentication:
isi statistics protocol list --protocols lsass_out --degraded
Protocol latency:
isi statistics protocol list --totalby class
# L3 Cache
isi_cache_stats
isi_cache_stats –c (*reset command)
isi_cache_stats –v
isi_for_array –s isi_cache_stats –v | grep –A3 l3
# SMB Config
isi_for_array sysctl isi.stats.client.smb2.max_clients
# Netstat
SMB: isi_for_array -s "netstat -an |grep 445 | grep -i ESTABLISHED |wc -l"
# tcpdump
tcpdump -i vlan0 -s 0 -w /ifs/data/nasssip30.pcap host 172.28.30.15
# IP move
isi networks support sc_put_ip --sc-iface=5:10gige-agg-1 --sc-ip=172.28.31.16
