I/O tuning for linux san storage

[Robert_Ngo]

i have a RHEL server that have very heavy ftp tranffic at certain period of time during the day. this is causing us not able to access filesystem on SAN storage, the ftp filesystem is on SAN storage.

the system is a DL 580 G3 that are connected to HP 1024 SAN storage and SAN disk is manage by vxvm, what are the kernel tuning we can do to improve the performance of I/O?

sunrpc.nlm_debug = 0
sunrpc.nfsd_debug = 0
sunrpc.nfs_debug = 0
sunrpc.rpc_debug = 0
vxvm.vxinfo.volkio_commitwaitq = volkioq_commitwait_start count: 0
vxvm.vxinfo.volkio_commitwaitq = volkioq_commitwait_start:
vxvm.vxinfo.volkioqueue = volkioq_start count: 0
vxvm.vxinfo.volkioqueue = volkioq_start:
vxvm.vxinfo.volsio_otherq = volsioq_done count: 0
vxvm.vxinfo.volsio_otherq = volsioq_done:
vxvm.vxinfo.volsio_otherq = volsioq_restart count: 0
vxvm.vxinfo.volsio_otherq = volsioq_restart:
vxvm.vxinfo.volsio_otherq = volsioq_redone count: 0
vxvm.vxinfo.volsio_otherq = volsioq_redone:
vxvm.vxinfo.volsio_idleq = volsioq_idle_start count: 0
vxvm.vxinfo.volsio_idleq = volsioq_idle_start:
vxvm.vxinfo.volsioqueue = volsioq_start count: 0
vxvm.vxinfo.volsioqueue = volsioq_start:
vxvm.vxinfo.volktrans = vol_config_loaded: 1
vxvm.vxinfo.volktrans = vol_commit_in_progress: 0
vxvm.vxinfo.volktrans = vol_ktrans_locked: 0
vxvm.vxinfo.volktrans = vol_ktrans_pending_abort: 0
vxvm.vxinfo.volktrans = vol_ktrans_abort_is_possible: 0
vxvm.vxinfo.volktrans = transaction state: last transaction committed
vxvm.vxinfo.voliod_info = voliod_running: 0
vxvm.vxinfo.voliod_info = voliod_need: 0
vxvm.vxinfo.voliod_info = voliod_count: 10
vxvm.vxinfo.voliod_info = voliod_want_count: 10
vxvm.vxinfo.voliod_info = volkio_overrun: 0
vxvm.vxinfo.voliomem = memory reservation pool usage
vxvm.vxinfo.voliomem = id: max mem used: total allocated mem: mem in use: min mem: max mem
vxvm.vxinfo.voliomem = 0 0 20000 0 20000 400000
vxvm.vxinfo.volkiocount_locked = volkiocount: 0
vxvm.vxinfo.volkiocount = volkiocount: 0
vxvm.vxio.voldrl_max_seq_dirty = 3
vxvm.vxio.vol_max_volumes = 3824
vxvm.vxio.vol_rvio_maxpool_sz = 67108864
vxvm.vxio.vol_vvr_use_nat = 0
vxvm.vxio.vol_max_wrspool_sz = 16777216
vxvm.vxio.vol_nm_hb_timeout = 10
vxvm.vxio.vol_dcm_replay_size = 262144
vxvm.vxio.volraid_minpool_size = 65536
vxvm.vxio.volcvm_smartsync = 1
vxvm.vxio.vol_min_lowmem_sz = 4194304
vxvm.vxio.vxvm_read_ahead = 64
vxvm.vxio.vxvm_chk_blocksize = 199 0 1024
vxvm.vxio.vxvm_set_blocksize = 199 255 2147483647
vxvm.vxio.vol_max_nmpool_sz = 16777216
vxvm.vxio.vol_max_rdback_sz = 67108864
vxvm.vxio.volpagemod_max_memsz = 1536
vxvm.vxio.vol_fmr_logsz = 4
vxvm.vxio.voliomem_maxpool_sz = 4194304
vxvm.vxio.voliomem_chunk_size = 32768
vxvm.vxio.vol_max_nlogs = 20
vxvm.vxio.vol_max_nconfigs = 20
vxvm.vxio.vxtask_max_monitors = 32
vxvm.vxio.vol_checkpt_default = 20480
vxvm.vxio.voliot_max_open = 32
vxvm.vxio.voliot_errbuf_dflt = 16384
vxvm.vxio.voliot_iobuf_default = 8192
vxvm.vxio.voliot_iobuf_max = 65536
vxvm.vxio.voliot_iobuf_limit = 131072
vxvm.vxio.vol_maxstablebufsize = 256
vxvm.vxio.vol_mvr_maxround = 512
vxvm.vxio.vol_maxparallelio = 256
vxvm.vxio.voldrl_max_drtregs = 2048
vxvm.vxio.voldrl_min_regionsz = 1024
vxvm.vxio.vol_default_iodelay = 50
vxvm.vxio.maxiosize = 256
vxvm.vxio.vol_maxioctl = 32768
vxvm.vxio.vol_maxkiocount = 12000
vxdmp.dmp_retry_count = 5
vxdmp.dmp_restore_daemon_cycles = 10
vxdmp.dmp_restore_daemon_policy = 1
vxdmp.dmp_restore_daemon_interval = 300
vxdmp.dmp_enable_restore_daemon = 2
vxdmp.dmp_pathswitch_blks_shift = 11
abi.fake_utsname = 0
abi.trace = 0
abi.defhandler_libcso = 68157441
abi.defhandler_lcall7 = 68157441
abi.defhandler_elf = 0
abi.defhandler_coff = 117440515
dev.cdrom.check_media = 0
dev.cdrom.lock = 1
dev.cdrom.debug = 0
dev.cdrom.autoeject = 0
dev.cdrom.autoclose = 1
dev.cdrom.info = CD-ROM information, Id: cdrom.c 3.12 2000/10/18
dev.cdrom.info =
dev.cdrom.info = drive name: hdb
dev.cdrom.info = drive speed: 24
dev.cdrom.info = drive # of slots: 1
dev.cdrom.info = Can close tray: 1
dev.cdrom.info = Can open tray: 1
dev.cdrom.info = Can lock tray: 1
dev.cdrom.info = Can change speed: 1
dev.cdrom.info = Can select disk: 0
dev.cdrom.info = Can read multisession: 1
dev.cdrom.info = Can read MCN: 1
dev.cdrom.info = Reports media changed: 1
dev.cdrom.info = Can play audio: 1
dev.cdrom.info = Can write CD-R: 0
dev.cdrom.info = Can write CD-RW: 0
dev.cdrom.info = Can read DVD: 1
dev.cdrom.info = Can write DVD-R: 0
dev.cdrom.info = Can write DVD-RAM: 0
dev.cdrom.info =
dev.cdrom.info =
dev.audit.debug = 0
dev.audit.paranoia = 0
dev.audit.max-messages = 1024
dev.audit.allow-suspend = 1
dev.audit.attach-all = 0
dev.raid.speed_limit_max = 10000
dev.raid.speed_limit_min = 100
dev.rtc.max-user-freq = 64
debug.rpmarch =
debug.kerneltype =
net.unix.max_dgram_qlen = 10
net.token-ring.rif_timeout = 60000
net.ipv4.conf.eth0.disable_policy = 0
net.ipv4.conf.eth0.disable_xfrm = 0
net.ipv4.conf.eth0.arp_filter = 0
net.ipv4.conf.eth0.tag = 0
net.ipv4.conf.eth0.log_martians = 0
net.ipv4.conf.eth0.bootp_relay = 0
net.ipv4.conf.eth0.medium_id = 0
net.ipv4.conf.eth0.proxy_arp = 0
net.ipv4.conf.eth0.accept_source_route = 1
net.ipv4.conf.eth0.send_redirects = 1
net.ipv4.conf.eth0.rp_filter = 1
net.ipv4.conf.eth0.shared_media = 1
net.ipv4.conf.eth0.secure_redirects = 1
net.ipv4.conf.eth0.accept_redirects = 1
net.ipv4.conf.eth0.mc_forwarding = 0
net.ipv4.conf.eth0.forwarding = 0
net.ipv4.conf.lo.disable_policy = 0
net.ipv4.conf.lo.disable_xfrm = 0
net.ipv4.conf.lo.arp_filter = 0
net.ipv4.conf.lo.tag = 0
net.ipv4.conf.lo.log_martians = 0
net.ipv4.conf.lo.bootp_relay = 0
net.ipv4.conf.lo.medium_id = 0
net.ipv4.conf.lo.proxy_arp = 0
net.ipv4.conf.lo.accept_source_route = 1
net.ipv4.conf.lo.send_redirects = 1
net.ipv4.conf.lo.rp_filter = 1
net.ipv4.conf.lo.shared_media = 1
net.ipv4.conf.lo.secure_redirects = 1
net.ipv4.conf.lo.accept_redirects = 1
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.lo.forwarding = 0
net.ipv4.conf.default.disable_policy = 0
net.ipv4.conf.default.disable_xfrm = 0
net.ipv4.conf.default.arp_filter = 0
net.ipv4.conf.default.tag = 0
net.ipv4.conf.default.log_martians = 0
net.ipv4.conf.default.bootp_relay = 0
net.ipv4.conf.default.medium_id = 0
net.ipv4.conf.default.proxy_arp = 0
net.ipv4.conf.default.accept_source_route = 1
net.ipv4.conf.default.send_redirects = 1
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.shared_media = 1
net.ipv4.conf.default.secure_redirects = 1
net.ipv4.conf.default.accept_redirects = 1
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.default.forwarding = 0
net.ipv4.conf.all.disable_policy = 0
net.ipv4.conf.all.disable_xfrm = 0
net.ipv4.conf.all.arp_filter = 0
net.ipv4.conf.all.tag = 0
net.ipv4.conf.all.log_martians = 0
net.ipv4.conf.all.bootp_relay = 0
net.ipv4.conf.all.medium_id = 0
net.ipv4.conf.all.proxy_arp = 0
net.ipv4.conf.all.accept_source_route = 0
net.ipv4.conf.all.send_redirects = 1
net.ipv4.conf.all.rp_filter = 1
net.ipv4.conf.all.shared_media = 1
net.ipv4.conf.all.secure_redirects = 1
net.ipv4.conf.all.accept_redirects = 0
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.all.forwarding = 0
net.ipv4.neigh.eth0.locktime = 100
net.ipv4.neigh.eth0.proxy_delay = 80
net.ipv4.neigh.eth0.anycast_delay = 100
net.ipv4.neigh.eth0.proxy_qlen = 64
net.ipv4.neigh.eth0.unres_qlen = 3
net.ipv4.neigh.eth0.gc_stale_time = 60
net.ipv4.neigh.eth0.delay_first_probe_time = 5
net.ipv4.neigh.eth0.base_reachable_time = 30
net.ipv4.neigh.eth0.retrans_time = 100
net.ipv4.neigh.eth0.app_solicit = 0
net.ipv4.neigh.eth0.ucast_solicit = 3
net.ipv4.neigh.eth0.mcast_solicit = 3
net.ipv4.neigh.lo.locktime = 100
net.ipv4.neigh.lo.proxy_delay = 80
net.ipv4.neigh.lo.anycast_delay = 100
net.ipv4.neigh.lo.proxy_qlen = 64
net.ipv4.neigh.lo.unres_qlen = 3
net.ipv4.neigh.lo.gc_stale_time = 60
net.ipv4.neigh.lo.delay_first_probe_time = 5
net.ipv4.neigh.lo.base_reachable_time = 30
net.ipv4.neigh.lo.retrans_time = 100
net.ipv4.neigh.lo.app_solicit = 0
net.ipv4.neigh.lo.ucast_solicit = 3
net.ipv4.neigh.lo.mcast_solicit = 3
net.ipv4.neigh.default.gc_thresh3 = 1024
net.ipv4.neigh.default.gc_thresh2 = 512
net.ipv4.neigh.default.gc_thresh1 = 128
net.ipv4.neigh.default.gc_interval = 30
net.ipv4.neigh.default.locktime = 100
net.ipv4.neigh.default.proxy_delay = 80
net.ipv4.neigh.default.anycast_delay = 100
net.ipv4.neigh.default.proxy_qlen = 64
net.ipv4.neigh.default.unres_qlen = 3
net.ipv4.neigh.default.gc_stale_time = 60
net.ipv4.neigh.default.delay_first_probe_time = 5
net.ipv4.neigh.default.base_reachable_time = 30
net.ipv4.neigh.default.retrans_time = 100
net.ipv4.neigh.default.app_solicit = 0
net.ipv4.neigh.default.ucast_solicit = 3
net.ipv4.neigh.default.mcast_solicit = 3
net.ipv4.ipfrag_secret_interval = 600
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_frto = 0
net.ipv4.tcp_tw_reuse = 0
net.ipv4.icmp_ratemask = 6168
net.ipv4.icmp_ratelimit = 100
net.ipv4.tcp_adv_win_scale = 2
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_rmem = 4096 87380 174760
net.ipv4.tcp_wmem = 4096 16384 131072
net.ipv4.tcp_mem = 195584 196096 196608
net.ipv4.tcp_dsack = 1
net.ipv4.tcp_ecn = 0
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_fack = 1
net.ipv4.tcp_orphan_retries = 0
net.ipv4.inet_peer_gc_maxtime = 120
net.ipv4.inet_peer_gc_mintime = 10
net.ipv4.inet_peer_maxttl = 600
net.ipv4.inet_peer_minttl = 120
net.ipv4.inet_peer_threshold = 65664
net.ipv4.igmp_max_memberships = 20
net.ipv4.route.secret_interval = 600
net.ipv4.route.min_adv_mss = 256
net.ipv4.route.min_pmtu = 552
net.ipv4.route.mtu_expires = 600
net.ipv4.route.gc_elasticity = 8
net.ipv4.route.error_burst = 500
net.ipv4.route.error_cost = 100
net.ipv4.route.redirect_silence = 2048
net.ipv4.route.redirect_number = 9
net.ipv4.route.redirect_load = 2
net.ipv4.route.gc_interval = 60
net.ipv4.route.gc_timeout = 300
net.ipv4.route.gc_min_interval = 0
net.ipv4.route.max_size = 2097152
net.ipv4.route.gc_thresh = 131072
net.ipv4.route.max_delay = 10
net.ipv4.route.min_delay = 2
net.ipv4.icmp_ignore_bogus_error_responses = 0
net.ipv4.icmp_echo_ignore_broadcasts = 1
net.ipv4.icmp_echo_ignore_all = 0
net.ipv4.ip_local_port_range = 1024 65000
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_fin_timeout = 60
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 2
net.ipv4.tcp_keepalive_time = 1800
net.ipv4.ipfrag_time = 30
net.ipv4.ip_dynaddr = 0
net.ipv4.ipfrag_low_thresh = 196608
net.ipv4.ipfrag_high_thresh = 262144
net.ipv4.tcp_max_tw_buckets = 180000
net.ipv4.tcp_max_orphans = 32768
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syn_retries = 5
net.ipv4.ip_nonlocal_bind = 0
net.ipv4.ip_no_pmtu_disc = 0
net.ipv4.ip_autoconfig = 0
net.ipv4.ip_default_ttl = 64
net.ipv4.ip_forward = 0
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_timestamps = 1
net.core.divert_version = 0.46
net.core.hot_list_length = 128
net.core.optmem_max = 10240
net.core.message_burst = 50
net.core.message_cost = 5
net.core.mod_cong = 290
net.core.lo_cong = 100
net.core.no_cong = 20
net.core.no_cong_thresh = 20
net.core.netdev_max_backlog = 300
net.core.dev_weight = 64
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.core.rmem_max = 262144
net.core.wmem_max = 262144
vm.stack_defer_threshold = 3072
vm.inactive_clean_percent = 30
vm.dcache_priority = 0
vm.hugetlb_pool = 0
vm.max_map_count = 65536
vm.max-readahead = 31
vm.min-readahead = 3
vm.page-cluster = 3
vm.pagetable_cache = 25 50
vm.kswapd = 512 32 8
vm.pagecache = 1 15 100
vm.overcommit_ratio = 50
vm.overcommit_memory = 0
vm.bdflush = 50 500 0 0 500 3000 80 50 0
kernel.sercons_esc = -1
kernel.overflowgid = 65534
kernel.overflowuid = 65534
kernel.random.uuid = 5c258405-9242-42d4-a08c-7b4330b37dd9
kernel.random.boot_id = 650e84f4-0568-4a41-bd2e-9de86cb21505
kernel.random.write_wakeup_threshold = 128
kernel.random.read_wakeup_threshold = 8
kernel.random.entropy_avail = 0
kernel.random.poolsize = 512
kernel.pid_max = 32768
kernel.threads-max = 62976
kernel.cad_pid = 1
kernel.sysrq-timer = 10
kernel.sysrq-sticky = 0
kernel.sysrq-key = 84
kernel.sysrq = 0
kernel.sem = 250 32000 128 128
kernel.msgmnb = 32768
kernel.msgmni = 2052
kernel.msgmax = 16384
kernel.shmmni = 4096
kernel.shmall = 2097152
kernel.shmmax = 4294967295
kernel.rtsig-max = 1024
kernel.rtsig-nr = 1
kernel.acct = 4 2 30
kernel.hotplug = /sbin/hotplug
kernel.modprobe = /sbin/modprobe
kernel.printk = 6 4 1 7
kernel.ctrl-alt-del = 0
kernel.real-root-dev = 256
kernel.task_size = -1073741824
kernel.cap-bound = -257
kernel.tainted = 1
kernel.core_pattern = core
kernel.core_setuid_ok = 0
kernel.core_uses_pid = 0
kernel.print_fatal_signals = 0
kernel.panic_on_oops = 1
kernel.panic = 0
kernel.domainname = (none)
kernel.hostname =
kernel.version = #1 SMP Thu Apr 22 00:09:08 EDT 2004
kernel.osrelease = 2.4.21-15.ELhugemem
kernel.ostype = Linux
fs.nfs.nlm_tcpport = 0
fs.nfs.nlm_udpport = 0
fs.nfs.nlm_timeout = 10
fs.nfs.nlm_grace_period = 0
fs.quota.syncs = 8
fs.quota.free_dquots = 0
fs.quota.allocated_dquots = 0
fs.quota.cache_hits = 0
fs.quota.writes = 0
fs.quota.reads = 0
fs.quota.drops = 0
fs.quota.lookups = 0
fs.aio-pinned = 0
fs.aio-max-pinned = 884735
fs.aio-max-size = 131072
fs.aio-max-nr = 65536
fs.aio-nr = 0
fs.lease-break-time = 45
fs.dir-notify-enable = 1
fs.leases-enable = 1
fs.overflowgid = 65534
fs.overflowuid = 65534
fs.dentry-state = 449026 331911 45 0 0 0
fs.file-max = 65536
fs.file-nr = 3691 1883 65536
fs.inode-state = 316385 183427 0 0 0 0 0
fs.inode-nr = 316385 183427

[Ivan Ferreira]

You cannot be sure if the problem is I/O related. With default parameters, the system should be able to andle heavy I/O situations. Maybe, you have a problem with the storage configuration, multipath configurations, HBA parameters, HBA driver, or kernel version. Maybe is CPU related.

But, the most important part of tuning probably are the I/O schedulers:

http://www.redhat.com/magazine/008jun05/features/schedulers/

[Robert_Ngo]

i have send the server log file for HP to analyze, they does not find any problem with the hardware, storage team also have not found anything unusual on the SAN storage during that time.

[Robert_Ngo]

all the disk have the same read/write latency, it there going to be effect on the i/o performance to tune these parameters?


read_latency: 2048
write_latency: 8192
max_bomb_segments: 6

[Sivakumar MJ._1]

All XP Arrays work based on Hi Star Crossbar Architecture..

When data is transfered to XP from the host, the data first written in the XP Cache Memory and then to the Disk.

You have to consider the OS tuning or App. tuning.

[William B. Parkinson]

You still can't rule io on either side. To look at the network you show run SAR. To look at the disk and VXXM you have to run iostat.
run these at about 20 second intervals and then use csvpng and tlviz to see what is going on. I have seen on occasion the san disks saturate with high utilization in turn saturating the VXXM managers.