topic Re: Problem with NIC bonding in RedHat in Operating System - Linux

Problem with NIC bonding in RedHat

Cesar Remondino_1 — Wed, 11 May 2011 10:10:42 GMT

Hi,

I'm trying to configure NIC bonding in a linux box (Proliant DL380 G3).
When I boot the system, I only see one of the bonding NICs (eth1), but if I issue a "service network restart", both NICs appear.
I notice that in the ifconfig output the IP appears in every interface (bond0, eth0, eth1), and also in the routing table:
]# netstat -rn
Kernel IP routing table
Destination Gateway Genmask Flags MSS Window irtt Iface
192.168.92.0 0.0.0.0 255.255.255.0 U 0 0 0 bond0
192.168.92.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
192.168.92.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1
169.254.0.0 0.0.0.0 255.255.0.0 U 0 0 0 bond0
0.0.0.0 192.168.92.1 0.0.0.0 UG 0 0 0 bond0

CONFIG FILES:

# cat ifcfg-eth0
DEVICE=eth0
BOOTPROTO=none
ONBOOT=yes
MASTER=bond0
SLAVE=yes
USERCTL=no
HWADDR=00:0b:cd:f3:d6:c9
TYPE=Ethernet
IPV6INIT=no
ETHTOOL_OPTS="autoneg off speed 100 duplex full"

# cat ifcfg-eth1
DEVICE=eth1
BOOTPROTO=none
ONBOOT=yes
MASTER=bond0
SLAVE=yes
USERCTL=no
HWADDR=00:0b:cd:f3:d6:c8
TYPE=Ethernet
IPV6INIT=no
ETHTOOL_OPTS="autoneg off speed 100 duplex full"

# cat ifcfg-bond0
DEVICE=bond0
BOOTPROTO=none
IPADDR=192.168.92.180
BROADCAST=192.168.92.255
NETMASK=255.255.255.0
NETWORK=192.168.92.0
GATEWAY=192.168.92.1
ONBOOT=yes
TYPE=Ethernet
USERCTL=no
PEERDNS=yes
IPV6INIT=no

# cat /etc/modprobe.conf
alias eth0 tg3
alias eth1 tg3
alias bond0 bonding
options bond0 miimon=100 mode=1
alias scsi_hostadapter cciss
alias usb-controller ohci-hcd

----------------- after system reboot:
# ifconfig
bond0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C9
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c9/64 Scope:Link
UP BROADCAST RUNNING MASTER MULTICAST MTU:1500 Metric:1
RX packets:9142 errors:0 dropped:0 overruns:0 frame:0
TX packets:1084 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:1012759 (989.0 KiB) TX bytes:227266 (221.9 KiB)

eth1 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C9
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c9/64 Scope:Link
UP BROADCAST RUNNING SLAVE MULTICAST MTU:1500 Metric:1
RX packets:9142 errors:0 dropped:0 overruns:0 frame:0
TX packets:1084 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1012759 (989.0 KiB) TX bytes:227266 (221.9 KiB)
Interrupt:201

lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:186 errors:0 dropped:0 overruns:0 frame:0
TX packets:186 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:22098 (21.5 KiB) TX bytes:22098 (21.5 KiB)

# ethtool eth1
Settings for eth1:
Supported ports: [ MII ]
Supported link modes: 10baseT/Half 10baseT/Full
100baseT/Half 100baseT/Full
1000baseT/Half 1000baseT/Full
Supports auto-negotiation: Yes
Advertised link modes: Not reported
Advertised auto-negotiation: No
Speed: 100Mb/s
Duplex: Full
Port: Twisted Pair
PHYAD: 1
Transceiver: internal
Auto-negotiation: off
Supports Wake-on: g
Wake-on: d
Current message level: 0x000000ff (255)
Link detected: yes

# ethtool eth0
Settings for eth0:
Cannot get device settings: Resource temporarily unavailable
Supports Wake-on: g
Wake-on: d
Current message level: 0x000000ff (255)
Link detected: no

----------------------- after "service network restart":
# ifconfig
bond0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C9
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::200:ff:fe00:0/64 Scope:Link
UP BROADCAST RUNNING MASTER MULTICAST MTU:1500 Metric:1
RX packets:340 errors:0 dropped:0 overruns:0 frame:0
TX packets:198 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:33450 (32.6 KiB) TX bytes:30100 (29.3 KiB)

eth0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C9
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c9/64 Scope:Link
UP BROADCAST RUNNING SLAVE MULTICAST MTU:1500 Metric:1
RX packets:268 errors:0 dropped:0 overruns:0 frame:0
TX packets:193 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:25533 (24.9 KiB) TX bytes:29702 (29.0 KiB)
Interrupt:193

eth1 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C9
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c9/64 Scope:Link
UP BROADCAST RUNNING NOARP SLAVE MULTICAST MTU:1500 Metric:1
RX packets:72 errors:0 dropped:0 overruns:0 frame:0
TX packets:5 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:7917 (7.7 KiB) TX bytes:398 (398.0 b)
Interrupt:201

# ethtool eth0
Settings for eth0:
Supported ports: [ MII ]
Supported link modes: 10baseT/Half 10baseT/Full
100baseT/Half 100baseT/Full
1000baseT/Half 1000baseT/Full
Supports auto-negotiation: Yes
Advertised link modes: Not reported
Advertised auto-negotiation: No
Speed: 100Mb/s
Duplex: Full
Port: Twisted Pair
PHYAD: 1
Transceiver: internal
Auto-negotiation: off
Supports Wake-on: g
Wake-on: d
Current message level: 0x000000ff (255)
Link detected: yes
# ethtool eth1
Settings for eth1:
Supported ports: [ MII ]
Supported link modes: 10baseT/Half 10baseT/Full
100baseT/Half 100baseT/Full
1000baseT/Half 1000baseT/Full
Supports auto-negotiation: Yes
Advertised link modes: Not reported
Advertised auto-negotiation: No
Speed: 100Mb/s
Duplex: Full
Port: Twisted Pair
PHYAD: 1
Transceiver: internal
Auto-negotiation: off
Supports Wake-on: g
Wake-on: d
Current message level: 0x000000ff (255)
Link detected: yes

Re: Problem with NIC bonding in RedHat

Matti_Kurkela — Wed, 11 May 2011 13:26:00 GMT

Which version of RedHat are you running? Is it up to date?

After a reboot, you seem to have your NICs assigned this way:
eth0 = 00:0B:CD:F3:D6:C8
eth1 = 00:0B:CD:F3:D6:C9

But the HWADDR lines in your network configuration suggest the opposite order:
eth0 = 00:0b:cd:f3:d6:c9
eth1 = 00:0b:cd:f3:d6:c8

When starting the network interfaces, uses the HWADDR lines to verify it's configuring the right NICs. If the MAC addresses of the NICs and HWADDR lines in respective configuration lines don't match, the system will *rename the NICs* to make them match. Apparently in your case, something goes wrong in this renaming procedure. I think I vaguely remember seeing some bug reports about this in some versions of RHEL.

You might be able to confirm this by running "ifconfig -a" after system reboot, but before running "service network restart" manually: if I'm correct, you'll see the NIC with the ...d6:c8 MAC address in unconfigured state, with a strange name that looks very different from regular NIC names. (That's a temporary name used when swapping ethX names between two or more NICs.)

You might want to work around the problem by removing the need to rename the NICs: i.e. switch the MAC addresses in the HWADDR lines of ifcfg-eth0 and ifcfg-eth1 files so that they match the "natural" after-reboot state.

MK

Re: Problem with NIC bonding in RedHat

Alzhy — Wed, 11 May 2011 13:32:50 GMT

Bizarre.. I suggest you open a case with RHEL suppt? Your config looks good.

Re: Problem with NIC bonding in RedHat

Cesar Remondino_1 — Wed, 11 May 2011 15:14:54 GMT

Redhat version is:
# cat /etc/redhat-release
Red Hat Enterprise Linux ES release 4 (Nahant)

I've tried to switch HWADDR info between ifcfg-eth0 and ifcfg-eth1, reboot the system, and I have the same behaviour.

Here's the output of an ifconfig -a after rebooting

# ifconfig -a
bond0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C8
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c8/64 Scope:Link
UP BROADCAST RUNNING MASTER MULTICAST MTU:1500 Metric:1
RX packets:846 errors:0 dropped:0 overruns:0 frame:0
TX packets:333 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:92480 (90.3 KiB) TX bytes:46408 (45.3 KiB)

eth0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C8
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
BROADCAST NOARP SLAVE MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Interrupt:201

eth1 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C8
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
inet6 addr: fe80::20b:cdff:fef3:d6c8/64 Scope:Link
UP BROADCAST RUNNING SLAVE MULTICAST MTU:1500 Metric:1
RX packets:846 errors:0 dropped:0 overruns:0 frame:0
TX packets:333 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:92480 (90.3 KiB) TX bytes:46408 (45.3 KiB)
Interrupt:193

lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:12 errors:0 dropped:0 overruns:0 frame:0
TX packets:12 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:1402 (1.3 KiB) TX bytes:1402 (1.3 KiB)

sit0 Link encap:IPv6-in-IPv4
NOARP MTU:1480 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

Re: Problem with NIC bonding in RedHat

Alzhy — Wed, 11 May 2011 15:17:45 GMT

Can you post:

cat /proc/net/bonding/bond0

pre and post network restart?

Re: Problem with NIC bonding in RedHat

Cesar Remondino_1 — Wed, 11 May 2011 15:22:32 GMT

----------- PRE

# cat /proc/net/bonding/bond0
Ethernet Channel Bonding Driver: v2.6.0 (January 14, 2004)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth1
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: down
Link Failure Count: 1
Permanent HW addr: 00:0b:cd:f3:d6:c8

Slave Interface: eth1
MII Status: up
Link Failure Count: 0
Permanent HW addr: 00:0b:cd:f3:d6:c9

----------- POST

# cat /proc/net/bonding/bond0
Ethernet Channel Bonding Driver: v2.6.0 (January 14, 2004)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: eth0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: eth0
MII Status: up
Link Failure Count: 1
Permanent HW addr: 00:0b:cd:f3:d6:c8

Slave Interface: eth1
MII Status: up
Link Failure Count: 1
Permanent HW addr: 00:0b:cd:f3:d6:c9

Re: Problem with NIC bonding in RedHat

Alzhy — Wed, 11 May 2011 15:28:55 GMT

Try moving your binding options from modeprobe.conf to your ifcfg-bond0:

BONDING_OPTS="mode=active-backup miimon=100 primary=eth0"

and specify your primary and see if it makes any difference.

Or you can use your existing and just add a primary=eth0 in your modprobe.conf bond options.

Re: Problem with NIC bonding in RedHat

Cesar Remondino_1 — Wed, 11 May 2011 15:46:37 GMT

same behaviour, eth0 always cames up as DOWN

# ethtool eth0
Settings for eth0:
Cannot get device settings: Resource temporarily unavailable
Supports Wake-on: g
Wake-on: d
Current message level: 0x000000ff (255)
Link detected: no
# ifconfig -a eth0
eth0 Link encap:Ethernet HWaddr 00:0B:CD:F3:D6:C8
inet addr:192.168.92.180 Bcast:192.168.92.255 Mask:255.255.255.0
BROADCAST NOARP SLAVE MULTICAST MTU:1500 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0 frame:0
TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
Interrupt:201

Re: Problem with NIC bonding in RedHat

Alzhy — Thu, 12 May 2011 12:21:41 GMT

Hows about removing HWADDRESS from your ifcfg config files?

Re: Problem with NIC bonding in RedHat

Cesar Remondino_1 — Mon, 16 May 2011 09:45:35 GMT

I've already tried that with the same results.