Linux 3.10 kernel bridge轉發邏輯

Linux 3.10 kernel bridge轉發邏輯

之前分析過linux kernel 2.6.32的bridge轉發邏輯，下面分析一下linux kernel 3.10的bridge轉發邏輯。這樣正是CentOS 5和CentOS 7對應的內核。3.10 kernel中bridge邏輯的最大改變就是增加了vlan處理邏輯以及brdige入口函數的設置。

1. netdev_rx_handler_register

在分析之前首先要介紹一個重要函數：netdev_rx_handler_register，這個函數是2.6內核所沒有的。

lnetdev_rx_handler_register

/*
* dev: 要注冊接收函數的dev
* rx_handler: 要注冊的接收函數
* rx_handler_data: 指向rx_handler_data使用的數據
*/
int netdev_rx_handler_register(struct net_device *dev,
rx_handler_func_t *rx_handler,
void *rx_handler_data)
{
 ASSERT_RTNL();

 if (dev->rx_handler)
   return -EBUSY;

 /* Note: rx_handler_data must be set before rx_handler */
 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
 rcu_assign_pointer(dev->rx_handler, rx_handler);

 return 0;
}

這個函數可以給設備（net_device）注冊接收函數，然後在__netif_receive_skb函數中根據接收skb的設備接口，再調用這個被注冊的接收函數。比如為網橋下的接口注冊br_handle_frame函數，為bonding接口注冊bond_handle_frame函數。這相對於老式的網橋處理更靈活，有了這個機制也可以在模塊中自行注冊處理函數。比如3.10中的openvswitch（OpenvSwitch在3.10已經合入了內核）創建netdev vport的函數netdev_create。

lnetdev_create

static struct vport *netdev_create(const struct vport_parms *parms)
{
 struct vport *vport;
/....../
 err = netdev_rx_handler_register(netdev_vport->dev, netdev_frame_hook,vport);
/....../
}

這個函數在創建netdev vport時將設備的接收函數設置為netdev_frame_hook函數，這也是整個openvswitch的入口函數，如果查看OpenvSwitch的源碼可以看到當安裝於2.6內核時這裡是替換掉bridge的br_handle_frame_hook函數，從而由bridge邏輯進入OpenvSwitch邏輯。

2. Bridge轉發邏輯分析

還是先從netif_receive_skb函數分析，這個函數算是進入協議棧的入口。

lnetif_receive_skb

int netif_receive_skb(struct sk_buff *skb)
{
 int ret;
 if (skb_defer_rx_timestamp(skb))
   return NET_RX_SUCCESS;
 rcu_read_lock();
 /*RPS邏輯處理，現在內核中使用了RPS機制, 將報文分散到各個cpu的接收隊列中進行負載均衡處理*/
 #ifdef CONFIG_RPS
 if (static_key_false(&rps_needed)) {
   struct rps_dev_flow voidflow, *rflow = &voidflow;
   int cpu = get_rps_cpu(skb->dev, skb, &rflow);
   if (cpu >= 0) {
     ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
     rcu_read_unlock();
     return ret;
   }
 }
 #endif
 ret = __netif_receive_skb(skb);
 rcu_read_unlock();
 return ret;
}

netif_receive_skb只是對數據包進行了RPS的處理，然後調用__netif_receive_skb。

__netif_receive_skb並沒有其他多余的處理邏輯，主要調用 __netif_receive_skb_core，這個函數才真正相當於2.6內核的netif_receive_skb。以下代碼省略了和bridge無關的邏輯。

l__netif_receive_skb_core

static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc)
{
 struct packet_type *ptype, *pt_prev;
 rx_handler_func_t *rx_handler;
 struct net_device *orig_dev;
 struct net_device *null_or_dev;
 bool deliver_exact = false;
 int ret = NET_RX_DROP;
 __be16 type;
/*......*/
 orig_dev = skb->dev;
 skb_reset_network_header(skb);
 pt_prev = NULL;
 skb->skb_iif = skb->dev->ifindex;
/*ptype_all協議處理，tcpdump抓包就在這裡*/
 list_for_each_entry_rcu(ptype, &ptype_all, list) {
   if (!ptype->dev || ptype->dev == skb->dev) {
     if (pt_prev)
       ret = deliver_skb(skb, pt_prev, orig_dev);
     pt_prev = ptype;
    }
 }
/*調用接收設備的rx_handler*/
 rx_handler = rcu_dereference(skb->dev->rx_handler);
 if (rx_handler) {
   if (pt_prev) {
     ret = deliver_skb(skb, pt_prev, orig_dev);
     pt_prev = NULL;
   }
   switch (rx_handler(&skb)) {
     case RX_HANDLER_CONSUMED:
       ret = NET_RX_SUCCESS;
       goto out;
     case RX_HANDLER_ANOTHER:
       goto another_round;
     case RX_HANDLER_EXACT:
        deliver_exact = true;
     case RX_HANDLER_PASS:
       break;
     default:
       BUG();
   }
 }
/*根據 skb->protocol傳遞給上層協議*/
 type = skb->protocol;
 list_for_each_entry_rcu(ptype,&ptype_base[ntohs(type) & PTYPE_HASH_MASK], list) {
   if (ptype->type == type &&(ptype->dev == null_or_dev || ptype->dev == skb->dev ||ptype->dev == orig_dev)) {
     if (pt_prev)
       ret = deliver_skb(skb, pt_prev, orig_dev);
   pt_prev = ptype;
   }
 }
 if (pt_prev) {
   if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC)))
     goto drop;
   else
     ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
 } else {
drop:
   atomic_long_inc(&skb->dev->rx_dropped);
   kfree_skb(skb);
   ret = NET_RX_DROP;
}
out:
 return ret;
}

如果一個dev被添加到一個bridge（做為bridge的一個接口），的這個接口設備的rx_handler被設置為br_handle_frame函數，這是在br_add_if函數中設置的，而br_add_if (net/bridge/br_if.c)是在向網橋設備上添加接口時設置的。進入br_handle_frame也就進入了bridge的邏輯代碼。

lbr_add_if

int br_add_if(struct net_bridge *br, struct net_device *dev)
{
 /*......*/
 err = netdev_rx_handler_register(dev, br_handle_frame, p);
 /*......*/
}

lbr_handle_frame

rx_handler_result_t br_handle_frame(struct sk_buff **pskb)
{
 struct net_bridge_port *p;
 struct sk_buff *skb = *pskb;
 const unsigned char *dest = eth_hdr(skb)->h_dest;
 br_should_route_hook_t *rhook;
 if (unlikely(skb->pkt_type == PACKET_LOOPBACK))
   return RX_HANDLER_PASS;
 if (!is_valid_ether_addr(eth_hdr(skb)->h_source))
   goto drop;
 skb = skb_share_check(skb, GFP_ATOMIC);
 if (!skb)
   return RX_HANDLER_CONSUMED;
/*獲取dev對應的bridge port*/
 p = br_port_get_rcu(skb->dev);
/*特殊目的mac地址的處理*/
 if (unlikely(is_link_local_ether_addr(dest))) {
 /*
  * See IEEE 802.1D Table 7-10 Reserved addresses
  *
  * Assignment   Value
  * Bridge Group Address 01-80-C2-00-00-00
  * (MAC Control) 802.3 01-80-C2-00-00-01
  * (Link Aggregation) 802.3 01-80-C2-00-00-02
  * 802.1X PAE address 01-80-C2-00-00-03
  *
  * 802.1AB LLDP 01-80-C2-00-00-0E
  *
  * Others reserved for future standardization
  */
   switch (dest[5]) {
     case 0x00: /* Bridge Group Address */
     /* If STP is turned off,then must forward to keep loop detection */
       if (p->br->stp_enabled == BR_NO_STP)
         goto forward;
     break;
     case 0x01: /* IEEE MAC (Pause) */
       goto drop;
     default:
       /* Allow selective forwarding for most other protocols */
       if (p->br->group_fwd_mask & (1u << dest[5]))
         goto forward;
   }
   /* LOCAL_IN hook點，注意經過這個hook點並不代表發送到主機協議棧（只有特殊目的mac01-80-C2才會走到這裡）*/
   if (NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, skb->dev,
     NULL, br_handle_local_finish)) {
       return RX_HANDLER_CONSUMED; /* consumed by filter */
   } else {
     *pskb = skb;
     return RX_HANDLER_PASS; /* continue processing */
   }
 }
/*轉發邏輯*/
forward:
 switch (p->state) {
   case BR_STATE_FORWARDING:
     rhook = rcu_dereference(br_should_route_hook);
     if (rhook) {
       if ((*rhook)(skb)) {
         *pskb = skb;
         return RX_HANDLER_PASS;
       }
       dest = eth_hdr(skb)->h_dest;
      }
   /* fall through */
   case BR_STATE_LEARNING:
     /*skb的目的mac和bridge的mac一樣，則將skb發往本機協議棧*/
     if (ether_addr_equal(p->br->dev->dev_addr, dest))
       skb->pkt_type = PACKET_HOST;
     /*NF_BR_PRE_ROUTING hook點*/
     NF_HOOK(NFPROTO_BRIDGE, NF_BR_PRE_ROUTING, skb, skb->dev, NULL,br_handle_frame_finish);
   break;
default:
drop:
 kfree_skb(skb);
}
return RX_HANDLER_CONSUMED;
}

經過NF_BR_LOCAL_IN hook點會執行br_handle_local_finish函數。

lbr_handle_local_finish

static int br_handle_local_finish(struct sk_buff *skb)
{
 struct net_bridge_port *p = br_port_get_rcu(skb->dev);
 u16 vid = 0;
/*獲取skb的vlan id(3.10的bridge支持vlan)*/
 br_vlan_get_tag(skb, &vid);
/*更新bridge的mac表，注意vlan id也是參數，說明每個vlan有一個獨立的mac表*/
 br_fdb_update(p->br, p, eth_hdr(skb)->h_source, vid);
 return 0; /* process further */
}

經過NF_BR_PRE_ROUTING hook點會執行br_handle_frame_finish函數。

lbr_handle_frame_finish

int br_handle_frame_finish(struct sk_buff *skb)
{
 const unsigned char *dest = eth_hdr(skb)->h_dest;
 struct net_bridge_port *p = br_port_get_rcu(skb->dev);
 struct net_bridge *br;
 struct net_bridge_fdb_entry *dst;
 struct net_bridge_mdb_entry *mdst;
 struct sk_buff *skb2;
 u16 vid = 0;
 if (!p || p->state == BR_STATE_DISABLED)
   goto drop;
   /*這個判斷主要是vlan的相關檢查，如是否和接收接口配置的vlan相同*/
 if (!br_allowed_ingress(p->br, nbp_get_vlan_info(p), skb, &vid))
   goto out;
 /* insert into forwarding database after filtering to avoid spoofing */
 br = p->br;
  /*更新轉發數據庫*/
 br_fdb_update(br, p, eth_hdr(skb)->h_source, vid);
  /*多播mac的處理*/
 if (!is_broadcast_ether_addr(dest) && is_multicast_ether_addr(dest) &&
    br_multicast_rcv(br, p, skb))
   goto drop;
 if (p->state == BR_STATE_LEARNING)
   goto drop;
 BR_INPUT_SKB_CB(skb)->brdev = br->dev;
 /* The packet skb2 goes to the local host (NULL to skip). */
 skb2 = NULL;
/*如果網橋被設置為混雜模式*/
 if (br->dev->flags & IFF_PROMISC)
   skb2 = skb;
  dst = NULL;
/*如果skb的目的mac是廣播*/
 if (is_broadcast_ether_addr(dest))
   skb2 = skb;
 else if (is_multicast_ether_addr(dest)) { /*多播*/
   mdst = br_mdb_get(br, skb, vid);
 if (mdst || BR_INPUT_SKB_CB_MROUTERS_ONLY(skb)) {
   if ((mdst && mdst->mglist) ||
    br_multicast_is_router(br))
     skb2 = skb;
   br_multicast_forward(mdst, skb, skb2);
   skb = NULL;
   if (!skb2)
     goto out;
   } else
     skb2 = skb;
    br->dev->stats.multicast++;
  } else if ((dst = __br_fdb_get(br, dest, vid)) &&dst->is_local) {/*目的地址是本機mac，則發往本機協議棧*/
     skb2 = skb;
     /* Do not forward the packet since it's local. */
     skb = NULL;
  }
  if (skb) {
   if (dst) {
     dst->used = jiffies;
     br_forward(dst->dst, skb, skb2); //轉發給目的接口
   } else
      br_flood_forward(br, skb, skb2); //找不到目的接口則廣播
 }
 if (skb2)
    return br_pass_frame_up(skb2); //發往本機協議棧
out:
 return 0;
drop:
 kfree_skb(skb);
 goto out;
}

我們先看發往本機協議棧的函數br_pass_frame_up。

lbr_pass_frame_up

static int br_pass_frame_up(struct sk_buff *skb)
{
 struct net_device *indev, *brdev = BR_INPUT_SKB_CB(skb)->brdev;
 struct net_bridge *br = netdev_priv(brdev);
//更新統計計數(略)
 /* Bridge is just like any other port. Make sure the
  * packet is allowed except in promisc modue when someone
  * may be running packet capture.
  */
 if (!(brdev->flags & IFF_PROMISC) &&!br_allowed_egress(br, br_get_vlan_info(br), skb)) {
   kfree_skb(skb); //如果不是混雜模式且vlan處理不合要求則丟棄
   return NET_RX_DROP;
 }
//vlan處理邏輯
 skb = br_handle_vlan(br, br_get_vlan_info(br), skb);
  if (!skb)
   return NET_RX_DROP;
  indev = skb->dev;
 skb->dev = brdev; //重點，這裡修改了skb->dev為bridge
 //經過NF_BR_LOCAL_IN再次進入協議棧
 return NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_IN, skb, indev, NULL,
  netif_receive_skb);
}

再次進入netif_receive_skb，由於skb-dev被設置成了bridge，而bridge設備的rx_handler函數是沒有被設置的，所以就不會再次進入bridge邏輯，而直接進入了主機上層協議棧。

下面看轉發邏輯，轉發邏輯主要在br_forward函數中，而br_forward主要調用__br_forward函數。

l__br_forward

static void __br_forward(const struct net_bridge_port *to, struct sk_buff *skb)
{
 struct net_device *indev;
 //vlan處理
 skb = br_handle_vlan(to->br, nbp_get_vlan_info(to), skb);
 if (!skb)
   return;
 indev = skb->dev;
 skb->dev = to->dev; //skb->dev設置為出口設備dev
 skb_forward_csum(skb);
 //經過NF_BR_FORWARD hook點，調用br_forward_finish
 NF_HOOK(NFPROTO_BRIDGE, NF_BR_FORWARD, skb, indev, skb->dev,
 br_forward_finish);
}

lbr_forward_finish

int br_forward_finish(struct sk_buff *skb)
{
 //經過NF_BR_POST_ROUTING hook點,調用br_dev_queue_push_xmit
 return NF_HOOK(NFPROTO_BRIDGE, NF_BR_POST_ROUTING, skb, NULL, skb->dev,br_dev_queue_push_xmit);
}

lbr_dev_queue_push_xmit

int br_dev_queue_push_xmit(struct sk_buff *skb)
{
 /* ip_fragment doesn't copy the MAC header */
 if (nf_bridge_maybe_copy_header(skb) ||(packet_length(skb) > skb->dev->mtu && !skb_is_gso(skb))) {
   kfree_skb(skb);
 } else {
   skb_push(skb, ETH_HLEN);
   br_drop_fake_rtable(skb);
   dev_queue_xmit(skb); //發送到鏈路層
 }
 return 0;
}

Skb進入dev_queue_xmit就會調用相應設備驅動的發送函數。也就出了bridge邏輯。所以整個3.10kernel的bridge轉發邏輯如下圖所示：

注意，和2.6kernel一樣，bridge的OUTPUT hook點在bridge dev的發送函數中，這裡不再分析列出。