vxlanMgr | 南风未起｜云原生/Go技术分享博客

# 1.简介

# 1.1.vxlan

vxlan是在L3网络上创建L2覆盖网络，将以太帧封装在UDP数据包，涉及L2相邻性或处理UDP流量更高效的环境，vxlan相比IPIP更优。

--- vxlanMgr
	1.创建和管理vxlan隧道设备-->vxlan.calico
	2.维护vxlan转发数据库FDB,FDB会将远程节点VTEP(vxlan隧道端点)的MAC地址和IP映射到底层IP地址
	3.基于vxlan隧道配置远程Pod路由
	4.响应与vxlan相关的calico节点配置和IP池设置的变化

1
2
3
4
5

注意

vxlan依赖tun设备，由vxlanMgr创建及监听，非vxlan模式会绕过vxlanMgr直接清理

# 1.2.初始化

newVXLANManager()会实例化vxlan manager，维护vxlan路由、黑洞路由、VTEP网卡配置及涉及到的ipset规则同步。

func newVXLANManager(ipsetsDataplane common.IPSetsDataplane, rt routetable.RouteTableInterface, deviceName string, dpConfig Config, ..., ipVersion uint8, ...) *vxlanManager {
	nlHandle, _ := netlink.NewHandle()

    // 设置黑洞路由标记
	blackHoleProto := 80
	if dpConfig.DeviceRouteProtocol != syscall.RTPROT_BOOT {
		blackHoleProto = dpConfig.DeviceRouteProtocol
	}
	...
    
    // blackHole routeTable初始化
	if !dpConfig.RouteSyncDisabled {
		brt = routetable.New([]string{routetable.InterfaceNone}, 4, false, dpConfig.NetlinkTimeout,
			dpConfig.DeviceRouteSourceAddress, blackHoleProto, false, unix.RT_TABLE_MAIN,...)
		if ipVersion == 6 {
			brt = routetable.New(
				[]string{routetable.InterfaceNone},
				ipVersion,
				false,
				dpConfig.NetlinkTimeout,
				dpConfig.DeviceRouteSourceAddressIPv6,
				blackHoleProto,
				false,
				unix.RT_TABLE_MAIN,
				opRecorder,
				featureDetector,
			)
		}
	} else {
		brt = &routetable.DummyTable{}
	}

	return newVXLANManagerWithShims(ipsetsDataplane, rt, brt, deviceName,
		dpConfig, nlHandle, ipVersion,
		func(...) routetable.RouteTableInterface {
			return routetable.New(interfaceRegexes, ipVersion, vxlan, netlinkTimeout,
				deviceRouteSourceAddress, deviceRouteProtocol, removeExternalRoutes, unix.RT_TABLE_MAIN,
				opRecorder, featureDetector,
			)
		},
	)
}

func newVXLANManagerWithShims(...) *vxlanManager {
	
    noEncapProtocol := 80
	if dpConfig.DeviceRouteProtocol != syscall.RTPROT_BOOT {
		noEncapProtocol = dpConfig.DeviceRouteProtocol
	}

	return &vxlanManager{
		ipsetsDataplane: ipsetsDataplane,
		ipSetMetadata: ipsets.IPSetMetadata{
			MaxSize: dpConfig.MaxIPSetSize,
			SetID:   rules.IPSetIDAllVXLANSourceNets,
			Type:    ipsets.IPSetTypeHashNet,
		},
		hostname:            dpConfig.Hostname,
		routeTable:          rt,
		blackholeRouteTable: brt,
		routesByDest:        map[string]*proto.RouteUpdate{},
		localIPAMBlocks:     map[string]*proto.RouteUpdate{},
		vtepsByNode:         map[string]*proto.VXLANTunnelEndpointUpdate{},
		vxlanDevice:         deviceName,
		vxlanID:             dpConfig.RulesConfig.VXLANVNI,
		vxlanPort:           dpConfig.RulesConfig.VXLANPort,
		ipVersion:           ipVersion,
		externalNodeCIDRs:   dpConfig.ExternalNodesCidrs,
		routesDirty:         true,
		vtepsDirty:          true,
		dpConfig:            dpConfig,
		nlHandle:            nlHandle,
		noEncapProtocol:     noEncapProtocol,
		noEncapRTConstruct:  noEncapRTConstruct,
		logCtx:              logCtx,
	}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

注意

blackHole route会执行mark 80标记，实现区别于普通路由的安全同步管理

# 2.device

# 2.1.keepvxlan

m.KeepVXLANDeviceInSync()会周期性自愈式检查及修复vxlan设备及相关路由配置，以维护封装流量及非封装流量的相关依赖。

// configures the VXLAN tunnel device, then periodically checks that it is still correctly configured.
func (m *vxlanManager) KeepVXLANDeviceInSync(mtu int, xsumBroken bool, wait time.Duration) {
	...
	for {
        // local vtep
		localVTEP := m.getLocalVTEP()
		if localVTEP == nil {
			time.Sleep(1 * time.Second)
			continue
		}

        // vtep parent device
		if parent, err := m.getLocalVTEPParent(); err != nil {
			time.Sleep(1 * time.Second)
			continue
		} else {
            // 非封装流量routeTable
			if m.getNoEncapRouteTable() == nil {
				devRouteSrcAddr := m.dpConfig.DeviceRouteSourceAddress
				if m.ipVersion == 6 {
					devRouteSrcAddr = m.dpConfig.DeviceRouteSourceAddressIPv6
				}
				noEncapRouteTable := m.noEncapRTConstruct([]string{"^" + parent.Attrs().Name + "$"}, m.ipVersion, false, m.dpConfig.NetlinkTimeout, devRouteSrcAddr, m.noEncapProtocol, false)
				m.setNoEncapRouteTable(noEncapRouteTable)
			}
		}
        
        // 配置vxlan device
		err := m.configureVXLANDevice(mtu, localVTEP, xsumBroken)
		if err != nil {
			time.Sleep(1 * time.Second)
			continue
		}
		...
        
		time.Sleep(wait)
	}
}

func (m *vxlanManager) getLocalVTEPParent() (netlink.Link, error) {
	return m.getParentInterface(m.getLocalVTEP())
}

// returns the parent interface for the given local VTEP based on IP address.
func (m *vxlanManager) getParentInterface(localVTEP *proto.VXLANTunnelEndpointUpdate) (netlink.Link, error) {
	// 列出iface
    links, err := m.nlHandle.LinkList()
	...

    // 获取parent deviceIP
	family := netlink.FAMILY_V4
	parentDeviceIP := localVTEP.ParentDeviceIp
	if m.ipVersion == 6 {
		family = netlink.FAMILY_V6
		parentDeviceIP = localVTEP.ParentDeviceIpv6
	}
    
	for _, link := range links {
        // iface addr
		addrs, err := m.nlHandle.AddrList(link, family)
		...
        
        // addr匹配的parent device
		for _, addr := range addrs {
			if addr.IPNet.IP.String() == parentDeviceIP {
				return link, nil
			}
		}
	}
    
	return nil, fmt.Errorf("Unable to find parent interface with address %s", parentDeviceIP)
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

注意

noEncapRouteTable用于处理非封装流量路由，将非remote pod流量直接基于主机网卡发出

# 2.2.configure

m.configureVXLANDevice()会检查及创建vxlan.calico设备，对齐及更新差异的网卡配置，修改内核兼容参数后激活vxlan.calico网卡。

// configureVXLANDevice ensures the VXLAN tunnel device is up and configured correctly.
func (m *vxlanManager) configureVXLANDevice(mtu int, localVTEP *VXLANTunnelEndpointUpdate, xsumBroken bool) .. {
	// parent iface
    parent, err := m.getParentInterface(localVTEP)
	...
    
    // 解析local vtep mac
	mac, err := m.parseMacForIPVersion(localVTEP)
	...
    
    // vxlan.calico addr
	addr := localVTEP.Ipv4Addr
    // parent device addr
	parentDeviceIP := localVTEP.ParentDeviceIp
	if m.ipVersion == 6 {
		addr = localVTEP.Ipv6Addr
		parentDeviceIP = localVTEP.ParentDeviceIpv6
	}
    
	la := netlink.NewLinkAttrs()
    // vxlan.calico名称
	la.Name = m.vxlanDevice
    // mac设置
	la.HardwareAddr = mac
    
    // 实例化vxlan
    vxlan := &netlink.Vxlan{
        LinkAttrs:    la,											// name/mac
        VxlanId:      m.vxlanID,									// VNI
        Port:         m.vxlanPort,									// udp port
        VtepDevIndex: parent.Attrs().Index,							// 外层封装用的父物理网卡索引
        SrcAddr:      ip.FromString(parentDeviceIP).AsNetIP(),		// 封装外层源IP
    }

	// 尝试获取vxlan device
	link, err := m.nlHandle.LinkByName(m.vxlanDevice)
	if err != nil {
        // 尝试创建vxlan.calico
		m.nlHandle.LinkAdd(vxlan)
        ...

		// 检查vxlan device是否存在
		link, err = m.nlHandle.LinkByName(m.vxlanDevice)
		...
	}

	// vxlan device配置不兼容
	if incompat := vxlanLinksIncompat(vxlan, link); incompat != "" {
		// 清理旧网卡
		m.nlHandle.LinkDel(link)
        ...
        
        // 创建新网卡
		m.nlHandle.LinkAdd(vxlan)
        ...
        
        // 检查vxlan device是否存在
		link, err = m.nlHandle.LinkByName(vxlan.Name)
		...
	}
	...
    
	// MTU调整
	if oldMTU != mtu {
		m.nlHandle.LinkSetMTU(link, mtu)
        ...
	}

	// 设置vxlan.calico地址
	m.ensureAddressOnLink(addr, link)
    ...
    
	// 现代网卡硬件支持计算IP/UDP/TCP校验和,减少CPU开销-->TX校验和硬件卸载
    // 部分旧内核/网卡驱动存在缺陷,TX校验和卸载只会计算内层报文校验和,不会正确计算vxlan外层UDP校验和
	if xsumBroken {
        // ethtool -K vxlan-xxx tx off
		ethtool.EthtoolTXOff(m.vxlanDevice)
        ...
	}

	// 激活vxlan.calico设备
	m.nlHandle.LinkSetUp(link)
    ...

	return nil
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

注意

vxlan.calico参数差异会清理重建，必要时关闭device tx检查确保报文封装正确

# 2.3.ensurelink

m.ensureAddressOnLink()会设置预期的vxlan.calico地址，EthtoolTXOff()会基于ioctl修改ethtool tx参数，关闭TX检查。

// ensures that the provided IP address is configured on the provided Link. If there are other addresses,
// this function will remove them, ensuring that the desired IP address is the _only_ address on the Link.
func (m *vxlanManager) ensureAddressOnLink(ipStr string, link netlink.Link) error {
    ...
    
    // vxlan.calico addr
	_, net, err := net.ParseCIDR(ipStr + suffix)
	...
	addr := netlink.Addr{IPNet: net}
    
    // vxlan.calico地址列表
	existingAddrs, err := m.nlHandle.AddrList(link, family)
	...

	// 清理非addr地址
	addrPresent := false
	for _, existing := range existingAddrs {
		if reflect.DeepEqual(existing.IPNet, addr.IPNet) {
			addrPresent = true
			continue
		}

		m.nlHandle.AddrDel(link, &existing)
        ...
	}

	// add the desired address to the interface if needed.
	if !addrPresent {
		m.nlHandle.AddrAdd(link, &addr)
        ...
	}
    
	return nil
}

// EthtoolTXOff disables the TX checksum offload on the specified interface
func EthtoolTXOff(name string) error {
    // device名称长度检查
	if len(name)+1 > unix.IFNAMSIZ {
		return fmt.Errorf("name too long")
	}

	// 初始化ioctl socket
	socket, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM, 0)
	...
    
	defer func() {
		// 关闭socket
        unix.Close(socket)
		...
	}()

	// 分配ethtool struct内存
	alloc := memory.Allocator{}
	defer func() {
		alloc.Close()
		...
	}()
	valueUPtr, err := alloc.UnsafeCalloc(int(unsafe.Sizeof(EthtoolValue{})))
	...
    
    // 退出前释放
	defer func() {
		alloc.UnsafeFree(valueUPtr)
		...
	}()
	value := (*EthtoolValue)(valueUPtr)

    // 读取TX checksum状态(ethtool -K <iface>)
	*value = EthtoolValue{Cmd: unix.ETHTOOL_GTXCSUM}
	request := IFReqData{Data: uintptr(valueUPtr)}
	copy(request.Name[:], name)
	ioctlEthtool(socket, &request)
    ...
    
    // 关闭直接返回
	if value.Data == 0 { // if already off, don't try to change
		return nil
	}

	// ethtool -K <iface> tx off
	*value = EthtoolValue{Cmd: unix.ETHTOOL_STXCSUM, Data: 0 /* off */}
	return ioctlEthtool(socket, &request)
}

注意

ethtool tx检查及关闭基于ioctl工具，利用原始指针设置cmd/name

# 3.vxlanMgr

# 3.1.opUpdate

m.OnUpdate()根据不同类型的protobuf消息，维护本地vxlan路由、ipam block及vtep隧道端点，基于修改状态标记同步及路由刷新。

func (m *vxlanManager) OnUpdate(protoBufMsg interface{}) {
	switch msg := protoBufMsg.(type) {
    // route update
	case *proto.RouteUpdate:
		// 基于正确的IP Version消费Msg
		cidr, err := ip.CIDRFromString(msg.Dst)
		...
		if m.ipVersion != cidr.Version() {
			// Skip since the update is for a mismatched IP version
			return
		}

		// 先清理dst route
		m.deleteRoute(msg.Dst)

        // remote node workload
		if msg.Type == proto.RouteType_REMOTE_WORKLOAD && msg.IpPoolType == proto.IPPoolType_VXLAN {
            // 标记同步
			m.routesByDest[msg.Dst] = msg
			m.routesDirty = true
		}

        // local vxlan block(排除/32或/128或localWorkload Pod)
		if routeIsLocalVXLANBlock(msg) {
			m.localIPAMBlocks[msg.Dst] = msg
			m.routesDirty = true
        // 清理local vxlan block
		} else if _, ok := m.localIPAMBlocks[msg.Dst]; ok {
			delete(m.localIPAMBlocks, msg.Dst)
			m.routesDirty = true
		}
	// route remove
	case *proto.RouteRemove:
		// IP Version检查
		cidr, err := ip.CIDRFromString(msg.Dst)
		...
		if m.ipVersion != cidr.Version() {
			// Skip since the update is for a mismatched IP version
			return
		}
        
        // 清理路由
		m.deleteRoute(msg.Dst)
    // vxlan.calico更新
	case *proto.VXLANTunnelEndpointUpdate:
		// IP Version检查
		if (m.ipVersion == 4 && msg.Ipv4Addr == "") || (m.ipVersion == 6 && msg.Ipv6Addr == "") {
			return
		}

        // local node vtep
		if msg.Node == m.hostname {
			m.setLocalVTEP(msg)
        // remote node vtep
		} else {
			m.vtepsByNode[msg.Node] = msg
		}
        
        // 标记更新
		m.routesDirty = true
		m.vtepsDirty = true
    // vxlan.calico移除
	case *proto.VXLANTunnelEndpointRemove:
        // local node vtep
		if msg.Node == m.hostname {
			m.setLocalVTEP(nil)
        // remote node vtep
		} else {
			delete(m.vtepsByNode, msg.Node)
		}
        
        // 标记更新
		m.routesDirty = true
		m.vtepsDirty = true
	}
}

func (m *vxlanManager) deleteRoute(dst string) {
	_, exists := m.routesByDest[dst]
	if exists {
		// In case the route changes type to one we no longer care about...
		delete(m.routesByDest, dst)
		m.routesDirty = true
	}

	if _, exists := m.localIPAMBlocks[dst]; exists {
		delete(m.localIPAMBlocks, dst)
		m.routesDirty = true
	}
}

注意

m.OnUpdate()会维护dst->route及node->vtep映射关系，route message基于workload endpoint及ipam block计算

# 3.2.deferwork

m.CompleteDeferredWork()将vtep/pod路由变化同步到L2/L3 route table，配置L2 route、vxlan route及no-encap route。

func (m *vxlanManager) CompleteDeferredWork() error {
    // 未更新
	if !m.routesDirty {
		return nil
	}

    // vtep变化
	if m.vtepsDirty {
		...
        
        // 合法的vxlan udp包源地址
		if m.vtepsDirty {
			allowedVXLANSources = append(allowedVXLANSources, m.externalNodeCIDRs...)
		}
		...
        
		// remote vtep
		for _, u := range m.vtepsByNode {
            // 解析mac
			mac, err := m.parseMacForIPVersion(u)
			...
            
            // L2 vxlan forwarding
			l2routes = append(l2routes, routetable.L2Target{
				VTEPMAC: mac,
				GW:      ip.FromString(addr),
				IP:      ip.FromString(parentDeviceIP),
			})
            
            // 合法的vxlan udp包源地址
			allowedVXLANSources = append(allowedVXLANSources, parentDeviceIP)
		}

        // 设置及更新L2 route
		m.routeTable.SetL2Routes(m.vxlanDevice, l2routes)
        // 设置及更新ipset
		m.ipsetsDataplane.AddOrReplaceIPSet(m.ipSetMetadata, allowedVXLANSources)
		m.vtepsDirty = false
	}

    // route变化
	if m.routesDirty {
        ...
        
		// 扫描dst route
		for _, r := range m.routesByDest {
            // 解析cidr
			cidr, err := ip.CIDRFromString(r.Dst)
			...

            // same subnet with local pod cidr.
			if r.GetSameSubnet() {
				if r.DstNodeIp == "" {
					continue
				}

                // no-encap route
				defaultRoute := routetable.Target{
					Type: routetable.TargetTypeNoEncap,
					CIDR: cidr,
					GW:   ip.FromString(r.DstNodeIp),
				}
				noEncapRoutes = append(noEncapRoutes, defaultRoute)
            // cross subnet
			} else {
				// remote node vtep
				vtep, ok := m.vtepsByNode[r.DstNodeName]
				...
                
                // vxlan route
				vxlanRoute := routetable.Target{
					Type: routetable.TargetTypeVXLAN,
					CIDR: cidr,
					GW:   ip.FromString(vtepAddr),
				}
				vxlanRoutes = append(vxlanRoutes, vxlanRoute)
			}
		}

        // 设置及更新vxlan route
		m.routeTable.SetRoutes(m.vxlanDevice, vxlanRoutes)
        // 设置及更新黑洞路由
		m.blackholeRouteTable.SetRoutes(routetable.InterfaceNone, m.blackholeRoutes())
        
        // 设置及更新no-encap route
		noEncapRouteTable := m.getNoEncapRouteTable()
		if noEncapRouteTable != nil {
            // local vtep parent
			parentDevice, err := m.getLocalVTEPParent()
            ...
            
            ifName := parentDevice.Attrs().Name
            noEncapRouteTable.SetRoutes(ifName, noEncapRoutes)
            ...
		}
        ...

		m.routesDirty = false
	}

	return nil
}

注意

blackHole route禁止本地Pod流量发到外部，ipset rule限制vxlan可访问的sourceIP