ipipMgr | 南风未起｜云原生/Go技术分享博客

# 1.简介

# 1.1.ipip

IPIP是一种IP封装协议，将IP数据包用外部的IP报头封装，外部报文的SrcIP是calico节点地址，DstIP是接收的calico节点地址。

--- ipipMgr
	1.创建和管理IPIP隧道设备
	2.基于IPIP隧道配置到达远程Pod的路由
	3.响应与IPIP相关的calico节点配置和IPPool设置的变化

1
2
3
4

注意

IPIP模式类似vxlan，无法直接路由的流量基于IPIP隧道交互

# 1.2.初始化

newIPIPManager()会实例化ipipMgr对象，维护nodeIP集合和配置IPIP隧道，确保跨节点流量利用IPIP隧道正常交互。

func newIPIPManager(ipsetsDataplane IPSetsDataplane, maxIPSetSize int, externalNodeCidrs []string) *ipipManager{
	return newIPIPManagerWithShim(ipsetsDataplane, maxIPSetSize, realIPIPNetlink{}, externalNodeCidrs)
}

func newIPIPManagerWithShim(ipsetsDataplane IPSetsDataplane, maxIPSetSize int, dataplane ipipDataplane,
externalNodeCIDRs []string) *ipipManager {
	ipipMgr := &ipipManager{
		ipsetsDataplane:    ipsetsDataplane,				// ipsetMgr
		activeHostnameToIP: map[string]string{},			// hostName->IP
		dataplane:          dataplane,						// netLink
		ipSetMetadata: ipsets.IPSetMetadata{		
			MaxSize: maxIPSetSize,							// 最大容量
			SetID:   rules.IPSetIDAllHostNets,				// 用途
            Type:    ipsets.IPSetTypeHashNet,				// hash:net
		},
		externalNodeCIDRs: externalNodeCIDRs,
	}
	return ipipMgr
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

注意

IPIP网络相关的route/device/ipset会基于ipipMgr维护

# 2.device

# 2.1.keepipip

d.KeepIPIPDeviceInSync()周期维护IPIP Tunl0设备，更新MTU/Addr/State相关参数，Tunl0找不到会自动修复，确保流量正常。

// configures the IPIP tunnel device, then periodically checks that it is still correctly configured.
func (d *ipipManager) KeepIPIPDeviceInSync(mtu int, address net.IP) {
	for {
        // IPIP隧道维护
		err := d.configureIPIPDevice(mtu, address)
		if err != nil {
			time.Sleep(1 * time.Second)
			continue
		}
		time.Sleep(10 * time.Second)
	}
}

// configureIPIPDevice ensures the IPIP tunnel device is up and configures correctly.
func (d *ipipManager) configureIPIPDevice(mtu int, address net.IP) error {
    // 尝试获取ipip tunl
	link, err := d.dataplane.LinkByName("tunl0")
	if err != nil {
        // 创建
		d.dataplane.RunCmd("ip", "tunnel", "add", "tunl0", "mode", "ipip")
		...
        
        // 再次检查
		link = d.dataplane.LinkByName("tunl0")
		...
	}
	...
    
    // MTU更新
	if oldMTU != mtu {
		d.dataplane.LinkSetMTU(link, mtu)
        ...
	}
    
    // 状态更新
	if attrs.Flags&net.FlagUp == 0 {
		d.dataplane.LinkSetUp(link)
        ...
	}

    // 设置addr
	d.setLinkAddressV4("tunl0", address)
    ...
    
	return nil
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

注意

IPIP Tunl0设备存在会基于配置状态更新MTU/Addr参数，不存在会重新创建

# 2.2.linkaddr

d.setLinkAddressV4()会基于上游对象配置的相关参数设置IPIP Tunl0地址，非预期地址会全部清理，IPIP隧道仅保留预期合法地址。

// setLinkAddressV4 updates the given link to set its local IP address.  It removes any other
// addresses.
func (d *ipipManager) setLinkAddressV4(linkName string, address net.IP) error {
	// 尝试获取ipip tun
	link, err := d.dataplane.LinkByName(linkName)
	...

    // 获取link addr
	addrs, err := d.dataplane.AddrList(link, netlink.FAMILY_V4)
	...

	found := false
	for _, oldAddr := range addrs {
		if address != nil && oldAddr.IP.Equal(address) {
			found = true
			continue
		}
		
        // 清理非预期地址
		d.dataplane.AddrDel(link, &oldAddr)
        ...
	}

    // 设置预期地址
	if !found && address != nil {
		mask := net.CIDRMask(32, 32)
		ipNet := net.IPNet{
			IP:   address.Mask(mask), // Mask the IP to match ParseCIDR()'s behaviour.
			Mask: mask,
		}
		addr := &netlink.Addr{
			IPNet: &ipNet,
		}
		d.dataplane.AddrAdd(link, addr)
        ...
	}

	return nil
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

注意

calico配置的IPIP Tunl0设备地址发生变化会自动更新，无效地址会全部清理

# 3.ipipMgr

# 3.1.opUpdate

d.OnUpdate()会处理dataplane推过来的message，基于event action更新activeHostnameToIP，以进行后续的IPIP网络维护。

func (d *ipipManager) OnUpdate(msg interface{}) {
	switch msg := msg.(type) {
    // 更新
	case *proto.HostMetadataUpdate:
		d.activeHostnameToIP[msg.Hostname] = msg.Ipv4Addr
		d.ipSetInSync = false
    // 清理
	case *proto.HostMetadataRemove:
		delete(d.activeHostnameToIP, msg.Hostname)
		d.ipSetInSync = false
	}
}

1
2
3
4
5
6
7
8
9
10
11
12

注意

activeHostnameToIP记录的hostName-->IP会由CompleteDeferredWork应用到IPIP网络

# 3.2.deferwork

m.CompleteDeferredWork()基于activeHostnameToIP生成最新的member注册到ipsetMgr，以更新IPIP隧道限制的srcIP集合。

func (m *ipipManager) CompleteDeferredWork() error {
	if !m.ipSetInSync {
        // 注册member(hostIP+externalNodeCIDR)
		members := make([]string, 0, len(m.activeHostnameToIP)+len(m.externalNodeCIDRs))
		for _, ip := range m.activeHostnameToIP {
			members = append(members, ip)
		}
		members = append(members, m.externalNodeCIDRs...)
        // 设置到ipsetMgr
        // ipset create node-ips hash:ip
        // ipset add node-ips 10.244.1.2
		// ipset add node-ips 10.244.2.3
		// ipset add node-ips 192.168.0.0/16
        // iptables -A INPUT -p ipip -m set --match-set AllHostNets src -m addrtype --dst-type LOCAL -j ACCEPT
		m.ipsetsDataplane.AddOrReplaceIPSet(m.ipSetMetadata, members)
		m.ipSetInSync = true
	}
    
	return nil
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

补充

这里更新的ipset规则会被dataplane初始化设置的iptables filter rule引用，将IPIP流量导入ipset匹配，检查合法的SrcIP来源