netiface
南风未起 2025-12-01 19:39:22 cni
# 1.分配
# 1.1.简介
calico-plugin向sandbox container创建路由、虚拟网卡及veth pair资源，将相关数据写入calico datastore数据库。
注意
calico-plugin遵循CNI标准接口，实现及注册了ADD/DEL命令
# 1.2.cmdAdd
calico.cmdAdd会加载CNI配置参数，创建veth pair及设置网卡相关参数和路由，网络相关资源会生成workloadendpoint对象及注册到存储。func cmdAdd(args *skel.CmdArgs) (err error) { ... // 1.解析CNI配置 conf := types.NetConf{} json.Unmarshal(args.StdinData, &conf) ... // 2.解析nodeName(/var/lib/calico/nodeName) nodename := utils.DetermineNodename(conf) // 3.WEPIdentifiers对象及初始化(podName/podNamespace/containerID/IfName) wepIDs, err := utils.GetIdentifiers(args, nodename) ... calicoClient, err := utils.CreateClient(conf) ... // 4.datastore未ready ci, err := calicoClient.ClusterInformation().Get(ctx, "default", options.GetOptions{}) ... if !*ci.Spec.DatastoreReady { return } ... // 5.获取对应前缀workloadEndpoint,一个pod对应一个workloadEndpoint wepIDs.Endpoint = "" // {node_name}-k8s-{strings.replace(pod_name, "-", "--")}-{wepIDs.Endpoint} wepPrefix, err := wepIDs.CalculateWorkloadEndpointName(true) ... endpoints := calicoClient.WorkloadEndpoints().List(options.ListOptions{wepPrefix, wepIDs.Namespace, true}) ... // 6.基于生成名称匹配workloadEndpoint for _, ep := range endpoints.Items { if wepIDs.WorkloadEndpointIdentifiers.NameMatches(ep.Name) { endpoint = &ep // Assign the WEP name to wepIDs' WEPName field. wepIDs.WEPName = endpoint.Name // Put the endpoint name from the matched WEP in the identifiers. wepIDs.Endpoint = ep.Spec.Endpoint break } } // 未匹配设置传入值 if endpoint == nil { wepIDs.Endpoint = args.IfName wepIDs.WEPName, err = wepIDs.CalculateWorkloadEndpointName(false) ... } ... // 7.If running under k8s then branch off into the kubernetes code, otherwise handle everything in this. if wepIDs.Orchestrator == api.OrchestratorKubernetes { result, err = k8s.CmdAddK8s(ctx, args, conf, *wepIDs, calicoClient, endpoint) ... } ... // Print result to stdout, in the format defined by the requested cniVersion. err = cnitypes.PrintResult(result, conf.CNIVersion) return }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
注意
比较核心的是
cmdAddK8s，负责容器网络资源分配，非K8S环境模块这里不再分析
# 1.3.cmdAddK8s
k8s.cmdAddK8s()向calico store注册workloadendpoint对象，激活veth pair网卡对，设置路由、交互ipam申请IP及设置网卡地址。// CmdAddK8s performs the "ADD" operation on a kubernetes pod Having kubernetes code in its own file avoids // polluting the mainline code. It's expected that the kubernetes case will // more special casing than the mainline code. func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, epIDs utils.WEPIdentifiers, calicoClient calicoclient.Interface, endpoint *libapi.WorkloadEndpoint) (*cniv1.Result, error) { ... // 1.linux/grpc dataplane d, err := dataplane.GetDataplane(conf, logger) ... // 2.hostlocal模式 if conf.IPAM.Type == "host-local" { // 2.1.calico支持subnet: usePodCidr声明,此时需替换为node.spec.cidr utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDRs) ... args.StdinData, err = json.Marshal(stdinData) ... // 2.2.ipam route解析 hlRoutes, ok := untypedRoutes.([]interface{}) ... for _, route := range hlRoutes { route := route.(map[string]interface{}) untypedDst, ok := route["dst"] ... dst, ok := untypedDst.(string) ... _, cidr, err := net.ParseCIDR(dst) ... routes = append(routes, cidr) } } // 3.基于default route调整路由 if len(routes) == 0 { routes = utils.DefaultRoutes } else if conf.IncludeDefaultRoutes { routes = append(utils.DefaultRoutes, routes...) } ... // 4.k8s policy if conf.Policy.PolicyType == "k8s" { // 4.1.获取namespace annotation annotNS, err := getK8sNSInfo(client, epIDs.Namespace) ... // 4.2.获取pod相关元数据 labels, annot, ports, profiles, generateName, serviceAccount, err = getK8sPodInfo(client, epIDs.Pod) ... // 4.3.calico-ipam模式 if conf.IPAM.Type == "calico-ipam" { // 4.3.1.取namespace annotation注入的ip池 v4pools = annotNS["cni.projectcalico.org/ipv4pools"] v6pools = annotNS["cni.projectcalico.org/ipv6pools"] // 4.3.2.取pod annotation注入的ip池 v4poolpod := annot["cni.projectcalico.org/ipv4pools"] if len(v4poolpod) != 0 { v4pools = v4poolpod } v6poolpod := annot["cni.projectcalico.org/ipv6pools"] if len(v6poolpod) != 0 { v6pools = v6poolpod } // 4.3.3.ip池更新到ipam配置 if len(v4pools) != 0 || len(v6pools) != 0 { json.Unmarshal(args.StdinData, &stdinData) ... if len(v4pools) > 0 { json.Unmarshal([]byte(v4pools), &v4PoolSlice) ... stdinData["ipam"].(map[string]interface{})["ipv4_pools"] = v4PoolSlice } if len(v6pools) > 0 { json.Unmarshal([]byte(v6pools), &v6PoolSlice) ... stdinData["ipam"].(map[string]interface{})["ipv6_pools"] = v6PoolSlice } newData, err := json.Marshal(stdinData) ... args.StdinData = newData } } } // 5.取pod annotation注入的直接申请的IP及申请ipam分配的IP ipAddrsNoIpam := annot["cni.projectcalico.org/ipAddrsNoIpam"] ipAddrs := annot["cni.projectcalico.org/ipAddrs"] // 6.交互ipam分配IP switch { // 6.1.未直接指定pod IP case ipAddrs == "" && ipAddrsNoIpam == "": // Call the IPAM plugin. result, err = utils.AddIPAM(conf, args, logger) ... // 6.2.禁止同时设置两类IP case ipAddrs != "" && ipAddrsNoIpam != "": // Can't have both ipAddrs and ipAddrsNoIpam annotations at the same time. return nil, e // 6.3.申请不经过IPAM的IP case ipAddrsNoIpam != "": // Validate that we're allowed to use this feature. if conf.IPAM.Type != "calico-ipam" || !conf.FeatureControl.IPAddrsNoIpam { return nil, e } // ipAddrsNoIpam annotation is set so bypass IPAM, and set the IPs manually. overriddenResult, err := overrideIPAMResult(ipAddrsNoIpam, logger) ... // 加载到results result, err = cniv1.NewResultFromResult(overriddenResult) ... // 6.4.申请经过IPAM的IP case ipAddrs != "": // Validate that we're allowed to use this feature. if conf.IPAM.Type != "calico-ipam" { return nil, e } // 重建场景先释放占用的IP if endpoint != nil { releaseIPAddrs(endpoint.Spec.IPNetworks, calicoClient, logger) ... } // 交互calico-ipam分配新申请的IP result, err = ipAddrsResult(ipAddrs, conf, args, logger) ... } ... // 7.初始化endpoint endpoint.Name = epIDs.WEPName endpoint.Namespace = epIDs.Namespace ... defer func() { // 出错交互IPAM释放IP if err!=nil { utils.ReleaseIPAllocation(logger, conf, args) } } // 8.ipam结果写入endpoint utils.PopulateEndpointNets(endpoint, result) ... // 9.生成hostVethName{cali+hash(namespace.name)[:11]} dvName := k8sconversion.NewConverter().VethNameForWorkload(epIDs.Namespace, epIDs.Pod) // 10.接入网络栈(vepair创建及IP/Mac参数设置) hostVethName, contVethMac := d.DoNetworking(ctx, calicoClient, args, result, dvName, routes, endpoint,annot) ... // 11.回填结果至endpoint mac, err := net.ParseMAC(contVethMac) ... endpoint.Spec.MAC = mac.String() endpoint.Spec.InterfaceName = hostVethName endpoint.Spec.ContainerID = epIDs.ContainerID ... // 12.取pod annotation设置的floating ip floatingIPs := annot["cni.projectcalico.org/floatingIPs"] if floatingIPs != "" { // feature gate检查 if !conf.FeatureControl.FloatingIPs { return nil, fmt.Errorf("requested feature is not enabled: floating_ips") } // 解析floating ip列表 ips, err := parseIPAddrs(floatingIPs, logger) ... // ipam分配的podIP有效,设置nat映射 for _, ip := range ips { // IPV6 NAT if strings.Contains(ip, ":") { endpoint.Spec.IPNATs = append(endpoint.Spec.IPNATs, libapi.IPNAT{ InternalIP: podnetV6.IP.String(), ExternalIP: ip, }) // IPV4 NAT } else { endpoint.Spec.IPNATs = append(endpoint.Spec.IPNATs, libapi.IPNAT{ InternalIP: podnetV4.IP.String(), ExternalIP: ip, }) } } } ... // 13.更新或创建wep对象 utils.CreateOrUpdate(ctxPatchCNI, calicoClient, endpoint) ... // Add host interface created above to the CNI result. result.Interfaces = append(result.Interfaces, &cniv1.Interface{ Name: endpoint.Spec.InterfaceName}) return result, nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
注意
d.DoNetworking()是容器网络栈建设核心，会创建veth pair及路由，设置容器内网卡地址等参数
# 1.4.doNetwork
calico支持linuxDataplane和grpcDataplane两种模式，linux模式类似本地直接进行容器网络创建及配置，grpc模式委托给外部接口。func (d *linuxDataplane) DoNetworking(ctx context.Context, calicoClient calicoclient.Interface, args *skel.CmdArgs, result *cniv1.Result, desiredVethName string, routes []*net.IPNet, endpoint *api.WorkloadEndpoint, annotations map[string]string) (hostVethName,contVethMAC string,err error) { // hostVeth名称{cali+hash(namespace.name)[:11]} hostVethName = desiredVethName // containerVeth名称 contVethName := args.IfName // 1.清理已存在的旧hostVeth if oldHostVeth, err := netlink.LinkByName(hostVethName); err == nil { netlink.LinkDel(oldHostVeth) ... } // 2.切换到容器网络命名空间 err = ns.WithNetNSPath(args.Netns, func(hostNS ns.NetNS) error { la := netlink.NewLinkAttrs() la.Name = contVethName la.MTU = d.mtu la.NumTxQueues = d.queues la.NumRxQueues = d.queues veth := &netlink.Veth{ LinkAttrs: la, PeerName: hostVethName, } // 2.1.创建veth pair设备(eth0-->calixxx) netlink.LinkAdd(veth) ... // 2.2.获取hostVeth hostVeth, err := netlink.LinkByName(hostVethName) ... // 2.3.hostVeth设置固定mac(避免自动生成的不稳定) mac, err := net.ParseMAC("EE:EE:EE:EE:EE:EE") ... netlink.LinkSetHardwareAddr(hostVeth, mac) ... // 2.4.IPV6禁用DAD(检测冲突),避免短暂网络延迟 if hasIPv6 { // This must be done before we set the links UP. disableDAD(contVethName) ... disableDAD(hostVethName) ... } // 2.5.hostVeth激活 netlink.LinkSetUp(hostVeth) ... // 2.6.获取contVeth+激活contVeth contVeth, err := netlink.LinkByName(contVethName) ... netlink.LinkSetUp(contVeth) ... // 2.7.优先用Pod设置的Mac if requestedContVethMac, found := annotations["cni.projectcalico.org/hwAddr"]; found { tmpContVethMAC, err := net.ParseMAC(requestedContVethMac) ... netlink.LinkSetHardwareAddr(contVeth, tmpContVethMAC) ... contVethMAC = tmpContVethMAC.String() } else { contVethMAC = contVeth.Attrs().HardwareAddr.String() } // 2.8.IPV4路由 if hasIPv4 { // 直连路由 gw := net.IPv4(169, 254, 1, 1) gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)} // ip route add 169.254.1.1/32 dev eth0 scope link(绕过网关) netlink.RouteAdd(&netlink.Route{ LinkIndex: contVeth.Attrs().Index, Scope: netlink.SCOPE_LINK, Dst: gwNet}) ... // 其它配置的路由 for _, r := range routes { if r.IP.To4() == nil { continue } // ip route add <dst> via 169.254.1.1 dev eth0 ip.AddRoute(r, gw, contVeth) ... } } // 2.9.IPV6路由 if hasIPv6 { // 2.9.1.启用IPV6 writeProcSys("/proc/sys/net/ipv6/conf/all/disable_ipv6", "0") ... writeProcSys("/proc/sys/net/ipv6/conf/default/disable_ipv6", "0") ... writeProcSys("/proc/sys/net/ipv6/conf/lo/disable_ipv6", "0") ... // 2.9.2.阻塞至hostVeth出现IPV6 link-local地址 for i := 0; i < 10; i++ { addresses, err = netlink.AddrList(hostVeth, netlink.FAMILY_V6) ... if err == nil { break } time.Sleep(50 * time.Millisecond) } ... // 2.9.3.取hostVeth IPV6作为nextLoop+路由设置 hostIPv6Addr := addresses[0].IP for _, r := range routes { if r.IP.To4() != nil { continue } // ip -6 route add 2001:db8:10::/64 via fe80::1234 dev eth0 ip.AddRoute(r, hostIPv6Addr, contVeth) ... } } // 2.10.设置contVeth IP for _, addr := range result.IPs { netlink.AddrAdd(contVeth, &netlink.Addr{IPNet: &addr.Address}) ... } // 2.11.配置网络栈参数(forward转发) d.configureContainerSysctls(hasIPv4, hasIPv6) ... // 2.12.hostVeth移动到hostNS netlink.LinkSetNsFd(hostVeth, int(hostNS.Fd())) ... return nil }) ... // 3.配置hostVeth网络栈参数(forward/rp_filter..) d.configureSysctls(hostVethName, hasIPv4, hasIPv6) ... // 4.重新激活hostVeth(网络命名空间调整状态会专为down) hostVeth, err := netlink.LinkByName(hostVethName) ... netlink.LinkSetUp(hostVeth) ... // 5.注册host route(ip route add <pod-ip>/32 dev caliXXXX scope link) SetupRoutes(hostVeth, result) ... return hostVethName, contVethMAC, err }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
注意
veth pair创建及设置网络栈参数后，会将hostVeth移动到主机网络命名空间作为容器内流量出口，hostVeth出去的流量直接走路由
# 2.回收
# 2.1.cmdDel
calico.cmdDel会加载CNI配置参数，回收workloadendpoint对象及veth pair设备，交互ipam plugin释放网卡IP。func cmdDel(args *skel.CmdArgs) (err error) { ... // 1.解析配置 json.Unmarshal(args.StdinData, &conf) ... // 2.解析nodeName nodeNameFile := "/var/lib/calico/nodename" if conf.NodenameFile != "" { nodeNameFile = conf.NodenameFile } // Determine which node name to use. nodename := utils.DetermineNodename(conf) ... // 3.生成wep身份 epIDs, err = utils.GetIdentifiers(args, nodename) ... // 4.检测datastore状态 ci, err = calicoClient.ClusterInformation().Get(ctx, "default", options.GetOptions{}) ... if !*ci.Spec.DatastoreReady { return } // 5.生成wep名称——{node_name}-k8s-{strings.replace(pod_name, "-", "--")}-{wepIDs.Endpoint} epIDs.WEPName, err = epIDs.CalculateWorkloadEndpointName(false) ... err = k8s.CmdDelK8s(ctx, calicoClient, *epIDs, args, conf, logger) return } // CmdDelK8s performs CNI DEL processing when running under Kubernetes. func CmdDelK8s(ctx context.Context, c calicoclient.Interface, epIDs utils.WEPIdentifiers, ...) error { // 1.linuxDataplane d, err := dataplane.GetDataplane(conf, logger) ... // 2.资源清理 for attempts := 5; attempts >= 0; attempts-- { // 2.1.获取wep对象 wep, err := c.WorkloadEndpoints().Get(ctx, epIDs.Namespace, epIDs.WEPName, options.GetOptions{}) // 2.2.清理关联的wep对象 if wep.Spec.ContainerID != "" && args.ContainerID == wep.Spec.ContainerID { c.WorkloadEndpoints().Delete(ctx,wep.Namespace,wep.Name, options.DeleteOptions{ResourceVersion: wep.ResourceVersion, UID: &wep.UID}) ... } ... } // 3.清理veth pair d.CleanUpNamespace(args) ... // 4.交互ipam插件释放IP utils.DeleteIPAM(conf, args, logger) ... return nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
注意
网络回收流程类似
bridge plugin，卸载网卡仅需清理一端，对端及相关路由自动回收，释放IP也是交互ipam plugin