policyMgr | 南风未起｜云原生/Go技术分享博客

# 1.简介

# 1.1.policy

network policy提供基于策略的网络控制，用于隔离应用减少攻击面。network policy需要网络插件监测策略和Pod变更及配置Pod流量。
注意

policy/profile均可以配置策略，相对来说policy更加细致，优先级更高

# 1.2.定义

new(RawEgress)PolicyManager()会实例化policyManager，将policy/profile转为rule写入raw/mangle/filter chain。

// simply renders policy/profile updates into iptables.Chain objects and sends them to the dataplane layer.
type policyManager struct {
	rawTable       iptablesTable						// raw table chain
	mangleTable    iptablesTable						// mangle table chain
	filterTable    iptablesTable						// filter table chain
	ruleRenderer   policyRenderer						// policy渲染为iptables rule
	...
	rawEgressOnly  bool									// 标记仅处理raw table engress流量
	neededIPSets   map[proto.PolicyID]set.Set[string]	// policy关联的ipset
	ipSetsCallback func(neededIPSets set.Set[string])	// 上报ipset的callback-->ipSetsV4.SetFilter
}

// iptables dataplane policy manager
func newPolicyManager(raw, mangle, filter iptablesTable, ruleRenderer policyRenderer...) *policyManager {
	return &policyManager{
		rawTable:     rawTable,
		mangleTable:  mangleTable,
		filterTable:  filterTable,
		ruleRenderer: ruleRenderer,
		ipVersion:    ipVersion,
	}
}

// eBPF dataplane policy manager
func newRawEgressPolicyManager(raw iptablesTable, ruleRenderer policyRenderer, ...,
	ipSetsCallback func(neededIPSets set.Set[string])) *policyManager {
	return &policyManager{
		rawTable:       rawTable,
		mangleTable:    &noopTable{},
		filterTable:    &noopTable{},
		ruleRenderer:   ruleRenderer,
		ipVersion:      ipVersion,
		rawEgressOnly:  true,
		neededIPSets:   make(map[proto.PolicyID]set.Set[string]),
		ipSetsCallback: ipSetsCallback,
	}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

注意

initDataplane会初始化policyManager，iptables/eBPF模式会分别实例化以上两类policyManager

# 2.处理

# 2.1.callback

m.OnUpdate()会将policy/profile渲染为iptables chains，eBPF模式仅保留outbound chain及维护外部ipset依赖关系。

func (m *policyManager) OnUpdate(msg interface{}) {
	switch msg := msg.(type) {
    // policy更新
	case *proto.ActivePolicyUpdate:
        // eBPF模式&未支持untracked(裁剪)
		if m.rawEgressOnly && !msg.Policy.Untracked {
			return
		}

        // policy渲染为iptables rule
		chains := m.ruleRenderer.PolicyToIptablesChains(msg.Id, msg.Policy, m.ipVersion)
        
        // eBPF模式,仅保留oubound chain
		if m.rawEgressOnly {
            // ipset收集器
			neededIPSets := set.New[string]()
            // 过滤的chains
			filteredChains := []*iptables.Chain(nil)
			for _, chain := range chains {
                // 匹配outbound chain
				if strings.Contains(chain.Name, string(rules.PolicyOutboundPfx)) {
					filteredChains = append(filteredChains, chain)
					neededIPSets.AddAll(chain.IPSetNames())
				}
			}
            // 更新chains及
			chains = filteredChains
            // 更新m.neededIPSets及执行ipSetsV4.SetFilter
			m.mergeNeededIPSets(msg.Id, neededIPSets)
		}
        
		// 更新raw/mangle/filter chains
		m.rawTable.UpdateChains(chains)
		m.mangleTable.UpdateChains(chains)
		m.filterTable.UpdateChains(chains)
    // policy移除
	case *proto.ActivePolicyRemove:
        // eBPF模式,执行ipSetsV4.SetFilter清理ipset
		if m.rawEgressOnly {
			m.mergeNeededIPSets(msg.Id, nil)
		}
        
        // raw/mangle/filter chain清理
		inName := rules.PolicyChainName(rules.PolicyInboundPfx, msg.Id)
		outName := rules.PolicyChainName(rules.PolicyOutboundPfx, msg.Id)
		// As above, we need to clean up in all the tables.
		m.filterTable.RemoveChainByName(inName)
		m.filterTable.RemoveChainByName(outName)
		m.mangleTable.RemoveChainByName(inName)
		m.mangleTable.RemoveChainByName(outName)
		m.rawTable.RemoveChainByName(inName)
		m.rawTable.RemoveChainByName(outName)
    // profile更新
	case *proto.ActiveProfileUpdate:
        // eBPF模式不处理profile
		if m.rawEgressOnly {
			return
		}
        
        // 将profile转为iptables chains
		inbound, outbound := m.ruleRenderer.ProfileToIptablesChains(msg.Id, msg.Profile, m.ipVersion)
        // 更新mangle/filter chains
		m.filterTable.UpdateChains([]*iptables.Chain{inbound, outbound})
		m.mangleTable.UpdateChains([]*iptables.Chain{outbound})
    // profile移除
	case *proto.ActiveProfileRemove:
        // 清理mangle/filter chains
		inName := rules.ProfileChainName(rules.ProfileInboundPfx, msg.Id)
		outName := rules.ProfileChainName(rules.ProfileOutboundPfx, msg.Id)
		m.filterTable.RemoveChainByName(inName)
		m.filterTable.RemoveChainByName(outName)
		m.mangleTable.RemoveChainByName(outName)
	}
}

func (s *IPSets) SetFilter(ipSetNames set.Set[string]) {
    // ipset更新标记
	markDirty := func(ipSetName string) {
		if ipSet := s.mainIPSetNameToIPSet[ipSetName]; ipSet != nil {
			s.dirtyIPSetIDs.Add(ipSet.SetID)
		}
	}
    
    // 剔除的ipset标记
	if s.neededIPSetNames != nil {
		s.neededIPSetNames.Iter(func(item string) error {
			if ipSetNames != nil && !ipSetNames.Contains(item) {
				// Name was needed before and now isn't, so mark as dirty.
				markDirty(item)
			}
			return nil
		})
	}
    
    // 新增的ipset标记
	if ipSetNames != nil {
		ipSetNames.Iter(func(item string) error {
			if s.neededIPSetNames != nil && !s.neededIPSetNames.Contains(item) {
				// Name wasn't needed before and now is, so mark as dirty.
				markDirty(item)
			}
			return nil
		})
	}
    
    // 更新待更新ipset快照
	s.neededIPSetNames = ipSetNames
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

注意

eBPF模式会将ipset生命周期转交给ipsetsMgr及仅保留outbound chain

# 2.2.render

r.Policy{Profile}ToIptablesChains()会将对象定义渲染为iptables chain+rule，生成的inbound/outbound chain写入table。

func (r *DefaultRuleRenderer) PolicyToIptablesChains(policyID *proto.PolicyID, policy *proto.Policy...) ... {
	// 渲染inbound chain
    inbound := iptables.Chain{
        // cali-pi-xxxx chain
		Name:  PolicyChainName(PolicyInboundPfx, policyID),
        // inbound rule渲染
		Rules: r.ProtoRulesToIptablesRules(policy.InboundRules, ipVersion, ..., policyID.Name)),
	}
    // 渲染outbound chain
	outbound := iptables.Chain{
        // cali-po-xxxx chain
		Name:  PolicyChainName(PolicyOutboundPfx, policyID),
        // outbound rule渲染
		Rules: r.ProtoRulesToIptablesRules(policy.OutboundRules, ipVersion, ..., policyID.Name)),
	}
	return []*iptables.Chain{&inbound, &outbound}
}

func (r *DefaultRuleRenderer) ProfileToIptablesChains(id *proto.ProfileID, profile *proto.Profile...) (...) {
	inbound = &iptables.Chain{
		Name:  ProfileChainName(ProfileInboundPfx, id),
		Rules: r.ProtoRulesToIptablesRules(profile.InboundRules, ipVersion,..., id.Name)),
	}
	outbound = &iptables.Chain{
		Name:  ProfileChainName(ProfileOutboundPfx, id),
		Rules: r.ProtoRulesToIptablesRules(profile.OutboundRules, ..., id.Name)),
	}
	return
}

func (r *DefaultRuleRenderer) ProtoRulesToIptablesRules(protoRules []*proto.Rule, ...) ...{
	...
    
    // policy rule转为iptables rule
	for _, protoRule := range protoRules {
		rules = append(rules, r.ProtoRuleToIptablesRules(protoRule, ipVersion)...)
	}
    
    // chain comment
	if len(chainComments) > 0 {
		if len(rules) == 0 {
			rules = append(rules, iptables.Rule{})
		}
		rules[0].Comment = append(rules[0].Comment, chainComments...)
	}
	return rules
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

注意

ProtoRulesToIptablesRules会基于policy/profile inbound/outbound rules渲染iptables chain/rule

# 2.3.chainrule

r.ProtoRuleToIptablesRules()会将policy/profile高级网络策略声明渲染为Netfilter可识别的iptables chain rule。

func (r *DefaultRuleRenderer) ProtoRuleToIptablesRules(pRule *proto.Rule, ipVersion uint8) []iptables.Rule {

    // rule协议筛选
	ruleCopy := FilterRuleToIPVersion(ipVersion, pRule)
	if ruleCopy == nil {
		return nil
	}
    
    // match状态机
	matchBlockBuilder := matchBlockBuilder{
		markAllBlocksPass: r.IptablesMarkScratch0,
		markThisBlockPass: r.IptablesMarkScratch1,
	}

	// ipset配置加载
	var ipSetConfig *ipsets.IPVersionConfig
	if ipVersion == 4 {
		ipSetConfig = r.IPSetConfigV4
	} else {
		ipSetConfig = r.IPSetConfigV6
	}
    
    // src port基于iptables限制切割为多个长度15的集合
	srcPortSplits := SplitPortList(ruleCopy.SrcPorts)
    // 多条multiport规则
	if len(srcPortSplits)+len(ruleCopy.SrcNamedPortIpSetIds) > 1 {
		// render a block for the source ports.
		matchBlockBuilder.AppendPortMatchBlock(ipSetConfig, ruleCopy.Protocol, srcPortSplits, ruleCopy.SrcNamedPortIpSetIds, src)
		// And remove them from the rule since they're already handled.
		ruleCopy.SrcPorts = nil
		ruleCopy.SrcNamedPortIpSetIds = nil
	}
    
    // dst port基于iptables限制切割为多个长度15的集合
	dstPortSplits := SplitPortList(ruleCopy.DstPorts)
    // 多条multiport规则
	if len(dstPortSplits)+len(ruleCopy.DstNamedPortIpSetIds) > 1 {
		// render a block for the destination ports.
		matchBlockBuilder.AppendPortMatchBlock(ipSetConfig, ruleCopy.Protocol, dstPortSplits, ruleCopy.DstNamedPortIpSetIds, dst)
		// And remove them from the rule since they're already handled.
		ruleCopy.DstPorts = nil
		ruleCopy.DstNamedPortIpSetIds = nil
	}

	// render srcNet rule
	if len(ruleCopy.SrcNet) > 1 {
		matchBlockBuilder.AppendCIDRMatchBlock(ruleCopy.SrcNet, src)
		// Since we're using a block for this, nil out the match.
		ruleCopy.SrcNet = nil
	}
    
    // render dstNet rule
	if len(ruleCopy.DstNet) > 1 {
		matchBlockBuilder.AppendCIDRMatchBlock(ruleCopy.DstNet, dst)
		// Since we're using a block for this, nil out the match.
		ruleCopy.DstNet = nil
	}
	// render not srcNet rule
	totalSrcMatches := len(ruleCopy.SrcNet) + len(ruleCopy.NotSrcNet)
	if totalSrcMatches > 1 {
		// We have some negated source CIDR matches and the total number of source
		// CIDR matches won't fit in the rule.  Render a block of rules to do the
		// negated match.
		matchBlockBuilder.AppendNegatedCIDRMatchBlock(ruleCopy.NotSrcNet, src)
		// Since we're using a block for this, nil out the match.
		ruleCopy.NotSrcNet = nil
	}
    
    // render not dstNet rule
	totalDstMatches := len(ruleCopy.DstNet) + len(ruleCopy.NotDstNet)
	if totalDstMatches > 1 {
		// We have some negated dest CIDR matches and the total number of dest
		// CIDR matches won't fit in the rule.  Render a block of rules to do the
		// negated match.
		matchBlockBuilder.AppendNegatedCIDRMatchBlock(ruleCopy.NotDstNet, dst)
		// Since we're using a block for this, nil out the match.
		ruleCopy.NotDstNet = nil
	}

	// 渲染剥离block剩余的规则
	match := r.CalculateRuleMatch(ruleCopy, ipVersion)

    // 渲染过match block
	if matchBlockBuilder.UsingMatchBlocks {
		// match rule + block rule 均成功才放行
		match = match.MarkSingleBitSet(matchBlockBuilder.markAllBlocksPass)
	}
    
    // action拆分为markBit和action
    // mark accept + allow
    // mark pass + pass
    // no mark + deny
	markBit, actions := r.CalculateActions(ruleCopy, ipVersion)
	rs := matchBlockBuilder.Rules
	if markBit != 0 {
		// match rule + mark
		rs = append(rs, iptables.Rule{
			Match:  match,
			Action: iptables.SetMarkAction{Mark: markBit},
		})
        
        // match rule调整,基于mark转发
		match = iptables.Match().MarkSingleBitSet(markBit)
	}
    
    // match mark rule + action
	for _, action := range actions {
		rs = append(rs, iptables.Rule{
			Match:  match,
			Action: action,
		})
	}

	// render rule annotations as comments on each rule.
	for i := range rs {
		for k, v := range pRule.GetMetadata().GetAnnotations() {
			rs[i].Comment = append(rs[i].Comment, fmt.Sprintf("%s=%s", k, v))
		}
	}

	return rs
}

注意

policy声明超出iptables chain rule限制会拆解为多条chain rule，基于mark识别放行或丢弃