admission准入
# 1.简介
# 1.1.admission
admission准入参数初始化基于command options,加载的命令行参数会通过applyTo应用到genericConfig,最终注册为准入插件。// NewAPIServerCommand creates a *cobra.Command object with default parameters func NewAPIServerCommand() *cobra.Command { s := options.NewServerRunOptions() ... } // NewServerRunOptions creates a new ServerRunOptions object with default parameters func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ ... Admission: kubeoptions.NewAdmissionOptions(), ... } ... return &s } // NewAdmissionOptions creates a new instance of AdmissionOptions. func NewAdmissionOptions() *AdmissionOptions { // 初始化AdmissionOptions options := genericoptions.NewAdmissionOptions() // 注册内置插件(安全策略、资源管理、证书管理、调度和节点限制...) RegisterAllAdmissionPlugins(options.Plugins) // 设置推荐的插件顺序 options.RecommendedPluginOrder = AllOrderedPlugins // 设置部分禁用的准入插件 options.DefaultOffPlugins = DefaultOffAdmissionPlugins() return &AdmissionOptions{ GenericAdmission: options, } } // NewAdmissionOptions creates a new instance of AdmissionOptions. func NewAdmissionOptions() *AdmissionOptions { options := &AdmissionOptions{ Plugins: admission.NewPlugins(), Decorators: admission.Decorators{admission.DecoratorFunc(admissionmetrics.WithControllerMetrics)}, // 推荐的插件执行顺序 RecommendedPluginOrder: []string{lifecycle,mutatingwebhook,validatingadmissionpolicy,validatingwebhook}, DefaultOffPlugins: sets.NewString(), } // 注册上述默认插件 server.RegisterAllAdmissionPlugins(options.Plugins) return options }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49注意
kube-apiserver准入控制支持的插件很多,最常用的一般是validatingwebhook和mutatingwebhook,用于管理动态注册的准入插件
# 1.2.build
applyTo会将AdmissionOptions加载的准入插件参数应用到config,进行准入插件的真正加载及初始化,将准入控制加入请求处理链中。// CreateServerChain creates the apiservers connected via delegation. func CreateServerChain(completedOptions completedServerRunOptions) (*aggregatorapiserver.APIAggregator, error) { kubeAPIServerConfig, serviceResolver, pluginInitializer, err := CreateKubeAPIServerConfig(completedOptions) ... return aggregatorServer, nil } // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them. func CreateKubeAPIServerConfig(s completedServerRunOptions) (...) { ... genericConfig, versionedInformers, serviceResolver, pluginInitializers, admissionPostStartHook, storageFactory, err := buildGenericConfig(s.ServerRunOptions, proxyTransport) ... return config, serviceResolver, pluginInitializers, nil } // BuildGenericConfig takes the master server options and produces the genericapiserver.Config. func buildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transport) (...) { ... admissionConfig := &kubeapiserveradmission.Config{ ExternalInformers: versionedInformers, LoopbackClientConfig: genericConfig.LoopbackClientConfig, CloudConfigFile: s.CloudProvider.CloudConfigFile, } // svc地址解析 serviceResolver = buildServiceResolver(s.EnableAggregatorRouting, genericConfig.LoopbackClientConfig.Host, versionedInformers) // 对象结构解析 schemaResolver := resolver.NewDefinitionsSchemaResolver(k8sscheme.Scheme, genericConfig.OpenAPIConfig.GetDefinitions) // 准入初始化回调 pluginInitializers, admissionPostStartHook, err = admissionConfig.New(proxyTransport, genericConfig.EgressSelector, serviceResolver, genericConfig.TracerProvider, schemaResolver) ... // 应用到genericConfig s.Admission.ApplyTo(genericConfig,versionedInformers,kubeClientConfig,featureGate,pluginInitializers...) ... return }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44注意
准入加载其实就是根据配置参数初始化准入插件,挂到通用配置上
# 1.3.plugin
admission初始化会读取文件中准入配置,进一步创建插件及加入admissionControl,admissionControl会挂到请求处理handler。// ApplyTo adds the admission chain to the server configuration. func (a *AdmissionOptions) ApplyTo(...) error { ... // 根据顺序梳理启用的插件 pluginNames := a.enabledPluginNames() // 由文件获取启用的插件配置(一般为空) pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme) ... // 构建admission initializer对象 genericInitializer := initializer.New(clientset, dynamicClient, informers, c.Authorization.Authorizer, features, c.DrainedNotify()) // 追加到pluginInitializers initializersChain := admission.PluginInitializers{genericInitializer} initializersChain = append(initializersChain, pluginInitializers...) // 生成admissionChain admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators) c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain) return nil } // NewFromPlugins returns an admission.Interface that will enforce admission control decisions of given plugins. func (ps *Plugins) NewFromPlugins(...) (Interface, error) { ... // 遍历启用插件 for _, pluginName := range pluginNames { // 加载插件配置 pluginConfig, err := configProvider.ConfigFor(pluginName) ... // 初始化插件 plugin, err := ps.InitPlugin(pluginName, pluginConfig, pluginInitializer) ... if plugin != nil { // 用decorator包装加入handlers(admissionmetrics.WithControllerMetrics) if decorator != nil { handlers = append(handlers, decorator.Decorate(plugin, pluginName)) } else { handlers = append(handlers, plugin) } // 记录到mutationPlugins/validationPlugins if _, ok := plugin.(MutationInterface); ok { mutationPlugins = append(mutationPlugins, pluginName) } if _, ok := plugin.(ValidationInterface); ok { validationPlugins = append(validationPlugins, pluginName) } } } ... return newReinvocationHandler(chainAdmissionHandler(handlers)), nil } // InitPlugin creates an instance of the named interface. func (ps *Plugins) InitPlugin(...) (Interface, error) { ... // 获取注册的插件 plugin, found, err := ps.getPlugin(name, config) ... // 初始化插件(webhookInitializer-->kubeInitializer-->pluginInitializer) pluginInitializer.Initialize(plugin) // 执行plugin.ValidateInitialization()进行验证 ValidateInitialization(plugin) ... return plugin, nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79注意
admission初始化就是基于配置执行注册的plugin回调,所有的plugin转为chainHandler再包装为reinvoker统一准入调度
# 1.4.admissionChain
准入插件初始化会转为
chainAdmissionHandler,将handler拆分为mutating及validating实现执行真正的plugin准入函数。// chainAdmissionHandler is an instance of admission.NamedHandler that performs admission control. type chainAdmissionHandler []Interface // Admit performs an admission control check using a chain of handlers, and returns immediately on first error func (admissionHandler chainAdmissionHandler) Admit(...) error { // 遍历所有handler for _, handler := range admissionHandler { // plugin handler不关心当前操作类型 if !handler.Handles(a.GetOperation()) { continue } // plugin实现MutationInterface,执行修改请求 if mutator, ok := handler.(MutationInterface); ok { mutator.Admit(ctx, a, o) ... } } return nil } // Validate performs an admission control check using a chain of handlers. func (admissionHandler chainAdmissionHandler) Validate(...) error { // 遍历所有handler for _, handler := range admissionHandler { // plugin handler不关心当前操作类型 if !handler.Handles(a.GetOperation()) { continue } // plugin实现ValidationInterface,执行校验请求 if validator, ok := handler.(ValidationInterface); ok { validator.Validate(ctx, a, o) ... } } return nil } // Handles will return true if any of the handlers handles the given operation func (admissionHandler chainAdmissionHandler) Handles(operation Operation) bool { // 遍历 plugin handler for _, handler := range admissionHandler { // 检查是否关心当前操作类型 if handler.Handles(operation) { return true } } return false }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50补充
chainHandler作为plugin执行入口,外层根据plugin关心操作及实现接口执行对应实现
# 1.5.reinvoker
newReinvocationHandler()会封装admissionChain,基于原始插件链增加重入机制,实现对象修改后其它依赖插件的延迟调用。// chainAdmissionHandler is an instance of admission.NamedHandler that performs admission control. type chainAdmissionHandler []Interface // newReinvocationHandler creates a handler that wraps the provided admission chain and reinvokes it. func newReinvocationHandler(admissionChain Interface) Interface { return &reinvoker{admissionChain} } // 修改请求 func (r *reinvoker) Admit(ctx context.Context, a Attributes, o ObjectInterfaces) error { // admissionChain实现MutationInterface if mutator, ok := r.admissionChain.(MutationInterface); ok { // 执行admissionChain修改请求 mutator.Admit(ctx, a, o) ... // 需重入 s := a.GetReinvocationContext() if s.ShouldReinvoke() { // 标记已重入调用,避免无限递归 s.SetIsReinvoke() // 再次执行修改请求 return mutator.Admit(ctx, a, o) } } return nil } // 校验请求 func (r *reinvoker) Validate(ctx context.Context, a Attributes, o ObjectInterfaces) error { // admissionChain实现ValidationInterface,执行校验 if validator, ok := r.admissionChain.(ValidationInterface); ok { return validator.Validate(ctx, a, o) } return nil } // 检查admissionChain是否关心当前操作类型 func (r *reinvoker) Handles(operation Operation) bool { return r.admissionChain.Handles(operation) }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43补充
reinvoker对chainHandler又进行一次包装,其实还是检查chainHandler实现的接口执行对应实现,只是在Admit阶段加入冲入检查
# 2.实现
# 2.1.svcaccount
pod创建未指定serviceAccount会默认关联serviceAccount和对应的secrets,这种默认关联的维护就是serviceAccount插件实现的。// Register registers a plugin func Register(plugins *admission.Plugins) { plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { // 初始化serviceAccount准入插件 serviceAccountAdmission := NewServiceAccount() return serviceAccountAdmission, nil }) } // NewServiceAccount return an admission.Interface implementation which limits admission of Pod CREATE requests. func NewServiceAccount() *Plugin { return &Plugin{ Handler: admission.NewHandler(admission.Create, admission.Update), LimitSecretReferences: false, MountServiceAccountToken: true, generateName: names.SimpleNameGenerator.GenerateName, } } // Admit verifies if the pod should be admitted func (s *Plugin) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) (err error) { // 处理pod或pod ephemeralcontainers子资源,其它不处理 if shouldIgnore(a) { return nil } // 只处理create操作 if a.GetOperation() != admission.Create { // we only mutate pods during create requests return nil } // 转为pod pod := a.GetObject().(*api.Pod) // mirror pod由只是由kubelet接管,不是由其创建,不能修改spec,仅执行校验 if _, isMirrorPod := pod.Annotations[api.MirrorPodAnnotationKey]; isMirrorPod { return s.Validate(ctx, a, o) } // pod未挂载sa if len(pod.Spec.ServiceAccountName) == 0 { // 设置为default sa pod.Spec.ServiceAccountName = DefaultServiceAccountName } // 获取sa serviceAccount, err := s.getServiceAccount(a.GetNamespace(), pod.Spec.ServiceAccountName) ... // pod未挂载 if s.MountServiceAccountToken && shouldAutomount(serviceAccount, pod) { // 基于sa刷新pod volume配置 s.mountServiceAccountToken(serviceAccount, pod) } // 未指定镜像拉取密钥 if len(pod.Spec.ImagePullSecrets) == 0 { // 将sa的imagePullSecret挂进去 pod.Spec.ImagePullSecrets = make([]api.LocalObjectReference, len(serviceAccount.ImagePullSecrets)) for i := 0; i < len(serviceAccount.ImagePullSecrets); i++ { pod.Spec.ImagePullSecrets[i].Name = serviceAccount.ImagePullSecrets[i].Name } } // 执行校验 return s.Validate(ctx, a, o) } // Validate the data we obtained func (s *Plugin) Validate(ctx context.Context,a admission.Attributes,o admission.ObjectInterfaces) (err error) { // 处理pod或pod ephemeralcontainers子资源,其它不处理 if shouldIgnore(a) { return nil } // 转为pod pod := a.GetObject().(*api.Pod) // pod ephemeralcontainers子资源更新 if a.GetOperation() == admission.Update && a.GetSubresource() == "ephemeralcontainers" { // 限制临时容器引用的secret return s.limitEphemeralContainerSecretReferences(pod, a) } // 不是创建请求不处理 if a.GetOperation() != admission.Create { // we only validate pod specs during create requests return nil } // mirror pod处理 if _, isMirrorPod := pod.Annotations[api.MirrorPodAnnotationKey]; isMirrorPod { // 不允许挂sa if len(pod.Spec.ServiceAccountName) != 0 { return admission.NewForbidden(a, fmt.Errorf("a mirror pod may not reference service accounts")) } // 不允许引用secret hasSecrets := false podutil.VisitPodSecretNames(pod, func(name string) bool { hasSecrets = true return false }, podutil.AllContainers) if hasSecrets { return admission.NewForbidden(a, fmt.Errorf("a mirror pod may not reference secrets")) } // 不允许挂载sa投影卷 for _, v := range pod.Spec.Volumes { if proj := v.Projected; proj != nil { for _, projSource := range proj.Sources { if projSource.ServiceAccountToken != nil { return admission.NewForbidden(a, fmt.Errorf("a mirror pod may not use ServiceAccountToken volume projections")) } } } } return nil } // 其它请求,不允许未挂载sa if len(pod.Spec.ServiceAccountName) == 0 { return admission.NewForbidden(a, fmt.Errorf("no service account specified for pod %s/%s", a.GetNamespace(), pod.Name)) } // 获取sa serviceAccount, err := s.getServiceAccount(a.GetNamespace(), pod.Spec.ServiceAccountName) ... // secret强制挂载(plugin开启强制挂载/sa开启kubernetes.io/enforce-mountable-secrets注解) if s.enforceMountableSecrets(serviceAccount) { // 验证pod只能挂载sa关联的secret/imagePullSecret if err := s.limitSecretReferences(serviceAccount, pod); err != nil { return admission.NewForbidden(a, err) } } return nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137补充
sa admission准入有两种实现,早期基于Secret挂载Token,v1.24版本后基于projected映射卷和tokenRequest动态签发
# 2.2.lifecycle
namespaceLifecycle准入插件用于控制namespace准入检查,禁止删除kube-system、kube-public、default这些系统命名空间。// Register registers a plugin func Register(plugins *admission.Plugins) { plugins.Register(PluginName, func(config io.Reader) (admission.Interface, error) { return NewLifecycle(sets.NewString("default", "kube-system", "kube-public")) }) } func newLifecycleWithClock(immortalNamespaces sets.String, clock utilcache.Clock) (*Lifecycle, error) { // LRU缓存 forceLiveLookupCache := utilcache.NewLRUExpireCacheWithClock(100, clock) return &Lifecycle{ Handler: admission.NewHandler(admission.Create, admission.Update, admission.Delete), immortalNamespaces: immortalNamespaces, forceLiveLookupCache: forceLiveLookupCache, }, nil } // Admit makes an admission decision based on the request attributes func (l *Lifecycle) Admit(ctx context.Context, a admission.Attributes, o admission.ObjectInterfaces) error { // 不允许删除default/kube-system/kube-public命名空间 if a.GetOperation() == admission.Delete && a.GetKind().GroupKind() == "Namespace" && l.immortalNamespaces.Has(a.GetName()) { return errors.NewForbidden(a.GetResource().GroupResource(), a.GetName(), fmt.Errorf("this namespace may not be deleted")) } // 忽略非namespace操作及未限制namespace的资源操作 if len(a.GetNamespace()) == 0 && a.GetKind().GroupKind() != "Namespace" { return nil } // namespace请求 if a.GetKind().GroupKind() == v1.SchemeGroupVersion.WithKind("Namespace").GroupKind() { // 删除行为,强制标记由apiserver读取状态 if a.GetOperation() == admission.Delete { l.forceLiveLookupCache.Add(a.GetName(), true, forceLiveLookupTTL) } // allow all operations to namespaces return nil } // 其它资源删除请求放行 if a.GetOperation() == admission.Delete { return nil } // accessReview请求放行(kubectl auth can-i watch services --as=system:serviceaccount:kube-system:coredns) if isAccessReview(a) { return nil } // 等待lifecycle插件缓存就绪 if !l.WaitForReady() { return admission.NewForbidden(a, fmt.Errorf("not yet ready to handle request")) } ... // 获取namespace namespace, err := l.namespaceLister.Get(a.GetNamespace()) ... // create请求,再等一会缓存 if !exists && a.GetOperation() == admission.Create { // 50ms延迟 time.Sleep(missingNamespaceWait) // 再次由缓存获取 namespace, err = l.namespaceLister.Get(a.GetNamespace()) ... } ... // 检查namespace删除状态 if _, ok := l.forceLiveLookupCache.Get(a.GetNamespace()); ok { // namespace删除,缓存还是Active状态 forceLiveLookup = exists && namespace.Status.Phase == v1.NamespaceActive } // apiserver直查 if !exists || forceLiveLookup { // 获取namespace namespace, err = l.client.CoreV1().Namespaces().Get(context.TODO(), a.GetNamespace(), metav1.GetOptions{}) ... } // 创建请求 if a.GetOperation() == admission.Create { // 不允许向正在删除的namespace写 if namespace.Status.Phase != v1.NamespaceTerminating { return nil } ... return admission.NewForbidden(...) } return nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95补充
lifecycle未实现Validate,基于Admit限制资源请求和namespace状态的删除操作
# 2.3.registry
addmission chainHandler会注册到请求的处理逻辑,整个调用链较深,忽略路由注册部分,最终会在installer.Install()注册到请求链上。// installAPIResources is a private method for installing the REST storage backing each api groupversionresource func (s *GenericAPIServer) installAPIResources(...) error { ... for _, groupVersion := range apiGroupInfo.PrioritizedVersions { ... // 执行注册 discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer) ... resourceInfos = append(resourceInfos, r...) } ... return nil } // InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container. func (g *APIGroupVersion) InstallREST(container *restful.Container) (...) { prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version) installer := &APIInstaller{ group: g, prefix: prefix, minRequestTimeout: g.MinRequestTimeout, } // 构造webservice apiResources, resourceInfos, ws, registrationErrors := installer.Install() ... // 注册webservice到container container.Add(ws) ... return aggregatedDiscoveryResources, removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors) } // Install handlers for API resources. func (a *APIInstaller) Install() (...) { ... // 初始化webservice ws := a.newWebService() ... // 遍历资源路径列表 for _, path := range paths { // 各资源向webservice注册handler apiResource, resourceInfo, err := a.registerResourceHandlers(path, a.group.Storage[path], ws) ... // 记录discovery apiResources if apiResource != nil { apiResources = append(apiResources, *apiResource) } // 记录storage apiResources if resourceInfo != nil { resourceInfos = append(resourceInfos, resourceInfo) } } return apiResources, resourceInfos, ws, errors } func (a *APIInstaller) registerResourceHandlers(...) (*metav1.APIResource,*storageversion.ResourceInfo,error) { // 先获取注册的admissionControl对象 admit := a.group.Admit ... for _, action := range actions { ... switch action.Verb { ... case "POST": // Create a resource. ... // 准入控制包装 if isNamedCreater { handler = restfulCreateNamedResource(namedCreater, reqScope, admit) } else { handler = restfulCreateResource(creater, reqScope, admit) } ... route := ws.POST(action.Path).To(handler)... ... routes = append(routes, route) ... } // route注册到webservice for _, route := range routes { ... ws.Route(route) } } ... return &apiResource, resourceInfo, nil } // wrapper admissionControl func restfulCreateResource(...) restful.RouteFunction { return func(req *restful.Request, res *restful.Response) { handlers.CreateResource(r, &scope, admit)(res.ResponseWriter, req.Request) } } // CreateResource returns a function that will handle a resource creation. func CreateResource(r rest.Creater, scope *RequestScope, admission admission.Interface) http.HandlerFunc { return createHandler(&namedCreaterAdapter{r}, scope, admission, false) } func createHandler(...) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { ... // 将后端存储的Create逻辑用匿名函数包装 requestFunc := func() (runtime.Object, error) { return r.Create( ctx, name, obj, rest.AdmissionToValidateObjectFunc(admit, admissionAttributes, scope), options, ) } ... // 将获取后端结果的调用基于FinishRequest包装 result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) { ... // 存储前先Admit,执行准入修改 if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) { if err := mutatingAdmission.Admit(ctx, admissionAttributes, scope); err != nil { return nil, err } } ... // 获取存储持久化结果 result, err := requestFunc() ... return result, err }) ... } }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152补充
admission注册到请求链本质是对installer动态注册的资源route handler包装,作用于认证/鉴权后,真正存储前