APIRoute
南风未起 2025-06-05 19:39:22 apiserver
# 1.存储
# 1.1.restoptions
restoptions基于apiserver启动加载的options构造，主要是一些etcd创建相关的回调及配置，后续的rest存储创建会用到这个对象。// NewServerRunOptions creates a new ServerRunOptions object with default parameters func NewServerRunOptions() *ServerRunOptions { s := ServerRunOptions{ ... // 绑定命令行参数,初始化etcd options Etcd: generic.NewEtcdOptions(storage.NewDefaultConfig(kubeoptions.DefaultEtcdPathPrefix, nil)), ... } ... return &s } // CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them func CreateKubeAPIServerConfig(s completedServerRunOptions) (...) { buildGenericConfig(s.ServerRunOptions, proxyTransport) ... return config, serviceResolver, pluginInitializers, nil } // BuildGenericConfig takes the master server options and produces the genericapiserver.Config. func buildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transport) (...) { ... // 构造storageFactory storageFactory, lastErr = storageFactoryConfig.Complete(s.Etcd).New() ... // etcd配置加载到通用配置 s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig) ... return } // ApplyWithStorageFactoryTo mutates the provided server.Config. func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error { ... // 用StorageFactoryRestOptionsFactory封装 c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory} return nil } func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (RESTOptions, error) { storageConfig, err := f.StorageFactory.NewConfig(resource) ... ret := generic.RESTOptions{ StorageConfig: storageConfig, Decorator: generic.UndecoratedStorage, // 修饰器,后续的etcd client创建 DeleteCollectionWorkers: f.Options.DeleteCollectionWorkers, EnableGarbageCollection: f.Options.EnableGarbageCollection, ResourcePrefix: f.StorageFactory.ResourcePrefix(resource), CountMetricPollPeriod: f.Options.StorageConfig.CountMetricPollPeriod, StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker, } // 默认开启 if f.Options.EnableWatchCache { ... // Decorator调整为带cacher实现 ret.Decorator = genericregistry.StorageWithCacher() } return ret, nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69补充
storage相关配置及初始化回调全部加载到restoptions，后续就可以基于该对象完成storage创建
# 1.2.rest
后端存储会经历三层封装，
rest对象绑定请求封装store，store基于storage做一层封装，storage负责真正的存储交互，区别在于path。// New returns a new instance of CustomResourceDefinitions from the given config. func (c completedConfig) New(delegation genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) { ... // 创建一个rest对象,内嵌store crdStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter) ... return s, nil } // NewREST returns a RESTStorage object that will work against API services. func NewREST(scheme *runtime.Scheme, optsGetter generic.RESTOptionsGetter) (*REST, error) { ... // 初始化store对象 store := &genericregistry.Store{...} options := &generic.StoreOptions{RESTOptions: optsGetter, AttrFunc: GetAttrs} // 补全store对象 store.CompleteWithOptions(options) ... return &REST{store}, nil } // CompleteWithOptions updates the store with the provided options and defaults common fields. func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { ... // optsGetter就是StorageFactoryRestOptionsFactory opts, err := options.RESTOptions.GetRESTOptions(e.DefaultQualifiedResource) ... if e.Storage.Storage == nil { e.Storage.Codec = opts.StorageConfig.Codec ... // 基于Decorator创建storage对象 e.Storage.Storage, e.DestroyFunc, err = opts.Decorator( opts.StorageConfig, prefix, keyFunc, e.NewFunc, e.NewListFunc, attrFunc, options.TriggerFunc, options.Indexers, ) ... e.StorageVersioner = opts.StorageConfig.EncodeVersioner ... } return nil } // Get retrieves the item from storage. func (e *Store) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { // 初始化查询对象 obj := e.NewFunc() // 查询索引 key, err := e.KeyFunc(ctx, name) ... // 执行查询 e.Storage.Get(ctx, key, storage.GetOptions{ResourceVersion: options.ResourceVersion}, obj) ... // 执行装饰器 if e.Decorator != nil { e.Decorator(obj) } return obj, nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71补充
rest绑定请求，基于内部store对象存储，store基于内部storage对象进行存储，层层包装
# 1.3.storage
storage的初始化基于restoptions，由restoptions.Decorator进行创建，Decorator本质是一个etcd client初始化模块。// CompleteWithOptions updates the store with the provided options and defaults common fields. func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error { ... if e.Storage.Storage == nil { e.Storage.Codec = opts.StorageConfig.Codec ... // 基于Decorator创建storage对象 e.Storage.Storage, e.DestroyFunc, err = opts.Decorator(...) ... } return nil } // UndecoratedStorage returns the given a new storage from the given config without any decoration. func UndecoratedStorage(...) (storage.Interface, factory.DestroyFunc, error) { return NewRawStorage(config, newFunc) } // NewRawStorage creates the low level kv storage. func NewRawStorage(config *storagebackend.ConfigForResource, newFunc func() runtime.Object) (...) { return factory.Create(*config, newFunc) } // Create creates a storage backend based on given config. func Create(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (...) { switch c.Type { ... // 创建etcd client入口 case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3: return newETCD3Storage(c, newFunc) default: return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type) } } func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (...) { // 开启历史版本压缩 stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval) ... // 初始化client client, err := newETCD3Client(c.Transport) ... // etcd存储占用监控 stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client, c.DBMetricPollInterval) ... // 资源释放回调 destroyFunc := func() { once.Do(func() { // 停止历史版本压缩 stopCompactor() // 存储占用监控 stopDBSizeMonitor() client.Close() }) } // 加密传输(secret加密) transformer := c.Transformer if transformer == nil { transformer = identity.NewEncryptCheckTransformer() } // etcd cleint基于store包装 return etcd3.New(client, c.Codec, newFunc, c.Prefix, c.GroupResource, transformer, c.Paging, c.LeaseManagerConfig), destroyFunc, nil } // New returns an etcd3 implementation of storage.Interface. func New(...) storage.Interface { return newStore(c, codec, newFunc, prefix, groupResource, transformer, pagingEnabled, leaseManagerConfig) } // New returns an etcd3 store implementation of storage.Interface. func newStore(...) *store { ... // 构造store,这是与etcd直接交互的对象,作为storage实现由外层store包装 result := &store{ client: c, codec: codec, versioner: versioner, transformer: transformer, pagingEnabled: pagingEnabled, pathPrefix: pathPrefix, groupResource: groupResource, groupResourceString: groupResource.String(), watcher: newWatcher(c, codec, groupResource, newFunc, versioner), leaseManager: newDefaultLeaseManager(c, leaseManagerConfig), } return result }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
补充
etcd client基于storage接口实现进行包装，客户端的请求就是基于rest->store->storage层层调用写入etcd
# 2.路由
# 2.1.installgroup
再来看
apiextensionserver创建，路由注册相关的部分就是将apiGroupInfo基于path和store注册到genericServer handler。// New returns a new instance of CustomResourceDefinitions from the given config. func (c completedConfig) New(delegation genericapiserver.DelegationTarget) (*CustomResourceDefinitions, error) { ... // 通用配置 apiResourceConfig := c.GenericConfig.MergedResourceConfig apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(apiextensions.GroupName, Scheme, metav1.ParameterCodec, Codecs) ... // 开启v1版本crd if apiResourceConfig.ResourceEnabled(v1.SchemeGroupVersion.WithResource("customresourcedefinitions")) { // 创建一个rest对象,内嵌store customResourceDefinitionStorage, err := customresourcedefinition.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter) ... // 记录对象及status子资源的rest存储 storage[resource] = customResourceDefinitionStorage storage[resource+"/status"] = customresourcedefinition.NewStatusREST(Scheme, customResourceDefinitionStorage) } // 追加到apiGroupInfo if len(storage) > 0 { apiGroupInfo.VersionedResourcesStorageMap[v1.SchemeGroupVersion.Version] = storage } // 注册路由 s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo) ... return s, nil } // InstallAPIGroup exposes the given api group in the API. func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error { // 复用InstallAPIGroups return s.InstallAPIGroups(apiGroupInfo) }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37补充
rest对象介入用户请求和后端存储之间，将用户请求以增删改查方式作用到etcd，这里先不分析
# 2.2.installapi
InstallAPIGroups()会遍历apiGroupInfo，将资源以path-->handler的route形式注册到webservice，ws最终挂到container。// InstallAPIGroups exposes given api groups in the API. func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) error { ... // 获取openapi模型数据,包含字段及参数的模型定义,基于make gen_openapi命令生成，全部数据在pkg/generated/openapi/ openAPIModels, err := s.getOpenAPIModels(APIGroupPrefix, apiGroupInfos...) ... // 遍历apiGroupInfo for _, apiGroupInfo := range apiGroupInfos { // 继续注册 s.installAPIResources(APIGroupPrefix, apiGroupInfo, openAPIModels) ... } return nil } // installAPIResources is a private method for installing the REST storage backing each api groupversionresource func (s *GenericAPIServer) installAPIResources(...) error { ... for _, groupVersion := range apiGroupInfo.PrioritizedVersions { ... // 初始化一个apiGroupVersion对象 apiGroupVersion, err := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix) ... apiGroupVersion.TypeConverter = typeConverter ... // 继续注册 discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer) ... resourceInfos = append(resourceInfos, r...) ... } // 注册删除回调 s.RegisterDestroyFunc(apiGroupInfo.destroyStorage) ... return nil } // InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container. func (g *APIGroupVersion) InstallREST(container *restful.Container) (...{ // /apis/apiextensions.k8s.io/v1 prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version) // 构造installer installer := &APIInstaller{ group: g, prefix: prefix, minRequestTimeout: g.MinRequestTimeout } // 继续安装构造webservice apiResources, resourceInfos, ws, registrationErrors := installer.Install() ... // ws注册到restContainer container.Add(ws) ... return aggregatedDiscoveryResources, removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors) }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62补充
整个调用链很长，承载
storage和路由信息的对象不断变化，依次经历APIGroupInfo、APIGroupVersion及APIInstaller
# 2.3.install
installer.Install()是真正的路由安装入口，依次会向各路由注册handler，这个handler其实是套了一层addmission准入的存储实现。// Install handlers for API resources. func (a *APIInstaller) Install() (...) { ... // 初始化一个webservice ws := a.newWebService() ... // 收集资源路由 for path := range a.group.Storage { paths[i] = path i++ } // 排序 sort.Strings(paths) // 遍历路由 for _, path := range paths { // 向ws注册route及相关handler apiResource, resourceInfo, err := a.registerResourceHandlers(path, a.group.Storage[path], ws) ... } return apiResources, resourceInfos, ws, errors } func (a *APIInstaller) registerResourceHandlers(...) (*metav1.APIResource,*storageversion.ResourceInfo,error) { ... // 分割资源名称 resource, subresource, err := splitSubresource(path) ... // 断言各种类型实现,实现了就可以生成对应handler creater, isCreater := storage.(rest.Creater) namedCreater, isNamedCreater := storage.(rest.NamedCreater) lister, isLister := storage.(rest.Lister) ... var apiResource metav1.APIResource ... // Get the list of actions for the given scope. switch { // 非命名空间级别资源(CRD) case !namespaceScoped: // Handle non-namespace scoped resources like nodes. resourcePath := resource resourceParams := params itemPath := resourcePath + "/{name}" nameParams := append(params, nameParam) proxyParams := append(nameParams, pathParam) suffix := "" if isSubresource { suffix = "/" + subresource itemPath = itemPath + suffix resourcePath = itemPath resourceParams = nameParams } apiResource.Name = path apiResource.Namespaced = false apiResource.Kind = resourceKind namer := handlers.ContextBasedNaming{ Namer: a.group.Namer, ClusterScoped: true, } ... // 基于路由生成action对象(/api/apiVersion/resource/{name}) actions = appendIf(actions, action{"GET", itemPath, nameParams, namer, false}, isGetter) ... default: ... } ... // 遍历支持的action for _, action := range actions { ... switch action.Verb { case "GET": // Get a resource. ... // 生成handler handler = restfulGetResource(getter, reqScope) ... // 基于action路由及handler生成route对象 route := ws.GET(action.Path).To(handler). Doc(doc). Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")). Operation("read"+namespaced+kind+strings.Title(subresource)+operationSuffix). Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...). Returns(http.StatusOK, "OK", producedObject). Writes(producedObject) ... // 注册路由参数 addParams(route, action.Params) routes = append(routes, route) ... } // 遍历所有路由 for _, route := range routes { // metadata记录GVK route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{ Group: reqScope.Kind.Group, Version: reqScope.Kind.Version, Kind: reqScope.Kind.Kind, }) // metadata记录Verb route.Metadata(ROUTE_META_ACTION, strings.ToLower(action.Verb)) // route注册到webservice ws.Route(route) } } ... return &apiResource, resourceInfo, nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124补充
registerResourceHandlers函数其实将近800行，这里只关注Get流程及路由注册，其它干扰项忽略
# 2.4.handler
handler是路由注册中很重要的一个对象，负责真正处理请求，handler本质上是基于etcd client执行存储交互，作用于请求与etcd中间。func restfulGetResource(r rest.Getter, scope handlers.RequestScope) restful.RouteFunction { return func(req *restful.Request, res *restful.Response) { handlers.GetResource(r, &scope)(res.ResponseWriter, req.Request) } } // GetResource returns a function that handles retrieving a single resource from a rest.Storage object. func GetResource(r rest.Getter, scope *RequestScope) http.HandlerFunc { return getResourceHandler(scope, func(ctx context.Context, name string, req *http.Request) (runtime.Object, error) { // check for export options := metav1.GetOptions{} // 获取请求参数 if vs := req.URL.Query(); len(values) > 0 { ... // 解析请求参数 metainternalversionscheme.ParameterCodec.DecodeParameters(vs,scope.MetaGroupVersion,&options) } ... // 执行rest.Get return r.Get(ctx, name, &options) }) } // getResourceHandler is an HTTP handler function for get requests. func getResourceHandler(scope *RequestScope, getter getterFunc) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { ... // 获取请求资源的name/namespace namespace, name, err := scope.Namer.Name(req) ... // 响应类型 outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope) ... // 查询对象 result, err := getter(ctx, name, req) ... // 对象序列化及响应 transformResponseObject(ctx, scope, req, w, http.StatusOK, outputMediaType, result) } }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48补充
不同请求的
handler其实都类似，相比于查询其它实现只是多出addmission准入检查，最终交互存储的还是rest(storage)对象
# 3.etcd
# 3.1.查询
get()相对其它实现简单一些，未包装addmission准入相关的一些逻辑，主要集中在查询时__internal版本数据解码为__outernal版本数据。// Get retrieves the item from storage. func (e *Store) Get(ctx context.Context, name string, options *metav1.GetOptions) (runtime.Object, error) { // 构造待填充对象(__internal) obj := e.NewFunc() // 构造name/namespace key, err := e.KeyFunc(ctx, name) ... // 执行Get查询 e.Storage.Get(ctx, key, storage.GetOptions{ResourceVersion: options.ResourceVersion}, obj) ... // 执行装饰器 if e.Decorator != nil { e.Decorator(obj) } return obj, nil } // Get retrieves the item from storage. func (s *DryRunnableStorage) Get(ctx Context, key string, opts storage.GetOptions, obj runtime.Object) error { return s.Storage.Get(ctx, key, opts, objPtr) } // Get implements storage.Interface.Get. func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, out runtime.Object) error { // 构造索引(path/key) preparedKey, err := s.prepareKey(key) ... // 查询etcd数据 getResp, err := s.client.KV.Get(ctx, preparedKey) ... // 查询Revision检查 s.validateMinimumResourceVersion(opts.ResourceVersion, uint64(getResp.Header.Revision)) ... // 获取键值对 kv := getResp.Kvs[0] // 敏感数据加密 data, _, err := s.transformer.TransformFromStorage(ctx, kv.Value, authenticatedDataString(preparedKey)) ... // 数据解码,__internal版本解码成客户端请求的版本 return decode(s.codec, s.versioner, data, out, kv.ModRevision) }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49注意
storage对etcd client的包装主要在编解码，持久化将数据转换为__internal版本的对象存储，获取将__internal版本解码为对外版本
# 3.2.创建
create()相对get()加入了addmission准入检查，还会开启etcd事务确保数据一致性，最终还是执行etcdCli.create完成对象持久化。func restfulCreateResource(...) restful.RouteFunction { return func(req *restful.Request, res *restful.Response) { handlers.CreateResource(r, &scope, admit)(res.ResponseWriter, req.Request) } } // CreateResource returns a function that will handle a resource creation. func CreateResource(r rest.Creater, scope *RequestScope, admission admission.Interface) http.HandlerFunc { return createHandler(&namedCreaterAdapter{r}, scope, admission, false) } // createHandler returns a function that will handle request. func createHandler(...) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { ... // 获取body body, err := limitedReadBodyWithRecordMetric(ctx, req, scope.MaxRequestBodyBytes,...) ... // 创建decoder(版本为__internal) decoder := scope.Serializer.DecoderToVersion(decodeSerializer, scope.HubGroupVersion) ... // 请求体解析为对应版本对象 obj, gvk, err := decoder.Decode(body, &defaultGVK, original) ... // 构造create方法,利用匿名函数包装以进行准入校验拦截 requestFunc := func() (runtime.Object, error) { return r.Create( ctx, name, obj, rest.ValidateObjectFunc(admit, admissionAttributes, scope), options, ) } ... // 执行创建,先进行准入修改,再执行上面包装的匿名函数 result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) { ... if m, ok := admit.(admission.MutationInterface); ok && m.Handles(admission.Create) { // 准入修改 mutatingAdmission.Admit(ctx, admissionAttributes, scope) ... } ... result, err := requestFunc() ... return result, err }) ... // 底层存储的调用结果序列化响应 transformResponseObject(ctx, scope, req, w, code, outputMediaType, result) } } // Create inserts a new item according to the unique key from the object. func (e *Store) Create(...) (runtime.Object, error) { ... // 准入校验 if createValidation != nil { createValidation(ctx, obj.DeepCopyObject()) ... } // 获取name name, err := e.ObjectNameFunc(obj) // 构造name/namespace key, err := e.KeyFunc(ctx, name) ... // 响应对象 out := e.NewFunc() // 调用etcd client e.Storage.Create(ctx, key, obj, out, ttl, dryrun.IsDryRun(options.DryRun)) ... return out, nil } func (s *DryRunnableStorage) Create(...) error { // dryRun不会写etcd if dryRun { // 数据存在 s.Storage.Get(ctx, key, storage.GetOptions{}, out) ... // 直接拷贝到out,模拟创建成功 return s.copyInto(obj, out) } // 写etcd return s.Storage.Create(ctx, key, obj, out, ttl) } // Create implements storage.Interface.Create. func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error { // 准备key(prefix/name/namespace) preparedKey, err := s.prepareKey(key) ... // 对象序列化 data, err := runtime.Encode(s.codec, obj) ... // 加密 newData, err := s.transformer.TransformToStorage(ctx, data, authenticatedDataString(preparedKey)) ... // 开启事务写 txnResp, err := s.client.KV.Txn(ctx).If(notFound(preparedKey)).Then( clientv3.OpPut(preparedKey, string(newData), opts...), ).Commit() ... // 调用结果转码返回 if out != nil { putResp := txnResp.Responses[0].GetResponsePut() decode(s.codec, s.versioner, data, out, putResp.Header.Revision) ... } return nil }1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133补充
由存储实现可以看出，
kube-apiserver除认证、鉴权、准入外，其实最重要的就是存储，三者均作用在route handler