bootstrap

1.创建

1.1.initialize

• bootstrap-controller初始化位于apiserver创建期间，构造的bootstrap-controller会注册到server postStartHook延迟启动。

• // CreateServerChain creates the apiservers connected via delegation.
  func CreateServerChain(completedOptions completedServerRunOptions) (*aggregatorapiserver.APIAggregator, error) {
  	...
  
      // apiserver初始化
  	kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer)
  	...
  
  	return aggregatorServer, nil
  }
  
  // CreateKubeAPIServer creates and wires a workable kube-apiserver
  func CreateKubeAPIServer(...) (*controlplane.Instance, error) {
      // 补充service网段/首地址/对外暴露地址后创建apiserver
  	return kubeAPIServerConfig.Complete().New(delegateAPIServer)
  }
  
  // New returns a new instance of Master from the given config.
  func (c completedConfig) New(delegationTarget genericapiserver.DelegationTarget) (*Instance, error) {
  	...
  
  	// api资源注册
  	if err := m.InstallLegacyAPI(&c, c.GenericConfig.RESTOptionsGetter)
  	...
      
  	return m, nil
  }
  
  // InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled.
  func (m *Instance) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter) error {
  	...
  
      // 初始化监听客户端(基于本地回环)
  	controllerName := "bootstrap-controller"
  	client := kubernetes.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
  	// 注册ns controller
  	m.GenericAPIServer.AddPostStartHookOrDie("start-system-namespaces-controller", func(hookContext genericapiserver.PostStartHookContext) error {
  		go systemnamespaces.NewController(client, c.ExtraConfig.VersionedInformers.Core().V1().Namespaces()).Run(hookContext.StopCh)
  		return nil
  	})
  
      // 初始化bootstrap controller
  	bootstrapController, err := c.NewBootstrapController(legacyRESTStorage, client)
  	...
      
      // 注册bootstrap controller的启停回调
  	m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook)
  	m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook)
  	...
  	
  	return nil
  }
  

• 

1.2.poststart

• bootstrap controller注册到postStartHook的回调会在prepared.Run()执行，以管理ns创建及service入口维护。

• // run server
  func (s preparedAPIAggregator) Run(stopCh <-chan struct{}) error {
  	return s.runnable.Run(stopCh)
  }
  
  // Run spawns the secure http server.
  func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
  	...
  
      // 非阻塞启动
  	stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(stopHttpServerCh, shutdownTimeout)
  	...
      
      // run shutdown hooks directly.
  	func() {
  		defer func() {
              // 通知不再接收新请求
  			preShutdownHooksHasStoppedCh.Signal()
  		}()
          // 执行preShutdownHook回调
  		s.RunPreShutdownHooks()
  	}()
  
  	// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
  	<-listenerStoppedCh
  	<-stoppedCh
  
  	return nil
  }
  
  // RunPreShutdownHooks runs the PreShutdownHooks for the server
  func (s *GenericAPIServer) RunPreShutdownHooks() error {
  	...
  
  	s.preShutdownHookLock.Lock()
  	defer s.preShutdownHookLock.Unlock()
  	s.preShutdownHooksCalled = true
  
      // 遍历preShuwdownHook回调
  	for hookName, hookEntry := range s.preShutdownHooks {
          // 执行回调
  		runPreShutdownHook(hookName, hookEntry)
          ...
  	}
  	return utilerrors.NewAggregate(errorList)
  }
  
  // NonBlockingRun spawns the secure http server.
  func (s preparedGenericAPIServer) NonBlockingRun(...) (<-chan struct{}, <-chan struct{}, error) {
  	...
      
      // 启动https server
  	if s.SecureServingInfo != nil && s.Handler != nil {
          ...
  		stoppedCh, listenerStoppedCh, err = s.SecureServingInfo.Serve(s.Handler, shutdownTimeout,internalStopCh)
  		...
  	}
  
  	// 退出通知
  	go func() {
  		<-stopCh
  		close(internalStopCh)
  	}()
  
      // 执行postStartHook回调
  	s.RunPostStartHooks(stopCh)
  
      // 通知systemd服务已ready
  	systemd.SdNotify(true, "READY=1\n")
  
  	return stoppedCh, listenerStoppedCh, nil
  }
  
  // RunPostStartHooks runs the PostStartHooks for the server
  func (s *GenericAPIServer) RunPostStartHooks(stopCh <-chan struct{}) {
  	s.postStartHookLock.Lock()
  	defer s.postStartHookLock.Unlock()
  	s.postStartHooksCalled = true
  	...
  
      // 遍历postStartHook回调
  	for hookName, hookEntry := range s.postStartHooks {
          // 执行回调
  		go runPostStartHook(hookName, hookEntry, context)
  	}
  }
  

• 

• 注意

  boostrap controller基于延迟启动策略，注册回调后由apiserver启动后调用

1.3.nscontrol

• namespace controller用于监听namespace资源，检查集群关键命名空间kube-system/kube-node-lease/kube-public/default存在。

• // NewController creates a new Controller to ensure system namespaces exist.
  func NewController(clientset kubernetes.Interface,namespaceInformer coreinformers.NamespaceInformer)*Controller{
  	// 关键的namespace
      systemNamespaces := []string{"kube-system","kube-node-lease","kube-public","default"}
      // 检查周期
  	interval := 1 * time.Minute
  
  	return &Controller{
  		client:           clientset,
          // namespace同步缓存
  		namespaceLister:  namespaceInformer.Lister(),
  		namespaceSynced:  namespaceInformer.Informer().HasSynced,
  		systemNamespaces: systemNamespaces,
  		interval:         interval,
  	}
  }
  
  // Run starts one worker.
  func (c *Controller) Run(stopCh <-chan struct{}) {
  	...
  	
      // 间隔100ms执行namespace同步检查
  	if !cache.WaitForCacheSync(stopCh, c.namespaceSynced) {
  		return
  	}
  
      // 间隔1min执行一次检查
  	go wait.Until(c.sync, c.interval, stopCh)
  
  	<-stopCh
  }
  
  // namespace sync check.
  func (c *Controller) sync() {
  	// Loop the system namespace list, and create them if they do not exist
  	for _, ns := range c.systemNamespaces {
          // 创建关键namespace
  		c.createNamespaceIfNeeded(ns)
          ...
  	}
  }
  
  // create crucial namespace.
  func (c *Controller) createNamespaceIfNeeded(ns string) error {
      // ns存在
  	if _, err := c.namespaceLister.Get(ns); err == nil {
  		// the namespace already exists
  		return nil
  	}
      ...
      
      // 创建(未存在)
  	_, err := c.client.CoreV1().Namespaces().Create(context.TODO(), newNs, metav1.CreateOptions{})
  	...
      
  	return err
  }
  

• 

• 注意

  namespace controller其实就是维持集群中必要的namespace存在，不存在则会创建

1.4.bootstrap

• bootstrap controller由于维护apiserver service，需设置cidr、addr及svcport，servicecidr的首地址会作为clusterIP。

• // InstallLegacyAPI will install the legacy APIs for the restStorageProviders if they are enabled.
  func (m *Instance) InstallLegacyAPI(c *completedConfig, restOptionsGetter generic.RESTOptionsGetter) error {
  	...
  
      // 基于本地回环的client
  	controllerName := "bootstrap-controller"
  	client := kubernetes.NewForConfigOrDie(c.GenericConfig.LoopbackClientConfig)
  	...
  
      // 初始化bootstrap controller
  	bootstrapController, err := c.NewBootstrapController(legacyRESTStorage, client)
  	...
      
      // 注册回调
  	m.GenericAPIServer.AddPostStartHookOrDie(controllerName, bootstrapController.PostStartHook)
  	m.GenericAPIServer.AddPreShutdownHookOrDie(controllerName, bootstrapController.PreShutdownHook)
  	...
      
  	return nil
  }
  
  // NewBootstrapController returns a controller for watching the core capabilities of the master
  func (c *completedConfig) NewBootstrapController(...) (*Controller, error) {
      // servicePort
  	_, publicServicePort, err := c.GenericConfig.SecureServing.HostPort()
  	...
  
      // 
  	return &Controller{
  		client:    client,
  		informers: c.ExtraConfig.VersionedInformers,
  
  		EndpointReconciler: c.ExtraConfig.EndpointReconcilerConfig.Reconciler,  // lease endpoint reconciler
  		EndpointInterval:   c.ExtraConfig.EndpointReconcilerConfig.Interval,	// 默认10s
  
  		ServiceClusterIPRegistry:          legacyRESTStorage.ServiceClusterIPAllocator,
  		ServiceClusterIPRange:             c.ExtraConfig.ServiceIPRange,				// 默认10.0.0.0/24
  		SecondaryServiceClusterIPRegistry: legacyRESTStorage.SecondaryServiceClusterIPAllocator,
  		SecondaryServiceClusterIPRange:    c.ExtraConfig.SecondaryServiceIPRange,
  
  		ServiceClusterIPInterval: c.ExtraConfig.RepairServicesInterval,		// 默认3min
  
  		ServiceNodePortRegistry: legacyRESTStorage.ServiceNodePortAllocator,
  		ServiceNodePortRange:    c.ExtraConfig.ServiceNodePortRange,
  		ServiceNodePortInterval: c.ExtraConfig.RepairServicesInterval,		// 默认3min
  
  		PublicIP: c.GenericConfig.PublicAddress,							// apiserver绑定IP,作为endpoint后端
  
  		ServiceIP:                 c.ExtraConfig.APIServerServiceIP,		// servicecidr首地址
  		ServicePort:               c.ExtraConfig.APIServerServicePort,		// 443
  		PublicServicePort:         publicServicePort,						// 6443
  		KubernetesServiceNodePort: c.ExtraConfig.KubernetesServiceNodePort,	// 缺省,意为clusterIP service
  	}, nil
  }
  

• 注意

  apiserver svc.spec.selector为空，表明这个特殊的service对应的endpoints不由ep controller控制

2.启动

2.1.start

• c.postStartHook()用于启动clusterIP和nodePort修复，激活default service及关键的namespace管理，维护apiserver入口。

• // PostStartHook initiates the core controller loops that must exist for bootstrapping.
  func (c *Controller) PostStartHook(hookContext genericapiserver.PostStartHookContext) error {
  	c.Start()
  	return nil
  }
  
  // Start begins the core controller loops that must exist for bootstrapping a cluster.
  func (c *Controller) Start() {
      // runner任务已存在
  	if c.runner != nil {
  		return
  	}
  
  	// 构造endpoint ports
  	endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https")
      // 首次启动移除endpoint后端,此时apiserver可能未ready
  	c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts)
      ...
  
  	// service nodeports修复回调
  	runRepairNodePorts := func(stopCh chan struct{}) {
  		repairNodePorts.RunUntil(wg.Done, stopCh)
  	}
  	...
      
      // clusterIP修复回调
      runRepairClusterIPs = func(stopCh chan struct{}) {
          repairClusterIPs.RunUntil(wg.Done, stopCh)
      }
      
      // 注册service同步、clusterIP修复、nodePorts修复任务
  	c.runner = async.NewRunner(c.RunKubernetesService, runRepairClusterIPs, runRepairNodePorts)
      
      // 启动同步任务
  	c.runner.Start()
  	...
  }
  
  // Start begins running.
  func (r *Runner) Start() {
  	r.lock.Lock()
  	defer r.lock.Unlock()
      // 未终止
  	if r.stop == nil {
  		c := make(chan struct{})
  		r.stop = &c
          // 遍历注册任务
  		for i := range r.loopFuncs {
              // 异步执行任务
  			go r.loopFuncs[i](*r.stop)
  		}
  	}
  }
  

• 

• 注意

  runner其实作为总调入口，会管理repairNodePorts、repairClusterIP和runKubernetesService任务的启停

2.2.service

• c.runKubernetesService()主要检查kubernetes service状态，根据apiserver的ready状态同步及更新kubernetes service信息。

• // RunKubernetesService periodically updates the kubernetes service
  func (c *Controller) RunKubernetesService(ch chan struct{}) {
  	// 间隔100ms检查apiserver就绪
  	wait.PollImmediateUntil(100*time.Millisecond, func() (bool, error) {
  		...
  		c.client.CoreV1().RESTClient().Get().AbsPath("/readyz").Do(context.TODO()).StatusCode(&code)
  		return code == http.StatusOK, nil
  	}, ch)
  
      // 间隔10s更新一次service
  	wait.NonSlidingUntil(func() {
  		c.UpdateKubernetesService(false)
          ...
  	}, c.EndpointInterval, ch)
  }
  
  // UpdateKubernetesService attempts to update the default Kube service.
  func (c *Controller) UpdateKubernetesService(reconcile bool) error {
  	// 创建default命名空间
  	createNamespaceIfNeeded(c.client.CoreV1(), "default")
      ...
  
      // 创建或更新kubernetes service
  	c.CreateOrUpdateMasterServiceIfNeeded("kubernetes", c.ServiceIP, servicePorts, serviceType, reconcile)
      ...
      
  	// 更新endpoint
  	c.EndpointReconciler.ReconcileEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts, reconcile)
      ...
      
  	return nil
  }
  
  // ReconcileEndpoints lists keys in a special etcd directory.
  func (r *leaseEndpointReconciler) ReconcileEndpoints(svc string, ip net.IP, ports []corev1.EndpointPort, reconcile bool) error {
  	r.reconcilingLock.Lock()
  	defer r.reconcilingLock.Unlock()
  
      // epr终止
  	if r.stopReconcilingCalled {
  		return nil
  	}
  
  	// 先更新apiserver lease记录的地址
  	r.masterLeases.UpdateLease(ip.String())
      ...
  
      // 执行endpoints更新
  	return r.doReconcile(serviceName, endpointPorts, reconcilePorts)
  }
  

• 

• 注意

  runKubernetesService()做三件事，default namespace创建、kubernetes svc创建及endpoints后端维护

2.3.clusterip

• repairClusterIPs.RunUntil()确保集群所有的clusterIP唯一分配，不会超出serviceCIDR网段范围，同时修复未正确创建的clusterIP。

• // RunUntil starts the controller until the provided ch is closed.
  func (c *Repair) RunUntil(onFirstSuccess func(), stopCh chan struct{}) {
  	...
      
      // 间隔3min执行一次
  	wait.Until(func() {
  		c.runOnce()
          ...
  		once.Do(onFirstSuccess)
  	}, c.interval, stopCh)
  }
  
  // doRunOnce verifies the state of the cluster IP allocations.
  func (c *Repair) doRunOnce() error {
  	...
  
      // 重试10s获取etcd已经使用的clusterIP快照
  	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
          // 遍历注册的分配器(primary/secondary)
  		for family, allocator := range c.allocatorByFamily {
  			// get snapshot if it is not there
  			if _, ok := snapshotByFamily[family]; !ok {
                  // 获取etcd对应的已使用clusterIP
  				snapshot, err := allocator.Get()
  				...
  				
                  // 记录
  				snapshotByFamily[family] = snapshot
  			}
  		}
  		return true, nil
  	})
  	...
  
  	// ensure that ranges are assigned
  	for family, snapshot := range snapshotByFamily {
          // cidr为空取内存中维护的serviceRange
  		if snapshot.Range == "" {
  			snapshot.Range = c.networkByFamily[family].String()
  		}
          // 构建快照分配器
  		stored, err := ipallocator.NewFromSnapshot(snapshot)
  		...
  		storedByFamily[family] = stored
  	}
  	...
  
      // 构建空白重建账本
  	for family, network := range c.networkByFamily {
          // 初始化内存分配器
  		rebuilt, err := ipallocator.NewInMemory(network)
  		...
  		rebuiltByFamily[family] = rebuilt
  	}
  	...
      
      // 获取svc列表
  	list, err := c.serviceClient.Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
  
  	// 重建当前状态
  	for _, svc := range list.Items {
          // 非clusterIP service
  		if !helper.IsServiceIPSet(&svc) {
  			// didn't need a cluster IP
  			continue
  		}
  
          // 遍历clusterIP
  		for _, ip := range svc.Spec.ClusterIPs {
              
  			ip := netutils.ParseIPSloppy(ip)
  			...
  			family := getFamilyByIP(ip)
  			...
  
  			// mark it as in-use
  			actualAlloc := rebuiltByFamily[family]
              // 尝试重新分配clusterIP
  			switch err := actualAlloc.Allocate(ip); err {
              // clusterIP合法
  			case nil:
  				actualStored := storedByFamily[family]
                  // 快照记录了当前clusterIP,说明正在用
  				if actualStored.Has(ip) {
  					// 由快照移除clusterIP
  					actualStored.Release(ip)
  				}
                  ...
                  
                  // 正在使用的由leaks移除
  				delete(c.leaksByFamily[family], ip.String()) // it is used, so it can't be leaked
  			...
  			}
  		}
  	}
  
  	// 剩余未使用的检查是否泄漏
  	for family, leaks := range c.leaksByFamily {
  		c.checkLeaked(leaks, storedByFamily[family], rebuiltByFamily[family])
  	}
  
  	// 重建快照
  	for family, rebuilt := range rebuiltByFamily {
  		c.saveSnapShot(rebuilt, c.allocatorByFamily[family], snapshotByFamily[family])
  		...
  	}
  
  	return nil
  }
  

• 

• 注意

  repairClusterIPs本质上根据etcd已分配的clusterIP快照和svc列表重建最新状态

2.4.nodeport

• repairNodePorts.RunUntil()确保集群所有的nodeport唯一分配，不会出现端口泄漏，泄露的端口会尝试修复，无法修复则发出告警。

• // RunUntil starts the controller until the provided ch is closed.
  func (c *Repair) RunUntil(onFirstSuccess func(), stopCh chan struct{}) {
  	...
      
      // 间隔3min触发一次
  	wait.Until(func() {
          // 执行修复
  		c.runOnce()
          ...
  		once.Do(onFirstSuccess)
  	}, c.interval, stopCh)
  }
  
  // doRunOnce verifies the state of the port allocations and returns an error if an unrecoverable problem occurs.
  func (c *Repair) doRunOnce() error {
  	...
      
      // 尝试执行10s
  	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
  		...
          // 获取快照
  		snapshot, err = c.alloc.Get()
  		return err == nil, err
  	})
  	...
      
  	// 初始化快照port范围
  	if snapshot.Range == "" {
  		snapshot.Range = c.portRange.String()
  	}
      
  	// 基于快照构建分配器
  	stored, err := portallocator.NewFromSnapshot(snapshot)
  	...
      
  
  	// 获取service列表
  	list, err := c.serviceClient.Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
  	...
  
      // 构建空的内存分配器
  	rebuilt, err := portallocator.NewInMemory(c.portRange)
  	...
      
  	// 检查service port占用
  	for i := range list.Items {
  		svc := &list.Items[i]
  		ports := collectServiceNodePorts(svc)
  		if len(ports) == 0 {
  			continue
  		}
  
          // 遍历svc使用port
  		for _, port := range ports {
              // 尝试分配
  			switch err := rebuilt.Allocate(port); err {
  			case nil:
                  // 快照记录port
  				if stored.Has(port) {
  					// 由快照移除,标记使用
  					stored.Release(port)
  				} 
                  ...
                  
                  // 清理泄漏的svc port
  				delete(c.leaks, port) // it is used, so it can't be leaked
  			...
  			}
  		}
  	}
  
  	// 对比是否有泄漏nodeport
  	for family, leaks := range c.leaksByFamily {
  		c.checkLeaked(leaks, stored, rebuilt)
  	}
  
  	// 最新状态写入快照
  	rebuilt.Snapshot(snapshot)
      ...
  
      // 创建或更新快照
  	c.alloc.CreateOrUpdate(snapshot)
      ...
      
  	return nil
  }
  

• 

2.5.shutdown

• c.preShutdownHook()用于退出前的清理，以撤销集群中的svc endpoint注册信息，确保退出的apiserver移除自已的地址，避免假节点现象。

• // PreShutdownHook triggers the actions needed to shut down the API Server cleanly.
  func (c *Controller) PreShutdownHook() error {
  	c.Stop()
  	return nil
  }
  
  // Stop cleans up this API Servers endpoint reconciliation leases so another master can take over more quickly.
  func (c *Controller) Stop() {
      // 终止service更新后台任务
  	if c.runner != nil {
  		c.runner.Stop()
  	}
      
      // 构造endpoint port
  	endpointPorts := createEndpointPortSpec(c.PublicServicePort, "https")
  	...
      
      // 异步清理协程
  	go func() {
  		defer close(finishedReconciling)
  		// 通知lease endpoint controller停止同步
  		c.EndpointReconciler.StopReconciling()
          // 由endpoint移除当前apiserver后端
  		c.EndpointReconciler.RemoveEndpoints(kubernetesServiceName, c.PublicIP, endpointPorts)
          ...
          
          // 资源释放
  		c.EndpointReconciler.Destroy()
  	}()
  
      // 等待完成
  	select {
  	...
  	}
  }
  
  // 摘除apiserver后端
  func (r *leaseEndpointReconciler) RemoveEndpoints(svc string, ip net.IP, ports []corev1.EndpointPort) error {
      // 移除apiserver地址租约
      r.masterLeases.RemoveLease(ip.String())
      ...
  
      // 更新endpoint后端池
  	return r.doReconcile(serviceName, endpointPorts, true)
  }
  
  // doReconcile remove apiserver from endpoint.
  func (r *leaseEndpointReconciler) doReconcile(svc string, ports []corev1.EndpointPort, reconcile bool) error {
      // 获取default endpoint
      e, err := r.epAdapter.Get(corev1.NamespaceDefault, serviceName, metav1.GetOptions{})
  	...
  
  	// 获取所有apiserver地址
  	masterIPs, err := r.masterLeases.ListLeases()
  	...
  
  	// endpoint设置skipMirror label为true,无需eps controller生成eps对象进行优化
  	skipMirrorChanged := setSkipMirrorTrue(e)
  
      // 检查endpoint的格式及内容匹配
  	formatCorrect, ipCorrect, portsCorrect := checkEndpointSubsetFormatWithLease(e, masterIPs, ports, reconcile)
  	
      // endpoint格式及内容匹配,未跳过endpointslice镜像
      if !skipMirrorChanged && formatCorrect && ipCorrect && portsCorrect {
          // endpoint信息转为endpointslice,创建或更新
  		return r.epAdapter.EnsureEndpointSliceFromEndpoints(corev1.NamespaceDefault, e)
  	}
  	...
      // 更新endpoint地址及端口
      ...
  
  	// 创建
  	if shouldCreate {
  		if _, err = r.epAdapter.Create(corev1.NamespaceDefault, e); errors.IsAlreadyExists(err) {
  			err = nil
  		}
      // 更新
  	} else {
  		_, err = r.epAdapter.Update(corev1.NamespaceDefault, e)
  	}
  	return err
  }
  
  // 监控协程释放
  func (r *leaseEndpointReconciler) Destroy() {
  	r.masterLeases.Destroy()
  }
  
  func (s *storageLeases) Destroy() {
  	s.destroyFn()
  }
  
  func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
  	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
  	...
  
  	stopDBSizeMonitor, err := startDBSizeMonitorPerEndpoint(client, c.DBMetricPollInterval)
  	...
  
      // destroyFn定义
  	destroyFunc := func() {
  		once.Do(func() {
              // 停止etcd压缩协程
  			stopCompactor()
              // 停止etcd数据大小监听协程
  			stopDBSizeMonitor()
  			client.Close()
  		})
  	}
  	...
      
  	return etcd3.New(...), destroyFunc, nil
  }
  

• 

• 注意

  bootstrap controller资源回收主要更新endpoints后端池及结束etcd压缩及统计任务