kubeadm 源码分析

cmds.AddCommand(NewCmdInit(out))cmds.AddCommand(NewCmdJoin(out))cmds.AddCommand(NewCmdVersion(out))

cmd.PersistentFlags().StringVar(    &cfg.Secrets.GivenToken, "token", "",    "Shared secret used to secure cluster bootstrap; if none is provided, one will be generated for you",  )
cmd.PersistentFlags().StringSliceVar(    &cfg.API.AdvertiseAddresses, "api-advertise-addresses", []string{},    "The IP addresses to advertise, in case autodetection fails",  )
cmd.PersistentFlags().StringSliceVar(    &cfg.API.ExternalDNSNames, "api-external-dns-names", []string{},    "The DNS names to advertise, in case you have configured them yourself",  )
cmd.PersistentFlags().StringVar(    &cfg.Networking.ServiceSubnet, "service-cidr", kubeadmapi.DefaultServicesSubnet,    "Use alterantive range of IP address for service VIPs",  )
cmd.PersistentFlags().StringVar(    &cfg.Networking.PodSubnet, "pod-network-cidr", "",    "Specify range of IP addresses for the pod network; if set, the control plane will automatically allocate CIDRs for every node",  )
cmd.PersistentFlags().StringVar(    &cfg.Networking.DNSDomain, "service-dns-domain", kubeadmapi.DefaultServiceDNSDomain,    `Use alternative domain for services, e.g. "myorg.internal"`,  )
cmd.PersistentFlags().StringVar(    &cfg.CloudProvider, "cloud-provider", "",    `Enable cloud provider features (external load-balancers, storage, etc), e.g. "gce"`,  )
cmd.PersistentFlags().StringVar(    &cfg.KubernetesVersion, "use-kubernetes-version", kubeadmapi.DefaultKubernetesVersion,    `Choose a specific Kubernetes version for the control plane`,  )
// …    这里忽略了未来将要Deprecated的flag

// RunInit executes master node provisioning, including certificates, needed static pod manifests, etc.func RunInit(out io.Writer, cmd *cobra.Command, args []string, cfg *kubeadmapi.MasterConfiguration) error {  // 如果未指定“api-advertise-addresses”，就选择默认路由对应interface的ip  // Auto-detect the IP  if len(cfg.API.AdvertiseAddresses) == 0 {    // TODO(phase1+) perhaps we could actually grab eth0 and eth1    ip, err := netutil.ChooseHostInterface()    if err != nil {      return err    }    cfg.API.AdvertiseAddresses = []string{ip.String()}  }
  // 如果指定了“cloud-provider”，则初始化cloud provider，并向k8s注册  // TODO(phase1+) create a custom flag  if cfg.CloudProvider != "" {    if cloudprovider.IsCloudProvider(cfg.CloudProvider) {      fmt.Printf("<cmd/init> cloud provider %q initialized for the control plane. Remember to set the same cloud provider flag on the kubelet.\n", cfg.CloudProvider)    } else {      return fmt.Errorf("<cmd/init> cloud provider %q is not supported, you can use any of %v, or leave it unset.\n", cfg.CloudProvider, cloudprovider.CloudProviders())    }  }
  // 如果未指定“token”，则自动创建一个token  // 然后将该token保存在“/etc/kubernetes/pki”目录下：tokens.csv  if err := kubemaster.CreateTokenAuthFile(&cfg.Secrets); err != nil {    return err  }
}
  // 根据componentconfig的配置信息创建master节点上的static pod对象，  // 并将static pod以json文件格式保存到“/etc/kubernetes/manifests”目录下：  //     kube-apiserver.json  kube-controller-manager.json  kube-scheduler.json  // 如果用户未指定“external etcd”，则还会创建etcd static pod：etcd.json  if err := kubemaster.WriteStaticPodManifests(cfg); err != nil {    return err  }
  // 基于token创建服务相关的key和setificate，保存在“/etc/kubernetes/pki”目录下：  //     ca-key.pem  ca-pub.pem ca.pem  //     apiserver-key.pem  apiserver-pub.pem apiserver.pem  //     sa-key.pem  sa-pub.pem  caKey, caCert, err := kubemaster.CreatePKIAssets(cfg)  if err != nil {    return err  }
  // 创建client使用的certificate，并以conf文件保存在“/etc/kubernetes/pki”目录下：  //     admin.conf kubelet.conf  // admin.conf和kubelet.conf分别是kubectl和kubelet使用的kubeconfig  kubeconfigs, err := kubemaster.CreateCertsAndConfigForClients(cfg.API.AdvertiseAddresses, []string{"kubelet", "admin"}, caKey, caCert)  if err != nil {    return err  }  for name, kubeconfig := range kubeconfigs {    if err := kubeadmutil.WriteKubeconfigIfNotExists(name, kubeconfig); err != nil {      return err    }  }
  // 利用admin.conf创建client，并等待apiserver起来  // 使用这个client就相当于执行了如下命令：  //     kubectl --kubeconfig=”/etc/kubernetes/admin.conf” xxxx  // 后面的addons都是通过该client创建的  client, err := kubemaster.CreateClientAndWaitForAPI(kubeconfigs["admin"])  if err != nil {    return err  }
  // 设置默认不允许调度pod到master上的taint  // 这种情况下master上就只运行“/etc/kubernetes/manifests”目录下的static pod  // 和master上的一些addons（设置了toleration）  schedulePodsOnMaster := false  if err := kubemaster.UpdateMasterRoleLabelsAndTaints(client, schedulePodsOnMaster); err != nil {    return err  }
  // 创建kube-discovery deployment和secret，并等待kube-discovery server起来  // kube-discovery server侦听9898端口  // kube-discovery是必须的addon组件  // 当需要往k8s集群添加node时，“kubeadm join”首先通过URL向kube-discovery server请求cluster的相关信息  //      http://MasterIP:9898/cluster-info/v1/?token-id=XXX.YYYY  // kube-discovery服务收到请求之后，将cluster ca证书，endpoint列表和token以k8s secrets的方式回复给发送请求的节点  if err := kubemaster.CreateDiscoveryDeploymentAndSecret(cfg, client, caCert); err != nil {    return err  }
  // 创建其他必须的addon，主要是：kube-proxy, kube-dns  // kube-proxy 为daemonSet，而kube-dns为deployment  if err := kubemaster.CreateEssentialAddons(cfg, client); err != nil {    return err  }}

kubeadm join --token <token> <master-ip>

cmd.PersistentFlags().StringVar(    &cfg.Secrets.GivenToken, "token", "",    "(required) Shared secret used to secure bootstrap. Must match the output of 'kubeadm init'",  )

// RunJoin executes worked node provisioning and tries to join an existing cluster.func RunJoin(out io.Writer, cmd *cobra.Command, args []string, s *kubeadmapi.NodeConfiguration) error {  // 用户必须要指定token  // 如果用户指定了token，则检查token的格式是否合法  // token格式要求（比如：f0c861.753c505740ecde4c）:  //     1. 点分格式（2-part dot-separated format）  //     2. 前面部分为3个字节的tokenID，比如：f0c861  //     3. 后面部分为8个字节的token，比如：753c505740ecde4c  ok, err := kubeadmutil.UseGivenTokenIfValid(&s.Secrets)  if !ok {    if err != nil {      return fmt.Errorf("<cmd/join> %v (see --help)\n", err)    }    return fmt.Errorf("Must specify --token (see --help)\n")  }
  // 1. 访问kube-discovery服务，获取cluster相关信息（cluster info对象，主要包括cluster ca证书，endpoint列表和token信息）  //     URL: http://MasterIP:9898/cluster-info/v1/?token-id=XXX.YYYY  // 2. 使用用户指定的token检验cluster info对象的签名，如果检验失败则自然不允许加入该k8s集群  clusterInfo, err := kubenode.RetrieveTrustedClusterInfo(s)  if err != nil {    return err  }
  // 利用endpoint列表的其中一个apiserver的URL创建能与k8s master建立连接的ConnectionDetails对象  connectionDetails, err := kubenode.EstablishMasterConnection(s, clusterInfo)  if err != nil {    return err  }
  // 通过ConnectionDetails对象与apiserver建立连接，并向apiserver请求为该节点创建证书，然后根据该证书创建kubeconfig对象  kubeconfig, err := kubenode.PerformTLSBootstrap(connectionDetails)  if err != nil {    return err  }
  // 将kubeconfig对象写入“/etc/kubernetes/kubelet.config”文件  err = kubeadmutil.WriteKubeconfigIfNotExists("kubelet", kubeconfig)  if err != nil {    return err  }}

/usr/bin/kubelet --kubeconfig=/etc/kubernetes/kubelet.conf --require-kubeconfig=true --pod-manifest-path=/etc/kubernetes/manifests --allow-privileged=true --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin --cluster-dns=100.64.0.10 --cluster-domain=cluster.local