使用operator-sdk开发operator

使用operator-sdk创建一个2048小游戏的operator，并构建镜像，使用域名访问。

创建项目

mkdir game && cd game
go mod init game
operator-sdk init --domain game.com --project-name game --plugins go/v4 --owner wgh
# --domain string            domain for groups (default "my.domain")
# --owner string             owner to add to the copyright
# --project-name string      name of this project
# --plugins strings   plugin keys to be used for this subcommand execution
# --repo string              name to use for go module (e.g., github.com/user/repo), defaults to the go package of the current working directory。不在$GOPATH/src中也可以使用这个参数。
tree .
.
├── cmd
│   └── main.go
├── config
│   ├── default
│   │   ├── kustomization.yaml
│   │   ├── manager_auth_proxy_patch.yaml
│   │   └── manager_config_patch.yaml
│   ├── manager
│   │   ├── kustomization.yaml
│   │   └── manager.yaml
│   ├── manifests
│   │   └── kustomization.yaml
│   ├── prometheus
│   │   ├── kustomization.yaml
│   │   └── monitor.yaml
│   ├── rbac
│   │   ├── auth_proxy_client_clusterrole.yaml
│   │   ├── auth_proxy_role_binding.yaml
│   │   ├── auth_proxy_role.yaml
│   │   ├── auth_proxy_service.yaml
│   │   ├── kustomization.yaml
│   │   ├── leader_election_role_binding.yaml
│   │   ├── leader_election_role.yaml
│   │   ├── role_binding.yaml
│   │   ├── role.yaml
│   │   └── service_account.yaml
│   └── scorecard
│       ├── bases
│       │   └── config.yaml
│       ├── kustomization.yaml
│       └── patches
│           ├── basic.config.yaml
│           └── olm.config.yaml
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│   └── boilerplate.go.txt
├── Makefile
├── PROJECT
├── README.md
└── test
    ├── e2e
    │   ├── e2e_suite_test.go
    │   └── e2e_test.go
    └── utils
        └── utils.go

14 directories, 33 files

创建api

operator-sdk create api --group game --version v1 --kind Game --resource --controller
# --group string         resource Group
# --kind string          resource Kind
# --resource             if set, generate the resource without prompting the user (default true)
# --controller           if set, generate the controller without prompting the user (default true)

tree .
.
├── api
│   └── v1
│       ├── game_types.go
│       ├── groupversion_info.go
│       └── zz_generated.deepcopy.go
├── bin
│   └── controller-gen
├── cmd
│   └── main.go
├── config
│   ├── crd
│   │   ├── kustomization.yaml
│   │   └── kustomizeconfig.yaml
│   ├── default
│   │   ├── kustomization.yaml
│   │   ├── manager_auth_proxy_patch.yaml
│   │   └── manager_config_patch.yaml
│   ├── manager
│   │   ├── kustomization.yaml
│   │   └── manager.yaml
│   ├── manifests
│   │   └── kustomization.yaml
│   ├── prometheus
│   │   ├── kustomization.yaml
│   │   └── monitor.yaml
│   ├── rbac
│   │   ├── auth_proxy_client_clusterrole.yaml
│   │   ├── auth_proxy_role_binding.yaml
│   │   ├── auth_proxy_role.yaml
│   │   ├── auth_proxy_service.yaml
│   │   ├── game_editor_role.yaml
│   │   ├── game_viewer_role.yaml
│   │   ├── kustomization.yaml
│   │   ├── leader_election_role_binding.yaml
│   │   ├── leader_election_role.yaml
│   │   ├── role_binding.yaml
│   │   ├── role.yaml
│   │   └── service_account.yaml
│   ├── samples
│   │   ├── game_v1_game.yaml
│   │   └── kustomization.yaml
│   └── scorecard
│       ├── bases
│       │   └── config.yaml
│       ├── kustomization.yaml
│       └── patches
│           ├── basic.config.yaml
│           └── olm.config.yaml
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│   └── boilerplate.go.txt
├── internal
│   └── controller
│       ├── game_controller.go
│       ├── game_controller_test.go
│       └── suite_test.go
├── Makefile
├── PROJECT
├── README.md
└── test
    ├── e2e
    │   ├── e2e_suite_test.go
    │   └── e2e_test.go
    └── utils
        └── utils.go

设计api

修改api/v1/game_types.go文件：

// Replicas 字段定义了游戏的副本数量,默认为 1,最小为 1,最大为 3。Host为游戏服务的地址。
type GameSpec struct {
    //+kubebuilder:default:=1
    //+kubebuilder:validation:Maximum:=3
    //+kubebuilder:validation:Minimum:=1
    Replicas int32  `json:"replicas,omitempty"`
    Image    string `json:"image,omitempty"`
    Host     string `json:"host,omitempty"`
}

const (
    Init     = "Init"
    Running  = "Running"
    NotReady = "NotReady"
    Failed   = "Failed"
)

// GameStatus defines the observed state of Game
type GameStatus struct {
    // Phase 字段定义了游戏的当前状态,可以是 Init、Running、NotReady 或 Failed。
    Phase string `json:"phase,omitempty"`
    // Replicas 字段定义了游戏的副本数量。
    Replicas int32 `json:"replicas,omitempty"`
    // ReadyReplicas 字段定义了游戏的就绪副本数量。
    ReadyReplicas int32 `json:"readyReplicas,omitempty"`
    // LabelSelector 字段定义了游戏的标签选择器。用于HPA。
    LabelSelector string `json:"labelSelector,omitempty"`
}

// 添加了scale子资源，并添加子资源specpath，statuspath，selectorpath对应字段的定义。最后添加打印列。
//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
//+kubebuilder:printcolumn:name="PHASE",type=string,JSONPath=`.status.phase`,description="Game Phase"
//+kubebuilder:printcolumn:name="Host",type=string,JSONPath=`.spec.host`,description="Host address"
//+kubebuilder:printcolumn:name="DESIRED",type=integer,JSONPath=`.spec.replicas`,description="Desired Replicas"
//+kubebuilder:printcolumn:name="CURRENT",type=integer,JSONPath=`.status.replicas`,description="Current Replicas"
//+kubebuilder:printcolumn:name="READY",type=integer,JSONPath=`.status.readyReplicas`,description="Ready Replicas"
//+kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp`,description="CreationTimestamp"

修改控制器逻辑

修改internal/controller/game_controller.go文件：

package controller

import (
    "context"
    appsv1 "k8s.io/api/apps/v1"
    corev1 "k8s.io/api/core/v1"
    networkingv1 "k8s.io/api/networking/v1"
    "k8s.io/apimachinery/pkg/api/errors"
    metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    "k8s.io/apimachinery/pkg/types"
    "k8s.io/apimachinery/pkg/util/intstr"
    utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    "k8s.io/utils/pointer"
    "reflect"
    "sigs.k8s.io/controller-runtime/pkg/controller"

    "k8s.io/apimachinery/pkg/runtime"
    ctrl "sigs.k8s.io/controller-runtime"
    "sigs.k8s.io/controller-runtime/pkg/client"
    "sigs.k8s.io/controller-runtime/pkg/log"

    gamev1 "game/api/v1"
)

// GameReconciler reconciles a Game object
type GameReconciler struct {
    client.Client
    Scheme *runtime.Scheme
}

// 添加rbac规则
//+kubebuilder:rbac:groups=game.game.com,resources=games,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=game.game.com,resources=games/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=game.game.com,resources=games/finalizers,verbs=update
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Game object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile
func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
    defer utilruntime.HandleCrash()

    logger := log.FromContext(ctx)
    logger.Info("Reconciling Game", "name", req.String())

    // 根据req获取Game资源
    game := &gamev1.Game{}
    if err := r.Get(ctx, req.NamespacedName, game); err != nil {
        return ctrl.Result{}, client.IgnoreNotFound(err)
    }
    // 判断是否在被删除
    if game.DeletionTimestamp != nil {
        logger.Info("Deleting Game", "name", req.String())
        return ctrl.Result{}, nil
    }
    // 调用syncGame
    if err := r.syncGame(ctx, game); err != nil {
        logger.Error(err, "Failed to sync Game", "name", req.String())
        return ctrl.Result{}, err
    }

    return ctrl.Result{}, nil
}

const (
    gameLabelName = "game-label"
    port          = 80
)

// syncGame 确保Game的Deployment，Service和Ingress都处于预期的状态，
func (r *GameReconciler) syncGame(ctx context.Context, game *gamev1.Game) error {
    logger := log.FromContext(ctx)
    // 创建深拷贝，防止直接修改原有对象
    game = game.DeepCopy()
    name := types.NamespacedName{
        Namespace: game.Namespace,
        Name:      game.Name,
    }
    // 创建OwnerReference数组，将Game资源设置为子资源的所有者。确保当Game资源被删除时，它的子资源也会被自动删除。
    ower := []metav1.OwnerReference{
        {
            APIVersion: game.APIVersion,
            Kind:       game.Kind,
            Name:       game.Name,
            // pointer.Bool(true)返回一个指向true的布尔型指针，赋值给Controller字段
            Controller: pointer.Bool(true),
            // 指定BlockOwnerDeletion为true，用于控制是否应该在删除父资源时同时删除子资源。
            BlockOwnerDeletion: pointer.Bool(true),
            UID:                game.UID,
        },
    }
    // 使用Game资源的名称创建一个标签映射，用于标识该资源所管理的子资源。
    labels := map[string]string{
        gameLabelName: name.Name,
    }
    // 设置元数据
    meta := metav1.ObjectMeta{
        Name:            name.Name,
        Namespace:       name.Namespace,
        Labels:          labels,
        OwnerReferences: ower,
    }
    // 创建Deployment
    deploy := &appsv1.Deployment{}
    if err := r.Get(ctx, name, deploy); err != nil {
        if !errors.IsNotFound(err) {
            return err
        }
        deploy = &appsv1.Deployment{
            ObjectMeta: meta,
            // 获取Deployment的期望Spec
            Spec: getDeploymentSpec(game, labels),
        }
        if err := r.Create(ctx, deploy); err != nil {
            return err
        }
        logger.Info("Create deployment success", "name", name.String())
    } else {
        // 比较期望Spec和当前的Spec是否完全一致，如果不同则更新Deployment为期望Spec。
        desire := getDeploymentSpec(game, labels)
        now := getSpecFromDeployment(deploy)
        if !reflect.DeepEqual(desire, now) {
            newDeploy := deploy.DeepCopy()
            newDeploy.Spec = desire
            if err := r.Update(ctx, newDeploy); err != nil {
                return err
            }
            logger.Info("Update deployment success", "name", name.String())
        }
    }
    // 创建Service
    svc := &corev1.Service{}
    if err := r.Get(ctx, name, svc); err != nil {
        if !errors.IsNotFound(err) {
            return err
        }
        svc = &corev1.Service{
            ObjectMeta: meta,
            Spec: corev1.ServiceSpec{
                Selector: labels,
                Ports: []corev1.ServicePort{
                    {
                        Name:     "http",
                        Port:     int32(port),
                        Protocol: corev1.ProtocolTCP,
                        TargetPort: intstr.IntOrString{
                            Type:   intstr.Int,
                            IntVal: port,
                        },
                    },
                },
            },
        }
        if err := r.Create(ctx, svc); err != nil {
            return err
        }
        logger.Info("Create service success", "name", name.String())
    }
    // 创建Ingress
    ing := &networkingv1.Ingress{}
    if err := r.Get(ctx, name, ing); err != nil {
        if !errors.IsNotFound(err) {
            return err
        }
        ing = &networkingv1.Ingress{
            ObjectMeta: meta,
            Spec:       getIngressSpec(game),
        }
        if err := r.Create(ctx, ing); err != nil {
            return err
        }
        logger.Info("Create ingress success", "name", name.String())
    }
    // 更新Game资源的Status
    newStatus := gamev1.GameStatus{
        Replicas:      deploy.Status.Replicas,
        ReadyReplicas: deploy.Status.ReadyReplicas,
    }
    // 如果就绪副本数等于当前副本数,则将状态设置为Running; 否则设置为NotReady。
    if game.Status.Replicas == newStatus.ReadyReplicas {
        newStatus.Phase = gamev1.Running
    } else {
        newStatus.Phase = gamev1.NotReady
    }
    // 如果Game资源的状态发生变化,则更新Game资源的状态子资源和Game资源。
    if !reflect.DeepEqual(game.Status, newStatus) {
        game.Status = newStatus
        logger.Info("Update game status", "name", name.String())
        if err := r.Client.Status().Update(ctx, game); err != nil {
            return err
        }
        game.Spec.Replicas = newStatus.Replicas
        logger.Info("Update game spec", "name", name.String())
        if err := r.Client.Update(ctx, game); err != nil {
            return err
        }
    }

    return nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *GameReconciler) SetupWithManager(mgr ctrl.Manager) error {
    return ctrl.NewControllerManagedBy(mgr).
        // MaxConcurrentReconciles是可以运行的最大并发Reconciles数。默认值为 1。
        WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
        For(&gamev1.Game{}).
        Owns(&appsv1.Deployment{}).
        Complete(r)
}

// getDeploymentSpec 根据Game和labels，生成Deployment的期望Spec
func getDeploymentSpec(game *gamev1.Game, labels map[string]string) appsv1.DeploymentSpec {
    return appsv1.DeploymentSpec{
        // 使用指定的副本数
        Replicas: pointer.Int32(game.Spec.Replicas),
        // 使用提供的标签映射创建一个LabelSelector
        Selector: metav1.SetAsLabelSelector(labels),
        // 创建Pod模板，设置标签，指定容器名字和镜像。
        Template: corev1.PodTemplateSpec{
            ObjectMeta: metav1.ObjectMeta{
                Labels: labels,
            },
            Spec: corev1.PodSpec{
                Containers: []corev1.Container{
                    {
                        Name:  "game",
                        Image: game.Spec.Image,
                    },
                },
            },
        },
    }
}

// getSpecFromDeployment 从Deployment中获取Spec
func getSpecFromDeployment(deploy *appsv1.Deployment) appsv1.DeploymentSpec {
    // 获取第一个容器的Spec，pod只有一个主容器
    container := deploy.Spec.Template.Spec.Containers[0]
    return appsv1.DeploymentSpec{
        Replicas: deploy.Spec.Replicas,
        Selector: deploy.Spec.Selector,
        Template: corev1.PodTemplateSpec{
            ObjectMeta: metav1.ObjectMeta{
                // 从Deployment的Pod模板规格中获取标签
                Labels: deploy.Spec.Template.Labels,
            },
            Spec: corev1.PodSpec{
                Containers: []corev1.Container{
                    {
                        Name:  container.Name,
                        Image: container.Image,
                    },
                },
            },
        },
    }
}

// getIngressSpec 根据Game和labels，生成Ingress的期望Spec
func getIngressSpec(game *gamev1.Game) networkingv1.IngressSpec {
    pathType := networkingv1.PathTypePrefix
    return networkingv1.IngressSpec{
        Rules: []networkingv1.IngressRule{
            {
                Host: game.Spec.Host,
                IngressRuleValue: networkingv1.IngressRuleValue{
                    HTTP: &networkingv1.HTTPIngressRuleValue{
                        Paths: []networkingv1.HTTPIngressPath{
                            {
                                Path:     "/",
                                PathType: &pathType,
                                Backend: networkingv1.IngressBackend{
                                    Service: &networkingv1.IngressServiceBackend{
                                        Name: game.Name,
                                        Port: networkingv1.ServiceBackendPort{
                                            Number: int32(port),
                                        },
                                    },
                                },
                            },
                        },
                    },
                },
            },
        },
    }
}

运行

修改Makefile中的镜像id，docker命令。dockerhub暂时用不了，先用阿里云吧。

IMAGE_TAG_BASE ?= registry.cn-hangzhou.aliyuncs.com/wgh9626/game
IMG ?= registry.cn-hangzhou.aliyuncs.com/wgh9626/game-controller:v$(VERSION)
CONTAINER_TOOL ?= nerdctl

安装crd。运行controller。

make install
make run

修改config/samples/game_v1_game.yaml文件

spec:
  replicas: 1
  image: alexwhen/docker-2048
  host: game.com

部署。

k apply -f config/samples/game_v1_game.yaml

查看ingress。

本地添加域名解析，测试访问。如果访问503，在ingress中添加class。

构建镜像，修改Dockerfile，添加goproxy，替换镜像id。

ENV GOPROXY=https://goproxy.cn,direct
#FROM gcr.io/distroless/static:nonroot
FROM gcr.ketches.cn/distroless/static:nonroot

这里会报错无法拉取go的镜像。本地下载修改tag推送到阿里云仓库，再次修改镜像id。

FROM registry.cn-hangzhou.aliyuncs.com/wgh9626/golang:1.20 AS builder

构建成功并推送

make docker-push

测试

验证扩容。首先部署controller。

make deploy

这里node节点会报错无法拉取controller镜像。本地保存再导入。

nerdctl pull registry.cn-hangzhou.aliyuncs.com/wgh9626/game-controller:v0.0.1
nerdctl save -o game-controller.tar ba2b03fa54a9
ctr -n k8s.io i import game-controller.tar

扩容pod。

k scale deployment game-sample --replicas 3

可以看到Game的PHASE，DESIRED，CURRENT，READY字段发生变化，扩容成功。

添加默认值及验证

要想实现这个功能，就需要添加webhook来校验字段的值。如果为空设置默认值，如果为非法字段提示报错信息。

operator-sdk create webhook --group game --version v1 --kind Game --defaulting --programmatic-validation
# 和kubebuilder的命令一样，可以添加下面三种webhook。
--conversion                if set, scaffold the conversion webhook
--defaulting                if set, scaffold the defaulting webhook即mutatingwebhook
--programmatic-validation   if set, scaffold the validating webhook即validatingwebhook

修改api/v1/game_webhook.go文件，定义了默认的镜像和默认的host即域名。并且在创建和更新时验证host是否含有特殊字符。

// 在创建或者更新时调用Default接口，mutating=true：可以修改请求对象，sideEffects=None: 这个webhook没有任何副作用，mutatingwebhook的name是mgame.kb.io
//+kubebuilder:webhook:path=/mutate-game-game-com-v1-game,mutating=true,failurePolicy=fail,sideEffects=None,groups=game.game.com,resources=games,verbs=create;update,versions=v1,name=mgame.kb.io,admissionReviewVersions=v1

var _ webhook.Defaulter = &Game{}

const (
    defaultImage = "alexwhen/docker-2048"
    defaultHost  = "game.com"
)

// Default implements webhook.Defaulter so a webhook will be registered for the type
func (r *Game) Default() {
    gamelog.Info("default", "name", r.Name)

    // TODO(user): fill in your defaulting logic.
    if r.Spec.Image == "" {
        r.Spec.Image = defaultImage
    }
    if r.Spec.Host == "" {
        r.Spec.Host = defaultHost
    }
}

// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
// 在创建或者更新时调用Validate接口
//+kubebuilder:webhook:path=/validate-game-game-com-v1-game,mutating=false,failurePolicy=fail,sideEffects=None,groups=game.game.com,resources=games,verbs=create;update,versions=v1,name=vgame.kb.io,admissionReviewVersions=v1

var _ webhook.Validator = &Game{}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateCreate() (admission.Warnings, error) {
    gamelog.Info("validate create", "name", r.Name)

    // TODO(user): fill in your validation logic upon object creation.
    specialChars := "!@#$%^&*()_+{}|:\"<>?`~"
    if strings.ContainsAny(r.Spec.Host, specialChars) {
        return nil, errors.New("host should not contain special characters")
    }
    return nil, nil
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
    gamelog.Info("validate update", "name", r.Name)

    // TODO(user): fill in your validation logic upon object update.
    specialChars := "!@#$%^&*()_+{}|:\"<>?`~"
    if strings.ContainsAny(r.Spec.Host, specialChars) {
        return nil, errors.New("host should not contain special characters")
    }
    return nil, nil
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateDelete() (admission.Warnings, error) {
    gamelog.Info("validate delete", "name", r.Name)

    // TODO(user): fill in your validation logic upon object deletion.
    return nil, nil
}

取消cert-manager相关注释，否则会因为没有证书导致controller启动失败。

先修改config/default/kustomization.yaml文件：

- ../certmanager
- path: webhookcainjection_patch.yaml
replacements:
  - source:
    ....

再修改config/crd/kustomization.yaml文件：

# patches here are for enabling the CA injection for each CRD
- path: patches/cainjection_in_games.yaml

重新安装crd

make install

生成manifests，即MutatingWebhookConfiguration和ValidatingWebhookConfiguration。

make manifests

构建镜像

# 修改版本号为0.0.2
vim Makefile
VERSION ?= 0.0.2
make docker-build && make docker-push

重新部署

make undeploy
make deploy

controller启动成功。

测试默认值和字段验证。首先测试默认值：

apiVersion: game.game.com/v1
kind: Game
metadata:
  labels:
    app.kubernetes.io/name: game
  name: game-default-test
spec:
  replicas: 1
  image:
  host:

创建成功，镜像和ingress都被设置成了默认的。

测试特殊字符：

apiVersion: game.game.com/v1
kind: Game
metadata:
  labels:
    app.kubernetes.io/name: game
  name: game-special-test
spec:
  replicas: 1
  image: alexwhen/docker-2048
  host: game#.com

创建pod报错：admission webhook "vgame.kb.io" denied the request: host should not contain special characters

符合预期。