使用operator-sdk创建一个2048小游戏的operator，并构建镜像，使用域名访问。
创建项目
mkdir game && cd game
go mod init game
operator-sdk init --domain game.com --project-name game --plugins go/v4 --owner wgh
# --domain string domain for groups (default "my.domain")
# --owner string owner to add to the copyright
# --project-name string name of this project
# --plugins strings plugin keys to be used for this subcommand execution
# --repo string name to use for go module (e.g., github.com/user/repo), defaults to the go package of the current working directory。不在$GOPATH/src中也可以使用这个参数。
tree .
.
├── cmd
│ └── main.go
├── config
│ ├── default
│ │ ├── kustomization.yaml
│ │ ├── manager_auth_proxy_patch.yaml
│ │ └── manager_config_patch.yaml
│ ├── manager
│ │ ├── kustomization.yaml
│ │ └── manager.yaml
│ ├── manifests
│ │ └── kustomization.yaml
│ ├── prometheus
│ │ ├── kustomization.yaml
│ │ └── monitor.yaml
│ ├── rbac
│ │ ├── auth_proxy_client_clusterrole.yaml
│ │ ├── auth_proxy_role_binding.yaml
│ │ ├── auth_proxy_role.yaml
│ │ ├── auth_proxy_service.yaml
│ │ ├── kustomization.yaml
│ │ ├── leader_election_role_binding.yaml
│ │ ├── leader_election_role.yaml
│ │ ├── role_binding.yaml
│ │ ├── role.yaml
│ │ └── service_account.yaml
│ └── scorecard
│ ├── bases
│ │ └── config.yaml
│ ├── kustomization.yaml
│ └── patches
│ ├── basic.config.yaml
│ └── olm.config.yaml
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│ └── boilerplate.go.txt
├── Makefile
├── PROJECT
├── README.md
└── test
├── e2e
│ ├── e2e_suite_test.go
│ └── e2e_test.go
└── utils
└── utils.go
14 directories, 33 files创建api
operator-sdk create api --group game --version v1 --kind Game --resource --controller
# --group string resource Group
# --kind string resource Kind
# --resource if set, generate the resource without prompting the user (default true)
# --controller if set, generate the controller without prompting the user (default true)
tree .
.
├── api
│ └── v1
│ ├── game_types.go
│ ├── groupversion_info.go
│ └── zz_generated.deepcopy.go
├── bin
│ └── controller-gen
├── cmd
│ └── main.go
├── config
│ ├── crd
│ │ ├── kustomization.yaml
│ │ └── kustomizeconfig.yaml
│ ├── default
│ │ ├── kustomization.yaml
│ │ ├── manager_auth_proxy_patch.yaml
│ │ └── manager_config_patch.yaml
│ ├── manager
│ │ ├── kustomization.yaml
│ │ └── manager.yaml
│ ├── manifests
│ │ └── kustomization.yaml
│ ├── prometheus
│ │ ├── kustomization.yaml
│ │ └── monitor.yaml
│ ├── rbac
│ │ ├── auth_proxy_client_clusterrole.yaml
│ │ ├── auth_proxy_role_binding.yaml
│ │ ├── auth_proxy_role.yaml
│ │ ├── auth_proxy_service.yaml
│ │ ├── game_editor_role.yaml
│ │ ├── game_viewer_role.yaml
│ │ ├── kustomization.yaml
│ │ ├── leader_election_role_binding.yaml
│ │ ├── leader_election_role.yaml
│ │ ├── role_binding.yaml
│ │ ├── role.yaml
│ │ └── service_account.yaml
│ ├── samples
│ │ ├── game_v1_game.yaml
│ │ └── kustomization.yaml
│ └── scorecard
│ ├── bases
│ │ └── config.yaml
│ ├── kustomization.yaml
│ └── patches
│ ├── basic.config.yaml
│ └── olm.config.yaml
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│ └── boilerplate.go.txt
├── internal
│ └── controller
│ ├── game_controller.go
│ ├── game_controller_test.go
│ └── suite_test.go
├── Makefile
├── PROJECT
├── README.md
└── test
├── e2e
│ ├── e2e_suite_test.go
│ └── e2e_test.go
└── utils
└── utils.go
设计api
修改api/v1/game_types.go文件：
// Replicas 字段定义了游戏的副本数量,默认为 1,最小为 1,最大为 3。Host为游戏服务的地址。
type GameSpec struct {
//+kubebuilder:default:=1
//+kubebuilder:validation:Maximum:=3
//+kubebuilder:validation:Minimum:=1
Replicas int32 `json:"replicas,omitempty"`
Image string `json:"image,omitempty"`
Host string `json:"host,omitempty"`
}
const (
Init = "Init"
Running = "Running"
NotReady = "NotReady"
Failed = "Failed"
)
// GameStatus defines the observed state of Game
type GameStatus struct {
// Phase 字段定义了游戏的当前状态,可以是 Init、Running、NotReady 或 Failed。
Phase string `json:"phase,omitempty"`
// Replicas 字段定义了游戏的副本数量。
Replicas int32 `json:"replicas,omitempty"`
// ReadyReplicas 字段定义了游戏的就绪副本数量。
ReadyReplicas int32 `json:"readyReplicas,omitempty"`
// LabelSelector 字段定义了游戏的标签选择器。用于HPA。
LabelSelector string `json:"labelSelector,omitempty"`
}
// 添加了scale子资源，并添加子资源specpath，statuspath，selectorpath对应字段的定义。最后添加打印列。
//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
//+kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
//+kubebuilder:printcolumn:name="PHASE",type=string,JSONPath=`.status.phase`,description="Game Phase"
//+kubebuilder:printcolumn:name="Host",type=string,JSONPath=`.spec.host`,description="Host address"
//+kubebuilder:printcolumn:name="DESIRED",type=integer,JSONPath=`.spec.replicas`,description="Desired Replicas"
//+kubebuilder:printcolumn:name="CURRENT",type=integer,JSONPath=`.status.replicas`,description="Current Replicas"
//+kubebuilder:printcolumn:name="READY",type=integer,JSONPath=`.status.readyReplicas`,description="Ready Replicas"
//+kubebuilder:printcolumn:name="AGE",type=date,JSONPath=`.metadata.creationTimestamp`,description="CreationTimestamp"修改控制器逻辑
修改internal/controller/game_controller.go文件：
package controller
import (
"context"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/utils/pointer"
"reflect"
"sigs.k8s.io/controller-runtime/pkg/controller"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
gamev1 "game/api/v1"
)
// GameReconciler reconciles a Game object
type GameReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// 添加rbac规则
//+kubebuilder:rbac:groups=game.game.com,resources=games,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=game.game.com,resources=games/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=game.game.com,resources=games/finalizers,verbs=update
//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps,resources=deployments/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Game object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile
func (r *GameReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
defer utilruntime.HandleCrash()
logger := log.FromContext(ctx)
logger.Info("Reconciling Game", "name", req.String())
// 根据req获取Game资源
game := &gamev1.Game{}
if err := r.Get(ctx, req.NamespacedName, game); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// 判断是否在被删除
if game.DeletionTimestamp != nil {
logger.Info("Deleting Game", "name", req.String())
return ctrl.Result{}, nil
}
// 调用syncGame
if err := r.syncGame(ctx, game); err != nil {
logger.Error(err, "Failed to sync Game", "name", req.String())
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
const (
gameLabelName = "game-label"
port = 80
)
// syncGame 确保Game的Deployment，Service和Ingress都处于预期的状态，
func (r *GameReconciler) syncGame(ctx context.Context, game *gamev1.Game) error {
logger := log.FromContext(ctx)
// 创建深拷贝，防止直接修改原有对象
game = game.DeepCopy()
name := types.NamespacedName{
Namespace: game.Namespace,
Name: game.Name,
}
// 创建OwnerReference数组，将Game资源设置为子资源的所有者。确保当Game资源被删除时，它的子资源也会被自动删除。
ower := []metav1.OwnerReference{
{
APIVersion: game.APIVersion,
Kind: game.Kind,
Name: game.Name,
// pointer.Bool(true)返回一个指向true的布尔型指针，赋值给Controller字段
Controller: pointer.Bool(true),
// 指定BlockOwnerDeletion为true，用于控制是否应该在删除父资源时同时删除子资源。
BlockOwnerDeletion: pointer.Bool(true),
UID: game.UID,
},
}
// 使用Game资源的名称创建一个标签映射，用于标识该资源所管理的子资源。
labels := map[string]string{
gameLabelName: name.Name,
}
// 设置元数据
meta := metav1.ObjectMeta{
Name: name.Name,
Namespace: name.Namespace,
Labels: labels,
OwnerReferences: ower,
}
// 创建Deployment
deploy := &appsv1.Deployment{}
if err := r.Get(ctx, name, deploy); err != nil {
if !errors.IsNotFound(err) {
return err
}
deploy = &appsv1.Deployment{
ObjectMeta: meta,
// 获取Deployment的期望Spec
Spec: getDeploymentSpec(game, labels),
}
if err := r.Create(ctx, deploy); err != nil {
return err
}
logger.Info("Create deployment success", "name", name.String())
} else {
// 比较期望Spec和当前的Spec是否完全一致，如果不同则更新Deployment为期望Spec。
desire := getDeploymentSpec(game, labels)
now := getSpecFromDeployment(deploy)
if !reflect.DeepEqual(desire, now) {
newDeploy := deploy.DeepCopy()
newDeploy.Spec = desire
if err := r.Update(ctx, newDeploy); err != nil {
return err
}
logger.Info("Update deployment success", "name", name.String())
}
}
// 创建Service
svc := &corev1.Service{}
if err := r.Get(ctx, name, svc); err != nil {
if !errors.IsNotFound(err) {
return err
}
svc = &corev1.Service{
ObjectMeta: meta,
Spec: corev1.ServiceSpec{
Selector: labels,
Ports: []corev1.ServicePort{
{
Name: "http",
Port: int32(port),
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.IntOrString{
Type: intstr.Int,
IntVal: port,
},
},
},
},
}
if err := r.Create(ctx, svc); err != nil {
return err
}
logger.Info("Create service success", "name", name.String())
}
// 创建Ingress
ing := &networkingv1.Ingress{}
if err := r.Get(ctx, name, ing); err != nil {
if !errors.IsNotFound(err) {
return err
}
ing = &networkingv1.Ingress{
ObjectMeta: meta,
Spec: getIngressSpec(game),
}
if err := r.Create(ctx, ing); err != nil {
return err
}
logger.Info("Create ingress success", "name", name.String())
}
// 更新Game资源的Status
newStatus := gamev1.GameStatus{
Replicas: deploy.Status.Replicas,
ReadyReplicas: deploy.Status.ReadyReplicas,
}
// 如果就绪副本数等于当前副本数,则将状态设置为Running; 否则设置为NotReady。
if game.Status.Replicas == newStatus.ReadyReplicas {
newStatus.Phase = gamev1.Running
} else {
newStatus.Phase = gamev1.NotReady
}
// 如果Game资源的状态发生变化,则更新Game资源的状态子资源和Game资源。
if !reflect.DeepEqual(game.Status, newStatus) {
game.Status = newStatus
logger.Info("Update game status", "name", name.String())
if err := r.Client.Status().Update(ctx, game); err != nil {
return err
}
game.Spec.Replicas = newStatus.Replicas
logger.Info("Update game spec", "name", name.String())
if err := r.Client.Update(ctx, game); err != nil {
return err
}
}
return nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *GameReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
// MaxConcurrentReconciles是可以运行的最大并发Reconciles数。默认值为 1。
WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
For(&gamev1.Game{}).
Owns(&appsv1.Deployment{}).
Complete(r)
}
// getDeploymentSpec 根据Game和labels，生成Deployment的期望Spec
func getDeploymentSpec(game *gamev1.Game, labels map[string]string) appsv1.DeploymentSpec {
return appsv1.DeploymentSpec{
// 使用指定的副本数
Replicas: pointer.Int32(game.Spec.Replicas),
// 使用提供的标签映射创建一个LabelSelector
Selector: metav1.SetAsLabelSelector(labels),
// 创建Pod模板，设置标签，指定容器名字和镜像。
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "game",
Image: game.Spec.Image,
},
},
},
},
}
}
// getSpecFromDeployment 从Deployment中获取Spec
func getSpecFromDeployment(deploy *appsv1.Deployment) appsv1.DeploymentSpec {
// 获取第一个容器的Spec，pod只有一个主容器
container := deploy.Spec.Template.Spec.Containers[0]
return appsv1.DeploymentSpec{
Replicas: deploy.Spec.Replicas,
Selector: deploy.Spec.Selector,
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
// 从Deployment的Pod模板规格中获取标签
Labels: deploy.Spec.Template.Labels,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: container.Name,
Image: container.Image,
},
},
},
},
}
}
// getIngressSpec 根据Game和labels，生成Ingress的期望Spec
func getIngressSpec(game *gamev1.Game) networkingv1.IngressSpec {
pathType := networkingv1.PathTypePrefix
return networkingv1.IngressSpec{
Rules: []networkingv1.IngressRule{
{
Host: game.Spec.Host,
IngressRuleValue: networkingv1.IngressRuleValue{
HTTP: &networkingv1.HTTPIngressRuleValue{
Paths: []networkingv1.HTTPIngressPath{
{
Path: "/",
PathType: &pathType,
Backend: networkingv1.IngressBackend{
Service: &networkingv1.IngressServiceBackend{
Name: game.Name,
Port: networkingv1.ServiceBackendPort{
Number: int32(port),
},
},
},
},
},
},
},
},
},
}
}
运行
修改Makefile中的镜像id，docker命令。dockerhub暂时用不了，先用阿里云吧。
IMAGE_TAG_BASE ?= registry.cn-hangzhou.aliyuncs.com/wgh9626/game
IMG ?= registry.cn-hangzhou.aliyuncs.com/wgh9626/game-controller:v$(VERSION)
CONTAINER_TOOL ?= nerdctl安装crd。运行controller。
make install
make run
修改config/samples/game_v1_game.yaml文件
spec:
replicas: 1
image: alexwhen/docker-2048
host: game.com部署。
k apply -f config/samples/game_v1_game.yaml
查看ingress。
本地添加域名解析，测试访问。如果访问503，在ingress中添加class。
构建镜像，修改Dockerfile，添加goproxy，替换镜像id。
ENV GOPROXY=https://goproxy.cn,direct
#FROM gcr.io/distroless/static:nonroot
FROM gcr.ketches.cn/distroless/static:nonroot这里会报错无法拉取go的镜像。本地下载修改tag推送到阿里云仓库，再次修改镜像id。
FROM registry.cn-hangzhou.aliyuncs.com/wgh9626/golang:1.20 AS builder构建成功并推送
make docker-push测试
验证扩容。首先部署controller。
make deploy
这里node节点会报错无法拉取controller镜像。本地保存再导入。
nerdctl pull registry.cn-hangzhou.aliyuncs.com/wgh9626/game-controller:v0.0.1
nerdctl save -o game-controller.tar ba2b03fa54a9
ctr -n k8s.io i import game-controller.tar
扩容pod。
k scale deployment game-sample --replicas 3
可以看到Game的PHASE，DESIRED，CURRENT，READY字段发生变化，扩容成功。
添加默认值及验证
要想实现这个功能，就需要添加webhook来校验字段的值。如果为空设置默认值，如果为非法字段提示报错信息。
operator-sdk create webhook --group game --version v1 --kind Game --defaulting --programmatic-validation
# 和kubebuilder的命令一样，可以添加下面三种webhook。
--conversion if set, scaffold the conversion webhook
--defaulting if set, scaffold the defaulting webhook即mutatingwebhook
--programmatic-validation if set, scaffold the validating webhook即validatingwebhook
修改api/v1/game_webhook.go文件，定义了默认的镜像和默认的host即域名。并且在创建和更新时验证host是否含有特殊字符。
// 在创建或者更新时调用Default接口，mutating=true：可以修改请求对象，sideEffects=None: 这个webhook没有任何副作用，mutatingwebhook的name是mgame.kb.io
//+kubebuilder:webhook:path=/mutate-game-game-com-v1-game,mutating=true,failurePolicy=fail,sideEffects=None,groups=game.game.com,resources=games,verbs=create;update,versions=v1,name=mgame.kb.io,admissionReviewVersions=v1
var _ webhook.Defaulter = &Game{}
const (
defaultImage = "alexwhen/docker-2048"
defaultHost = "game.com"
)
// Default implements webhook.Defaulter so a webhook will be registered for the type
func (r *Game) Default() {
gamelog.Info("default", "name", r.Name)
// TODO(user): fill in your defaulting logic.
if r.Spec.Image == "" {
r.Spec.Image = defaultImage
}
if r.Spec.Host == "" {
r.Spec.Host = defaultHost
}
}
// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
// 在创建或者更新时调用Validate接口
//+kubebuilder:webhook:path=/validate-game-game-com-v1-game,mutating=false,failurePolicy=fail,sideEffects=None,groups=game.game.com,resources=games,verbs=create;update,versions=v1,name=vgame.kb.io,admissionReviewVersions=v1
var _ webhook.Validator = &Game{}
// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateCreate() (admission.Warnings, error) {
gamelog.Info("validate create", "name", r.Name)
// TODO(user): fill in your validation logic upon object creation.
specialChars := "!@#$%^&*()_+{}|:\"<>?`~"
if strings.ContainsAny(r.Spec.Host, specialChars) {
return nil, errors.New("host should not contain special characters")
}
return nil, nil
}
// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
gamelog.Info("validate update", "name", r.Name)
// TODO(user): fill in your validation logic upon object update.
specialChars := "!@#$%^&*()_+{}|:\"<>?`~"
if strings.ContainsAny(r.Spec.Host, specialChars) {
return nil, errors.New("host should not contain special characters")
}
return nil, nil
}
// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
func (r *Game) ValidateDelete() (admission.Warnings, error) {
gamelog.Info("validate delete", "name", r.Name)
// TODO(user): fill in your validation logic upon object deletion.
return nil, nil
}取消cert-manager相关注释，否则会因为没有证书导致controller启动失败。
先修改config/default/kustomization.yaml文件：
- ../certmanager
- path: webhookcainjection_patch.yaml
replacements:
- source:
....再修改config/crd/kustomization.yaml文件：
# patches here are for enabling the CA injection for each CRD
- path: patches/cainjection_in_games.yaml重新安装crd
make install生成manifests，即MutatingWebhookConfiguration和ValidatingWebhookConfiguration。
make manifests构建镜像
# 修改版本号为0.0.2
vim Makefile
VERSION ?= 0.0.2
make docker-build && make docker-push重新部署
make undeploy
make deploycontroller启动成功。
测试默认值和字段验证。首先测试默认值：
apiVersion: game.game.com/v1
kind: Game
metadata:
labels:
app.kubernetes.io/name: game
name: game-default-test
spec:
replicas: 1
image:
host:创建成功，镜像和ingress都被设置成了默认的。
测试特殊字符：
apiVersion: game.game.com/v1
kind: Game
metadata:
labels:
app.kubernetes.io/name: game
name: game-special-test
spec:
replicas: 1
image: alexwhen/docker-2048
host: game#.com创建pod报错：admission webhook "vgame.kb.io" denied the request: host should not contain special characters
符合预期。