KubeVirt

Kubevirt Hook Sidecar

2024-05-12

简介 #

背景 #

在kubevirt中, 通过vmi的spec没办法涵盖所有的libvirt domain xml元素, 所以有了hook sidecar功能来允许我们在define domain之前自定义domainSpecXML

功能介绍 #

在kubevirt中, Hook Sidecar容器是sidecar container(和main application container跑在同一个pod中)用来在vm初始化完成前执行一些自定义操作.

sidecar container与main container(compute)通过gRPC通讯, 有两种主要的sidecar hooks

  1. OnDefineDomain: 这个hook帮助自定义libvirt的XML, 并通过gRPC协议返回最新的XML以创建vm
  2. PreCloudInitIso: 这个hook帮助定义cloud-init配置, 它运行并返回最新的cloud-init data
  3. Shutdown: 这个是v1alpha3版本才支持的

使用hook sidecar功能需要在kv.spec.configuration.developerConfiguration.featureGates中开启Sidecar功能

源码分析 #

kubevirt-boot-sidecar 介绍 #

以下以kubevirt-boot-sidecar为例讲述sidecar的工作流程, 这个sidecar支持修改引导设备顺序(boot)和开启交互式引导菜单(bootmenu)

kubevirt-boot-sidecar只实现了OnDefineDomain, 下面也是主要串一下OnDefineDomain相关的

sidecar工作流程 #

  1. virt-launcher刚启动时收集所有sidecar信息
    // cmd/virt-launcher/virt-launcher.go
func main() {
    hookSidecars := pflag.Uint("hook-sidecars", 0, "Number of requested hook sidecars, virt-launcher will wait for all of them to become available")
    // 收集所有sidecar的信息
    err := hookManager.Collect(*hookSidecars, *qemuTimeout)

    // 启动 cmd server, 这里面有 SyncVirtualMachine 方法, 具体的实现在 func (l *LibvirtDomainManager) SyncVMI
    // virt-handler在初始化完虚拟机硬盘等之后会通过 SyncVirtualMachine 调用SyncVMI函数开始创建domain
    // SyncVMI将vmi spec转换为domainSpec, 然后调用hooksManager.OnDefineDomain执行所有的sidecar的OnDefineDomain方法
    // 最终用OnDefineDomain编辑后的domainSpec创建domain
    cmdServerDone := startCmdServer(cmdclient.UninitializedSocketOnGuest(), domainManager, stopChan, options)
}

// pkg/hooks/manager.go
// numberOfRequestedHookSidecars为vmi注解 hooks.kubevirt.io/hookSidecars 的数组长度, 在virt-controller生成pod manifest的逻辑中计算得出
func (m *hookManager) Collect(numberOfRequestedHookSidecars uint, timeout time.Duration) error {
    // callbacksPerHookPoint
    callbacksPerHookPoint, err := m.collectSideCarSockets(numberOfRequestedHookSidecars, timeout)
    m.CallbacksPerHookPoint = callbacksPerHookPoint
}

// pkg/hooks/manager.go
func (m *hookManager) collectSideCarSockets(numberOfRequestedHookSidecars uint, timeout time.Duration) (map[string][]*callBackClient, error) {
    callbacksPerHookPoint := make(map[string][]*callBackClient)
    processedSockets := make(map[string]bool)
    timeoutCh := time.After(timeout)

    for uint(len(processedSockets)) < numberOfRequestedHookSidecars {
        sockets, err := os.ReadDir(m.hookSocketSharedDirectory)
        // 遍历 /var/run/kubevirt-hooks/ 目录下的 unix socket 文件
        for _, socket := range sockets {
            select {
            case <-timeoutCh:
                return nil, fmt.Errorf("Failed to collect all expected sidecar hook sockets within given timeout")
            default:
                if _, processed := processedSockets[socket.Name()]; processed {
                    continue
                }

                // 连接 sock 文件对应的 sidecar server 的 Info 函数获取 server 实现了哪些 hook(onDefineDomain或preCloudInitIso)
                callBackClient, notReady, err := processSideCarSocket(filepath.Join(m.hookSocketSharedDirectory, socket.Name()))
                if notReady {
                    log.Log.Info("Sidecar server might not be ready yet, retrying in the next iteration")
                    continue
                } else if err != nil {
                    return nil, err
                }

                // callbacksPerHookPoint[onDefineDomain|preCloudInitIso][]*callBackClient{}
                // 聚合出 onDefineDomain:["aaaa.sock","bbbb.sock"]
                for _, subscribedHookPoint := range callBackClient.subscribedHookPoints {
                    callbacksPerHookPoint[subscribedHookPoint.GetName()] = append(callbacksPerHookPoint[subscribedHookPoint.GetName()], callBackClient)
                }

                processedSockets[socket.Name()] = true
            }
        }
        time.Sleep(time.Second)
    }
    // {"onDefineDomain":[{"SocketPath":"/var/run/kubevirt-hooks/shim-xxxx.sock", "Version":"v1alpha3", "subscribedHookPoints": [{"name": "onDefineDomain", "priority": 0}]}]}
    return callbacksPerHookPoint, nil
}
  2. virt-launcher启动之后, virt-handler会执行一些本地盘等相关初始化配置后通过gRPC调用virt-launcher的SyncVirtualMachine方法开始创建domain
    1. SyncVMI
      1. Convert_v1_VirtualMachineInstance_To_api_Domain 将 vmi 转换为 domainSpec
      2. lookupOrCreateVirDomain 先LookupDomainByName, 如果已存在则直接退出
        1. preStartHook
          hooksManager := hooks.GetManager()
// 执行所有的 PreCloudInitIso sidecar
cloudInitData, err = hooksManager.PreCloudInitIso(vmi, cloudInitData)
        2. setDomainSpecWithHooks
          // pkg/virt-launcher/virtwarp/util/libvirt-helper.go
func SetDomainSpecStrWithHooks(virConn cli.Connection, vmi *v1.VirtualMachineInstance, wantedSpec *api.DomainSpec) (cli.VirDomain, error) {
    hooksManager := getHookManager()
    // 执行所有的 OnDefineDomain sidecar
    domainSpec, err := hooksManager.OnDefineDomain(wantedSpec, vmi)
    // 调用 virConn.DomainDefineXML 创建 domain
    return SetDomainSpecStr(virConn, vmi, domainSpec)
}

// /pkg/hooks/manager.go
func (m *hookManager) OnDefineDomain(domainSpec *virtwrapApi.DomainSpec, vmi *v1.VirtualMachineInstance) (string, error) {
    domainSpecXML, err := xml.MarshalIndent(domainSpec, "", "\t")

    callbacks, found := m.CallbacksPerHookPoint[hooksInfo.OnDefineDomainHookPointName]
    if !found {
        return string(domainSpecXML), nil
    }

    vmiJSON, err := json.Marshal(vmi)

    for _, callback := range callbacks {
        // 执行所有的sidecar OnDefineDomain函数, 一次次编辑domainSpecXML
        domainSpecXML, err = m.onDefineDomainCallback(callback, domainSpecXML, vmiJSON)
    }

    return string(domainSpecXML), nil
}

// /pkg/hooks/manager.go
func (m *hookManager) onDefineDomainCallback(callback *callBackClient, domainSpecXML, vmiJSON []byte) ([]byte, error) {
    // dial /var/run/kubevirt-hooks/shim-xxxx.sock
    conn, err := grpcutil.DialSocketWithTimeout(callback.SocketPath, 1)

    switch callback.Version {
    case hooksV1alpha3.Version:
        client := hooksV1alpha3.NewCallbacksClient(conn)
        // 调用sidecar server 的 OnDefineDomain 方法
        result, err := client.OnDefineDomain(ctx, &hooksV1alpha3.OnDefineDomainParams{
            DomainXML: domainSpecXML,
            Vmi:       vmiJSON,
        })
        domainSpecXML = result.GetDomainXML()
    }

    return domainSpecXML, nil
}

会发现上面主要是sidecar client视角, 没有介绍sidecar server在哪实现的, 最新的解决方案是搭配sidecar-shim, 下面开始介绍

...
2025-03-11