Files
Sylve/internal/services/libvirt/libvirt_utils.go
T

762 lines
20 KiB
Go

// SPDX-License-Identifier: BSD-2-Clause
//
// Copyright (c) 2025 The FreeBSD Foundation.
//
// This software was developed by Hayzam Sherif <hayzam@alchemilla.io>
// of Alchemilla Ventures Pvt. Ltd. <hello@alchemilla.io>,
// under sponsorship from the FreeBSD Foundation.
package libvirt
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/alchemillahq/sylve/internal/config"
utilitiesModels "github.com/alchemillahq/sylve/internal/db/models/utilities"
vmModels "github.com/alchemillahq/sylve/internal/db/models/vm"
libvirtServiceInterfaces "github.com/alchemillahq/sylve/internal/interfaces/services/libvirt"
"github.com/alchemillahq/sylve/internal/logger"
qemuimg "github.com/alchemillahq/sylve/pkg/qemu-img"
"github.com/alchemillahq/sylve/pkg/utils"
"github.com/digitalocean/go-libvirt"
"github.com/klauspost/cpuid/v2"
)
var flashImageToDiskCtx = utils.FlashImageToDiskCtx
var inspectDiskImageFormat = qemuimg.Info
var convertDiskImageToRaw = qemuimg.Convert
var sniffMediaMIME = func(path string) (string, error) {
mime, _, err := utils.SniffMIME(path)
return mime, err
}
func domainReasonToString(state libvirt.DomainState, reason int32) libvirtServiceInterfaces.DomainStateReason {
switch state {
case libvirt.DomainRunning:
switch reason {
case 0:
return libvirtServiceInterfaces.DomainReasonUnknown
case 1:
return libvirtServiceInterfaces.DomainReasonRunningBooted
case 2:
return libvirtServiceInterfaces.DomainReasonRunningMigrated
case 3:
return libvirtServiceInterfaces.DomainReasonRunningRestored
case 4:
return libvirtServiceInterfaces.DomainReasonRunningFromSnapshot
case 5:
return libvirtServiceInterfaces.DomainReasonRunningUnpaused
case 6:
return libvirtServiceInterfaces.DomainReasonRunningMigrationCanceled
case 7:
return libvirtServiceInterfaces.DomainReasonRunningSaveCanceled
case 8:
return libvirtServiceInterfaces.DomainReasonRunningWakeup
case 9:
return libvirtServiceInterfaces.DomainReasonRunningCrashed
default:
return libvirtServiceInterfaces.DomainReasonUnknown
}
case libvirt.DomainShutoff:
switch reason {
case 0:
return libvirtServiceInterfaces.DomainReasonUnknown
case 1:
return libvirtServiceInterfaces.DomainReasonShutoffShutdown
case 2:
return libvirtServiceInterfaces.DomainReasonShutoffDestroyed
case 3:
return libvirtServiceInterfaces.DomainReasonShutoffCrashed
case 4:
return libvirtServiceInterfaces.DomainReasonShutoffSaved
case 5:
return libvirtServiceInterfaces.DomainReasonShutoffFailed
case 6:
return libvirtServiceInterfaces.DomainReasonShutoffFromSnapshot
default:
return libvirtServiceInterfaces.DomainReasonUnknown
}
case libvirt.DomainPaused:
switch reason {
case 0:
return libvirtServiceInterfaces.DomainReasonUnknown
case 1:
return libvirtServiceInterfaces.DomainReasonPausedUser
case 2:
return libvirtServiceInterfaces.DomainReasonPausedMigration
case 3:
return libvirtServiceInterfaces.DomainReasonPausedSave
case 4:
return libvirtServiceInterfaces.DomainReasonPausedDump
case 5:
return libvirtServiceInterfaces.DomainReasonPausedIOError
case 6:
return libvirtServiceInterfaces.DomainReasonPausedWatchdog
case 7:
return libvirtServiceInterfaces.DomainReasonPausedFromSnapshot
case 8:
return libvirtServiceInterfaces.DomainReasonPausedShuttingDown
case 9:
return libvirtServiceInterfaces.DomainReasonPausedSnapshot
default:
return libvirtServiceInterfaces.DomainReasonUnknown
}
default:
return libvirtServiceInterfaces.DomainReasonUnknown
}
}
func (s *Service) FindISOByUUID(uuid string, includeImg bool) (string, error) {
var download utilitiesModels.Downloads
if err := s.DB.
Preload("Files").
Where("uuid = ?", uuid).
First(&download).Error; err != nil {
return "", fmt.Errorf("failed_to_find_download: %w", err)
}
fileExists := func(p string) bool {
if p == "" {
return false
}
fi, err := os.Stat(p)
return err == nil && !fi.IsDir()
}
compressedCache := map[string]bool{}
isCompressedContainer := func(p string) bool {
if !fileExists(p) {
return false
}
if cached, ok := compressedCache[p]; ok {
return cached
}
mime, err := sniffMediaMIME(p)
if err != nil {
compressedCache[p] = false
return false
}
switch mime {
case "application/gzip",
"application/x-bzip2",
"application/x-xz",
"application/zstd",
"application/x-compress":
compressedCache[p] = true
return true
default:
compressedCache[p] = false
return false
}
}
isISOPath := func(p string) bool {
if p == "" {
return false
}
l := strings.ToLower(p)
if strings.HasSuffix(l, ".iso") {
return true
}
mime, err := sniffMediaMIME(p)
if err != nil {
return false
}
return strings.Contains(strings.ToLower(mime), "iso9660")
}
qemuUsableCache := map[string]bool{}
isQemuUsableImage := func(p string) bool {
if !fileExists(p) {
return false
}
if cached, ok := qemuUsableCache[p]; ok {
return cached
}
info, err := inspectDiskImageFormat(p)
if err != nil || info == nil {
qemuUsableCache[p] = false
return false
}
format := strings.ToLower(strings.TrimSpace(info.Format))
usable := format != ""
qemuUsableCache[p] = usable
return usable
}
isResolvableMediaFile := func(p string) bool {
if !fileExists(p) {
return false
}
if isCompressedContainer(p) {
return false
}
if !includeImg {
return isISOPath(p)
}
if isQemuUsableImage(p) {
return true
}
return isISOPath(p)
}
candidates := make([]string, 0, 12)
seen := make(map[string]struct{})
addCandidate := func(p string) {
p = strings.TrimSpace(p)
if p == "" {
return
}
if _, ok := seen[p]; ok {
return
}
seen[p] = struct{}{}
candidates = append(candidates, p)
}
addCandidatesFromDir := func(dir string) {
if dir == "" {
return
}
entries, err := os.ReadDir(dir)
if err != nil {
return
}
for _, entry := range entries {
if entry.IsDir() {
continue
}
full := filepath.Join(dir, entry.Name())
addCandidate(full)
}
}
httpMainPath := filepath.Join(config.GetDownloadsPath("http"), download.Name)
pathFallbackPath := filepath.Join(config.GetDownloadsPath("path"), download.Name)
extractedRoot := filepath.Join(config.GetDownloadsPath("extracted"), download.UUID)
switch download.Type {
case "http":
addCandidate(httpMainPath)
addCandidate(download.ExtractedPath)
addCandidatesFromDir(download.ExtractedPath)
addCandidate(extractedRoot)
addCandidatesFromDir(extractedRoot)
case "torrent":
torrentsRoot := filepath.Join(config.GetDownloadsPath("torrents"), uuid)
for _, f := range download.Files {
addCandidate(filepath.Join(torrentsRoot, f.Name))
}
addCandidate(filepath.Join(torrentsRoot, download.Name))
addCandidatesFromDir(torrentsRoot)
addCandidate(download.ExtractedPath)
addCandidatesFromDir(download.ExtractedPath)
addCandidate(extractedRoot)
addCandidatesFromDir(extractedRoot)
case "path":
addCandidate(download.Path)
addCandidate(pathFallbackPath)
addCandidate(download.ExtractedPath)
addCandidatesFromDir(download.ExtractedPath)
addCandidate(extractedRoot)
addCandidatesFromDir(extractedRoot)
default:
return "", fmt.Errorf("unsupported_download_type: %s", download.Type)
}
for _, candidate := range candidates {
if isResolvableMediaFile(candidate) {
return candidate, nil
}
}
if download.Type == "torrent" {
return "", fmt.Errorf("iso_or_img_not_found_in_torrent: %s", uuid)
}
if download.Type == "path" {
return "", fmt.Errorf(
"iso_or_img_not_found_in_path: path=%s (exists=%t, allowed=%t) fallback=%s (exists=%t, allowed=%t) extracted=%s (exists=%t, allowed=%t)",
download.Path, fileExists(download.Path), isResolvableMediaFile(download.Path),
pathFallbackPath, fileExists(pathFallbackPath), isResolvableMediaFile(pathFallbackPath),
download.ExtractedPath, fileExists(download.ExtractedPath), isResolvableMediaFile(download.ExtractedPath),
)
}
return "", fmt.Errorf(
"iso_or_img_not_found: main=%s (exists=%t, allowed=%t) extracted=%s (exists=%t, allowed=%t)",
httpMainPath, fileExists(httpMainPath), isResolvableMediaFile(httpMainPath),
download.ExtractedPath, fileExists(download.ExtractedPath), isResolvableMediaFile(download.ExtractedPath),
)
}
func (s *Service) GetDomainStates() ([]libvirtServiceInterfaces.DomainState, error) {
var states []libvirtServiceInterfaces.DomainState
if err := s.requireConnection(); err != nil {
return states, err
}
flags := libvirt.ConnectListDomainsActive | libvirt.ConnectListDomainsInactive
domains, _, err := s.conn().ConnectListAllDomains(1, flags)
if err != nil {
return states, err
}
for _, d := range domains {
state, reason, err := s.conn().DomainGetState(d, 0)
if err != nil {
fmt.Printf("failed to get domain state: %v\n", err)
}
pState := libvirt.DomainState(state)
states = append(states, libvirtServiceInterfaces.DomainState{
Domain: d.Name,
State: pState,
Reason: domainReasonToString(pState, reason),
})
}
return states, nil
}
func (s *Service) IsDomainShutOff(rid uint) (bool, error) {
if err := s.requireConnection(); err != nil {
return false, err
}
domain, err := s.conn().DomainLookupByName(strconv.Itoa(int(rid)))
if err != nil {
return false, fmt.Errorf("failed_to_lookup_domain_by_name: %w", err)
}
state, _, err := s.conn().DomainGetState(domain, 0)
if err != nil {
return false, fmt.Errorf("failed_to_get_domain_state: %w", err)
}
if state == int32(libvirt.DomainShutoff) {
return true, nil
}
return false, nil
}
func (s *Service) IsDomainShutOffByID(id uint) (bool, error) {
var rid uint
if err := s.DB.Model(&vmModels.VM{}).
Where("id = ?", id).
Select("rid").
Scan(&rid).Error; err != nil {
return false, fmt.Errorf("failed_to_get_vm_rid: %w", err)
}
return s.IsDomainShutOff(rid)
}
func (s *Service) CreateVMDirectory(rid uint) (string, error) {
vmDir, err := config.GetVMsPath()
if err != nil {
return "", fmt.Errorf("failed to get VMs path: %w", err)
}
vmPath := fmt.Sprintf("%s/%d", vmDir, rid)
if _, err := os.Stat(vmPath); err == nil {
if err := os.RemoveAll(vmPath); err != nil {
return "", fmt.Errorf("failed to clear VM directory: %w", err)
}
}
if err := os.MkdirAll(vmPath, 0755); err != nil {
return "", fmt.Errorf("failed to create VM directory: %w", err)
}
return vmPath, nil
}
func (s *Service) ResetUEFIVars(rid uint) error {
vmDir, err := config.GetVMsPath()
if err != nil {
return fmt.Errorf("failed to get VMs path: %w", err)
}
vmPath := fmt.Sprintf("%s/%d", vmDir, rid)
uefiVarsBase := "/usr/local/share/uefi-firmware/BHYVE_UEFI_VARS.fd"
uefiVarsPath := filepath.Join(vmPath, fmt.Sprintf("%d_vars.fd", rid))
err = utils.CopyFile(uefiVarsBase, uefiVarsPath)
if err != nil {
if strings.Contains("failed_to_open_source", err.Error()) {
logger.L.Err(err).Msg("Error finding BHYVE_UEFI_VARS file, do we have bhyve-firmware?")
} else {
return err
}
}
return nil
}
func (s *Service) ValidateCPUPins(rid uint, pins []libvirtServiceInterfaces.CPUPinning, hostLogicalPerSocket int) error {
if len(pins) == 0 {
return nil
}
hostLogicalCores := utils.GetLogicalCores()
hostSocketCount := utils.GetSocketCount(cpuid.CPU.PhysicalCores,
cpuid.CPU.ThreadsPerCore)
if hostSocketCount <= 0 {
return fmt.Errorf("invalid_host_socket_count")
}
if hostLogicalCores <= 0 {
return fmt.Errorf("invalid_host_logical_cores")
}
if hostLogicalPerSocket <= 0 {
hostLogicalPerSocket = hostLogicalCores / hostSocketCount
if hostLogicalPerSocket <= 0 {
hostLogicalPerSocket = hostLogicalCores
}
}
seenSockets := make(map[int]struct{}, len(pins))
for i, pin := range pins {
if pin.Socket < 0 || pin.Socket >= hostSocketCount {
return fmt.Errorf("socket_index_out_of_range: socket=%d max=%d", pin.Socket, hostSocketCount-1)
}
if _, dup := seenSockets[pin.Socket]; dup {
return fmt.Errorf("duplicate_socket_in_request: socket=%d index=%d", pin.Socket, i)
}
seenSockets[pin.Socket] = struct{}{}
if len(pin.Cores) == 0 {
return fmt.Errorf("empty_core_list_for_socket: socket=%d", pin.Socket)
}
}
actualHostCores := make(map[int]struct{}, 128)
perSocketCounts := make(map[int]int, hostSocketCount)
totalPinned := 0
for _, pin := range pins {
baseCore := pin.Socket * hostLogicalPerSocket
perSocketSeen := make(map[int]struct{}, len(pin.Cores))
for j, c := range pin.Cores {
if c < 0 || c >= hostLogicalPerSocket {
return fmt.Errorf("core_index_out_of_range: core=%d (max=%d) socket=%d pos=%d",
c, hostLogicalPerSocket-1, pin.Socket, j)
}
if _, dup := perSocketSeen[c]; dup {
return fmt.Errorf("duplicate_core_within_socket: core=%d socket=%d", c, pin.Socket)
}
perSocketSeen[c] = struct{}{}
actualHostCore := baseCore + c
if actualHostCore >= hostLogicalCores {
return fmt.Errorf("calculated_core_out_of_range: socket=%d coreIdx=%d actualCore=%d max=%d",
pin.Socket, c, actualHostCore, hostLogicalCores-1)
}
if _, dup := actualHostCores[actualHostCore]; dup {
return fmt.Errorf("duplicate_core_across_sockets: core=%d", actualHostCore)
}
actualHostCores[actualHostCore] = struct{}{}
}
perSocketCounts[pin.Socket] += len(pin.Cores)
totalPinned += len(pin.Cores)
}
if totalPinned > hostLogicalCores {
return fmt.Errorf("cpu_pinning_exceeds_logical_cores: pinned=%d logical=%d", totalPinned, hostLogicalCores)
}
if hostLogicalPerSocket > 0 {
for sock, cnt := range perSocketCounts {
if cnt > hostLogicalPerSocket {
return fmt.Errorf("socket_capacity_exceeded: socket=%d pinned=%d cap=%d",
sock, cnt, hostLogicalPerSocket)
}
}
}
var vms []vmModels.VM
if err := s.DB.Preload("CPUPinning").Find(&vms).Error; err != nil {
return fmt.Errorf("failed_to_fetch_vms: %w", err)
}
occupied := make(map[int]uint, 512)
for _, vm := range vms {
if rid != 0 && uint(vm.RID) == rid {
continue
}
for _, p := range vm.CPUPinning {
baseCore := p.HostSocket * hostLogicalPerSocket
for _, c := range p.HostCPU {
globalCore := baseCore + c
if globalCore < 0 || globalCore >= hostLogicalCores {
continue
}
occupied[globalCore] = uint(vm.RID)
}
}
}
for c := range actualHostCores {
if owner, taken := occupied[c]; taken {
return fmt.Errorf("core_conflict: core=%d already_pinned_by_rid=%d", c, owner)
}
}
return nil
}
func (s *Service) GeneratePinArgs(pins []vmModels.VMCPUPinning) []string {
var args []string
vcpu := 0
socketCount := utils.GetSocketCount(cpuid.CPU.PhysicalCores, cpuid.CPU.ThreadsPerCore)
if socketCount <= 0 {
socketCount = 1
}
coresPerSocket := cpuid.CPU.LogicalCores / socketCount
if coresPerSocket <= 0 {
coresPerSocket = cpuid.CPU.LogicalCores
}
for _, p := range pins {
for _, localCPU := range p.HostCPU {
globalCPU := p.HostSocket*coresPerSocket + localCPU
args = append(args, fmt.Sprintf("-p %d:%d", vcpu, globalCPU))
vcpu++
}
}
return args
}
func (s *Service) GetVMConfigDirectory(rid uint) (string, error) {
vmDir, err := config.GetVMsPath()
if err != nil {
return "", fmt.Errorf("failed to get VMs path: %w", err)
}
return fmt.Sprintf("%s/%d", vmDir, rid), nil
}
func (s *Service) CreateCloudInitISO(vm vmModels.VM) error {
if vm.CloudInitData == "" && vm.CloudInitMetaData == "" {
return nil
}
vmPath, err := s.GetVMConfigDirectory(vm.RID)
if err != nil {
return fmt.Errorf("failed_to_get_vm_path: %w", err)
}
cloudInitISOPath := filepath.Join(vmPath, "cloud-init.iso")
if _, err := os.Stat(cloudInitISOPath); err == nil {
if err := os.Remove(cloudInitISOPath); err != nil {
return fmt.Errorf("failed_to_remove_existing_cloud_init_iso: %w", err)
}
}
cloudInitPath := filepath.Join(vmPath, "cloud-init")
if _, err := os.Stat(cloudInitPath); err == nil {
if err := os.RemoveAll(cloudInitPath); err != nil {
return fmt.Errorf("failed_to_remove_existing_cloud_init_directory: %w", err)
}
}
if err := os.MkdirAll(cloudInitPath, 0755); err != nil {
return fmt.Errorf("failed_to_create_cloud_init_directory: %w", err)
}
userDataPath := filepath.Join(cloudInitPath, "user-data")
metaDataPath := filepath.Join(cloudInitPath, "meta-data")
networkConfigPath := filepath.Join(cloudInitPath, "network-config")
err = os.WriteFile(userDataPath, []byte(vm.CloudInitData), 0644)
if err != nil {
return fmt.Errorf("failed_to_write_user_data: %w", err)
}
err = os.WriteFile(metaDataPath, []byte(vm.CloudInitMetaData), 0644)
if err != nil {
return fmt.Errorf("failed_to_write_meta_data: %w", err)
}
if vm.CloudInitNetworkConfig != "" {
err = os.WriteFile(networkConfigPath, []byte(vm.CloudInitNetworkConfig), 0644)
if err != nil {
return fmt.Errorf("failed_to_write_network_config: %w", err)
}
}
isoPath := filepath.Join(vmPath, "cloud-init.iso")
_, err = utils.RunCommand("/usr/sbin/makefs", "-t", "cd9660", "-o", "rockridge", "-o", "label=cidata", isoPath, cloudInitPath)
if err != nil {
return fmt.Errorf("failed_to_create_cloud_init_iso: %w", err)
}
return nil
}
func (s *Service) GetCloudInitISOPath(rid uint) (string, error) {
vmPath, err := s.GetVMConfigDirectory(rid)
if err != nil {
return "", fmt.Errorf("failed_to_get_vm_path: %w", err)
}
cloudInitISOPath := filepath.Join(vmPath, "cloud-init.iso")
if _, err := os.Stat(cloudInitISOPath); err != nil {
return "", fmt.Errorf("cloud_init_iso_not_found: %w", err)
}
return cloudInitISOPath, nil
}
func (s *Service) FlashCloudInitMediaToDisk(vm vmModels.VM) error {
enabledStorages := make([]vmModels.Storage, 0, len(vm.Storages))
for _, storage := range vm.Storages {
if storage.Enable {
enabledStorages = append(enabledStorages, storage)
}
}
if len(enabledStorages) == 0 {
return fmt.Errorf("need_storage_to_flash_cloud_init_disk")
} else if len(enabledStorages) > 2 {
return fmt.Errorf("too_many_storages_to_flash_cloud_init_disk")
}
if vm.CloudInitData == "" && vm.CloudInitMetaData == "" {
return nil
}
var mediaStorage *vmModels.Storage
var diskStorage *vmModels.Storage
for _, storage := range enabledStorages {
if storage.Type == vmModels.VMStorageTypeDiskImage {
mediaStorage = &storage
} else if storage.Type == vmModels.VMStorageTypeRaw ||
storage.Type == vmModels.VMStorageTypeZVol {
diskStorage = &storage
}
}
if mediaStorage == nil || diskStorage == nil {
return fmt.Errorf("media_and_disk_required")
}
mediaPath, err := s.FindISOByUUID(mediaStorage.DownloadUUID, true)
if err != nil {
return fmt.Errorf("failed_to_find_media_iso: %w", err)
}
mediaInfo, err := os.Stat(mediaPath)
if err != nil {
return fmt.Errorf("failed_to_stat_media_iso: %w", err)
}
mediaSize := mediaInfo.Size()
mediaFormat := string(qemuimg.FormatRaw)
if info, infoErr := inspectDiskImageFormat(mediaPath); infoErr == nil && info != nil {
if trimmed := strings.TrimSpace(info.Format); trimmed != "" {
mediaFormat = strings.ToLower(trimmed)
}
if info.VirtualSize > 0 {
mediaSize = info.VirtualSize
}
} else if infoErr != nil {
logger.L.Warn().
Uint("rid", vm.RID).
Str("mediaPath", mediaPath).
Err(infoErr).
Msg("cloud_init_media_format_probe_failed_assuming_raw")
}
if diskStorage.Size < mediaSize {
return fmt.Errorf("disk_too_small_for_media: disk_size=%d media_size=%d",
diskStorage.Size, mediaSize)
}
var storagePath string
if diskStorage.Type == vmModels.VMStorageTypeRaw {
storagePath = fmt.Sprintf(
"/%s/sylve/virtual-machines/%d/raw-%d/%d.img",
diskStorage.Dataset.Pool,
vm.RID,
diskStorage.ID,
diskStorage.ID,
)
if _, err := os.Stat(storagePath); err != nil {
return fmt.Errorf("disk_image_not_found: %w", err)
}
} else if diskStorage.Type == vmModels.VMStorageTypeZVol {
storagePath = fmt.Sprintf(
"/dev/zvol/%s/sylve/virtual-machines/%d/zvol-%d",
diskStorage.Dataset.Pool,
vm.RID,
diskStorage.ID,
)
if _, err := os.Stat(storagePath); err != nil {
return fmt.Errorf("zvol_not_found: %w", err)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
if strings.EqualFold(mediaFormat, string(qemuimg.FormatRaw)) {
if err := flashImageToDiskCtx(ctx, mediaPath, storagePath); err != nil {
return fmt.Errorf("failed_to_flash_media_to_disk: %w", err)
}
return nil
}
if err := convertDiskImageToRaw(mediaPath, storagePath, qemuimg.FormatRaw); err != nil {
return fmt.Errorf("failed_to_convert_media_to_raw_disk: %w", err)
}
return nil
}