diff --git a/internal/handlers/disk/disk.go b/internal/handlers/disk/disk.go index a11f5a4f..fa318121 100644 --- a/internal/handlers/disk/disk.go +++ b/internal/handlers/disk/disk.go @@ -76,8 +76,6 @@ func WipeDisk(diskService *disk.Service, infoService *info.Service) gin.HandlerF return func(c *gin.Context) { var r DiskActionRequest - fmt.Println("WipeDisk") - if err := c.ShouldBindJSON(&r); err != nil { validationErrors := utils.MapValidationErrors(err, DiskActionRequest{}) @@ -93,8 +91,6 @@ func WipeDisk(diskService *disk.Service, infoService *info.Service) gin.HandlerF id := infoService.StartAuditLog(c.GetString("Token"), fmt.Sprintf("wipe_disk|-|%s", r.Device), "started") err := diskUtils.DestroyDisk(r.Device) - fmt.Println("WipeDisk", r.Device, id) - if err != nil { c.JSON(http.StatusInternalServerError, internal.APIResponse[any]{ Status: "error", diff --git a/internal/handlers/info/terminal.go b/internal/handlers/info/terminal.go index e8236967..4c859c6a 100644 --- a/internal/handlers/info/terminal.go +++ b/internal/handlers/info/terminal.go @@ -63,11 +63,14 @@ func HandleTerminalWebsocket(c *gin.Context) { w := c.Writer r := c.Request - conn, err := WSUpgrader.Upgrade(w, r, nil) + subprotocols := websocket.Subprotocols(r) + conn, err := WSUpgrader.Upgrade(w, r, http.Header{"Sec-WebSocket-Protocol": {subprotocols[0]}}) + if err != nil { logger.L.Error().Msgf("WebSocket upgrade failed: %v", err) return } + defer conn.Close() var wsWriteMu sync.Mutex diff --git a/internal/handlers/routes.go b/internal/handlers/routes.go index bf38e80e..3737ebf1 100644 --- a/internal/handlers/routes.go +++ b/internal/handlers/routes.go @@ -90,8 +90,8 @@ func RegisterRoutes(r *gin.Engine, pools := zfs.Group("/pools") { pools.GET("", zfsHandlers.GetPools(zfsService)) - pools.POST("", zfsHandlers.GetPools(zfsService)) - pools.DELETE("/:name", zfsHandlers.GetPools(zfsService)) + pools.POST("", zfsHandlers.CreatePool(zfsService)) + pools.DELETE("/:name", zfsHandlers.DeletePool(zfsService)) } zfs.GET("/pool/io-delay", zfsHandlers.AvgIODelay(zfsService)) diff --git a/internal/handlers/zfs/pool.go b/internal/handlers/zfs/pool.go index 79d48f0f..1fc3ba09 100644 --- a/internal/handlers/zfs/pool.go +++ b/internal/handlers/zfs/pool.go @@ -10,29 +10,21 @@ package zfsHandlers import ( "net/http" + "strings" "sylve/internal" infoModels "sylve/internal/db/models/info" zfsServiceInterfaces "sylve/internal/interfaces/services/zfs" "sylve/internal/services/zfs" "github.com/gin-gonic/gin" + + zfsUtils "sylve/pkg/zfs" ) type AvgIODelayResponse struct { Delay float64 `json:"delay"` } -type CreatePoolRequest struct { - Name string `json:"name" binding:"required,min=3,max=128"` - Vdevs []string `json:"vdevs" binding:"required"` - Raid string `json:"raid"` - Options map[string]string `json:"options" binding:"required"` -} - -type DeletePoolRequest struct { - Name string `json:"name"` -} - // @Summary Get Average IO Delay // @Description Get the average IO delay of all pools // @Tags ZFS @@ -44,7 +36,7 @@ type DeletePoolRequest struct { // @Router /zfs/avg-io-delay [get] func AvgIODelay(zfsSerice *zfs.Service) gin.HandlerFunc { return func(c *gin.Context) { - info := zfsSerice.GetTotalIODelay() + info := zfsUtils.GetTotalIODelay() c.JSON(http.StatusOK, internal.APIResponse[AvgIODelayResponse]{ Status: "success", Message: "avg_io_delay", @@ -91,12 +83,12 @@ func AvgIODelayHistorical(zfsSerice *zfs.Service) gin.HandlerFunc { // @Accept json // @Produce json // @Security BearerAuth -// @Success 200 {object} internal.APIResponse[[]zfsServiceInterfaces.Zpool] "Success" +// @Success 200 {object} internal.APIResponse[[]*zfsUtils.Zpool] "Success" // @Failure 500 {object} internal.APIResponse[any] "Internal Server Error" // @Router /zfs/pools [get] func GetPools(zfsSerice *zfs.Service) gin.HandlerFunc { return func(c *gin.Context) { - pools, err := zfsSerice.GetPools() + pools, err := zfsUtils.ListZpools() if err != nil { c.JSON(http.StatusInternalServerError, internal.APIResponse[any]{ Status: "error", @@ -107,7 +99,7 @@ func GetPools(zfsSerice *zfs.Service) gin.HandlerFunc { return } - c.JSON(http.StatusOK, internal.APIResponse[[]zfsServiceInterfaces.Zpool]{ + c.JSON(http.StatusOK, internal.APIResponse[[]*zfsUtils.Zpool]{ Status: "success", Message: "pools", Error: "", @@ -122,13 +114,13 @@ func GetPools(zfsSerice *zfs.Service) gin.HandlerFunc { // @Accept json // @Produce json // @Security BearerAuth -// @Param request body CreatePoolRequest true "Request" +// @Param request body zfsServiceInterfaces.Zpool true "Request" // @Success 200 {object} internal.APIResponse[any] "Success" // @Failure 500 {object} internal.APIResponse[any] "Internal Server Error" // @Router /zfs/pools [post] func CreatePool(zfsSerice *zfs.Service) gin.HandlerFunc { return func(c *gin.Context) { - var request CreatePoolRequest + var request zfsServiceInterfaces.Zpool if err := c.ShouldBindJSON(&request); err != nil { c.JSON(http.StatusBadRequest, internal.APIResponse[any]{ Status: "error", @@ -139,7 +131,7 @@ func CreatePool(zfsSerice *zfs.Service) gin.HandlerFunc { return } - err := zfsSerice.CreatePool(request.Name, request.Vdevs, request.Raid, request.Options) + err := zfsSerice.CreatePool(request) if err != nil { c.JSON(http.StatusInternalServerError, internal.APIResponse[any]{ Status: "error", @@ -165,25 +157,25 @@ func CreatePool(zfsSerice *zfs.Service) gin.HandlerFunc { // @Accept json // @Produce json // @Security BearerAuth -// @Param request body DeletePoolRequest true "Request" // @Success 200 {object} internal.APIResponse[any] "Success" // @Failure 500 {object} internal.APIResponse[any] "Internal Server Error" // @Router /zfs/pools/{name} [delete] func DeletePool(zfsSerice *zfs.Service) gin.HandlerFunc { return func(c *gin.Context) { - var request DeletePoolRequest - if err := c.ShouldBindJSON(&request); err != nil { - c.JSON(http.StatusBadRequest, internal.APIResponse[any]{ - Status: "error", - Message: "invalid_request", - Error: err.Error(), - Data: nil, - }) - return - } + name := c.Param("name") - err := zfsSerice.DestroyPool(request.Name) + err := zfsUtils.DestroyPool(name) if err != nil { + if strings.HasPrefix(err.Error(), "error_getting_pool") { + c.JSON(http.StatusNotFound, internal.APIResponse[any]{ + Status: "error", + Message: "pool_not_found", + Error: err.Error(), + Data: nil, + }) + return + } + c.JSON(http.StatusInternalServerError, internal.APIResponse[any]{ Status: "error", Message: "pool_delete_failed", diff --git a/internal/interfaces/services/zfs/pool.go b/internal/interfaces/services/zfs/pool.go index 2ee500b9..7ef8c50d 100644 --- a/internal/interfaces/services/zfs/pool.go +++ b/internal/interfaces/services/zfs/pool.go @@ -8,28 +8,14 @@ package zfsServiceInterfaces -type RW struct { - Read uint64 `json:"read"` - Write uint64 `json:"write"` -} - type Vdev struct { - Name string `json:"name"` - Alloc uint64 `json:"alloc"` - Free uint64 `json:"free"` - Operations RW `json:"operations"` - Bandwidth RW `json:"bandwidth"` + Name string `json:"name"` + VdevDevices []string `json:"devices"` } type Zpool struct { - Name string `json:"name"` - Health string `json:"health"` - Allocated uint64 `json:"allocated"` - Size uint64 `json:"size"` - Free uint64 `json:"free"` - ReadOnly bool `json:"readOnly"` - Freeing uint64 `json:"freeing"` - Leaked uint64 `json:"leaked"` - DedupRatio float64 `json:"dedupRatio"` - Vdevs []Vdev `json:"vdevs"` + Name string `json:"name" binding:"required,alphanum,min=1,max=24"` + RaidType string `json:"raidType" binding:"omitempty,oneof= mirror raidz raidz2 raidz3"` + Vdevs []Vdev `json:"vdevs"` + Properties map[string]string `json:"properties"` } diff --git a/internal/interfaces/services/zfs/zfs.go b/internal/interfaces/services/zfs/zfs.go index 9cefc69d..24214288 100644 --- a/internal/interfaces/services/zfs/zfs.go +++ b/internal/interfaces/services/zfs/zfs.go @@ -11,12 +11,8 @@ package zfsServiceInterfaces import infoModels "sylve/internal/db/models/info" type ZfsServiceInterface interface { - GetPoolNames() ([]string, error) - GetPool(name string) (Zpool, error) - GetPools() ([]Zpool, error) - GetPoolIODelay(poolName string) float64 - GetTotalIODelay() float64 GetTotalIODelayHisorical() ([]infoModels.IODelay, error) + CreatePool(Zpool) error Cron() } diff --git a/internal/services/disk/disk.go b/internal/services/disk/disk.go index 498bae90..5911b9e5 100644 --- a/internal/services/disk/disk.go +++ b/internal/services/disk/disk.go @@ -17,6 +17,7 @@ import ( "sylve/internal/logger" diskUtils "sylve/pkg/disk" "sylve/pkg/utils" + "sylve/pkg/zfs" "sync" "syscall" @@ -209,13 +210,13 @@ func (s *Service) GetDiskDevices() ([]diskServiceInterfaces.Disk, error) { if len(disk.Partitions) == 0 { found := false - pools, err := s.ZFS.GetPools() + pools, err := zfs.ListZpools() if err == nil { for _, pool := range pools { for _, vdev := range pool.Vdevs { if vdev.Name == "/dev/"+d.Name { - disk.Usage = "ZFS Vdev" + disk.Usage = "ZFS" found = true break } diff --git a/internal/services/zfs/pool.go b/internal/services/zfs/pool.go index b6ac5ac1..6d3c0ea8 100644 --- a/internal/services/zfs/pool.go +++ b/internal/services/zfs/pool.go @@ -9,172 +9,11 @@ package zfs import ( - "fmt" - "strconv" - "strings" "sylve/internal/db" infoModels "sylve/internal/db/models/info" zfsServiceInterfaces "sylve/internal/interfaces/services/zfs" - "sylve/internal/logger" - diskUtils "sylve/pkg/disk" - "sylve/pkg/utils" ) -func (s *Service) GetPoolNames() ([]string, error) { - output, err := utils.RunCommand("zpool", "list", "-H", "-o", "name") - if err != nil { - return nil, err - } - - poolNames := strings.Fields(output) - - return poolNames, nil -} - -func (s *Service) GetPool(name string) (zfsServiceInterfaces.Zpool, error) { - pools, err := utils.RunCommand("zpool", "list", "-H", "-p", "-o", "name,health,alloc,size,free,readonly,freeing,leaked,dedupratio", name) - if err != nil { - return zfsServiceInterfaces.Zpool{}, err - } - - vdevs, err := utils.RunCommand("zpool", "iostat", "-v", "-H", "-P", "-p", name) - if err != nil { - return zfsServiceInterfaces.Zpool{}, err - } - - zpool, err := utils.ParseZpoolListOutput(pools, vdevs) - - return *zpool, err -} - -func (s *Service) GetPools() ([]zfsServiceInterfaces.Zpool, error) { - names, err := s.GetPoolNames() - if err != nil { - return []zfsServiceInterfaces.Zpool{}, err - } - - var pools []zfsServiceInterfaces.Zpool - - for _, name := range names { - pool, err := s.GetPool(name) - if err != nil { - return []zfsServiceInterfaces.Zpool{}, err - } - pools = append(pools, pool) - } - - return pools, nil -} - -func (s *Service) GetPoolIODelay(poolName string) float64 { - names, err := s.GetPoolNames() - - if err != nil { - logger.L.Debug().Msgf("Error getting pool names: %v", err) - return 0.0 - } - - if !utils.StringInSlice(poolName, names) { - logger.L.Debug().Msgf("Pool %s not found", poolName) - return 0.0 - } - - output, err := utils.RunCommand("zpool", "iostat", "-l", "-H", "-v", poolName, "1", "2") - if err != nil { - return 0.0 - } - - lines := strings.Split(strings.TrimSpace(output), "\n") - - var samples [][]string - var currentSample []string - seenPools := make(map[string]bool) - - for _, line := range lines { - if strings.TrimSpace(line) == "" { - if len(currentSample) > 0 { - samples = append(samples, currentSample) - currentSample = nil - seenPools = make(map[string]bool) - } - continue - } - fields := strings.Fields(line) - if len(fields) == 0 { - continue - } - pool := fields[0] - if seenPools[pool] { - samples = append(samples, currentSample) - currentSample = nil - seenPools = make(map[string]bool) - } - seenPools[pool] = true - currentSample = append(currentSample, line) - } - - if len(currentSample) > 0 { - samples = append(samples, currentSample) - } - - if len(samples) < 2 { - return 0.0 - } - - secondSample := samples[1] - sampleInterval := int64(1000000) - - for _, line := range secondSample { - if len(line) > 0 && (line[0] == ' ' || line[0] == '\t') { - continue - } - fields := strings.Fields(line) - if len(fields) < 9 || fields[0] != poolName { - continue - } - - readOps, err1 := strconv.ParseInt(fields[3], 10, 64) - writeOps, err2 := strconv.ParseInt(fields[4], 10, 64) - if err1 != nil || err2 != nil || (readOps+writeOps) == 0 { - return 0.0 - } - - totalReadWait := utils.ParseZfsTimeUnit(fields[7]) - totalWriteWait := utils.ParseZfsTimeUnit(fields[8]) - totalWaitAccumulated := (readOps * totalReadWait) + (writeOps * totalWriteWait) - averageWait := totalWaitAccumulated / (readOps + writeOps) - - return (float64(averageWait) / float64(sampleInterval)) * 100 - } - - return 0.0 -} - -func (s *Service) GetTotalIODelay() float64 { - names, err := s.GetPoolNames() - if err != nil { - logger.L.Debug().Msgf("Error getting pool names: %v", err) - return 0.0 - } - - var totalDelay float64 - count := 0 - - for _, name := range names { - delay := s.GetPoolIODelay(name) - if delay > 0 { - totalDelay += delay - count++ - } - } - - if count == 0 { - return 0.0 - } - - return totalDelay / float64(count) -} - func (s *Service) GetTotalIODelayHisorical() ([]infoModels.IODelay, error) { historicalData, err := db.GetHistorical[infoModels.IODelay](s.DB, 128) @@ -185,77 +24,6 @@ func (s *Service) GetTotalIODelayHisorical() ([]infoModels.IODelay, error) { return historicalData, nil } -func (s *Service) CreatePool(poolName string, vdevs []string, raidType string, options map[string]string) error { - if poolName == "" { - return fmt.Errorf("no pool name specified") - } - - if len(vdevs) == 0 { - return fmt.Errorf("no vdevs specified") - } - - pools, err := s.GetPools() - - if err != nil { - return fmt.Errorf("error getting existing pools: %v", err) - } - - for _, pool := range pools { - if pool.Name == poolName { - return fmt.Errorf("pool %s already exists", poolName) - } - - for _, vdev := range pool.Vdevs { - for _, newVdev := range vdevs { - if vdev.Name == newVdev { - return fmt.Errorf("vdev %s already in use by pool %s", newVdev, pool.Name) - } - } - } - } - - var args []string - - for k, v := range options { - args = append(args, "-O", fmt.Sprintf("%s=%s", k, v)) - } - - args = append(args, "-f") - args = append(args, poolName) - - if raidType != "" { - args = append(args, raidType) - } - - args = append(args, vdevs...) - - _, err = utils.RunCommand("zpool", append([]string{"create"}, args...)...) - if err != nil { - return fmt.Errorf("failed to create pool: %w", err) - } - - return nil -} - -func (s *Service) DestroyPool(poolName string) error { - pool, err := s.GetPool(poolName) - - if err != nil { - return fmt.Errorf("error getting pool: %v", err) - } - - _, err = utils.RunCommand("zpool", "destroy", "-f", pool.Name) - - if err != nil { - return err - } - - for _, vdev := range pool.Vdevs { - err = diskUtils.DestroyDisk(vdev.Name) - if err != nil { - return fmt.Errorf("error destroying disk %s: %v", vdev.Name, err) - } - } - +func (s *Service) CreatePool(zfsServiceInterfaces.Zpool) error { return nil } diff --git a/internal/services/zfs/zfs.go b/internal/services/zfs/zfs.go index a40faf8a..39d6afcd 100644 --- a/internal/services/zfs/zfs.go +++ b/internal/services/zfs/zfs.go @@ -12,6 +12,7 @@ import ( "sylve/internal/db" infoModels "sylve/internal/db/models/info" zfsServiceInterfaces "sylve/internal/interfaces/services/zfs" + "sylve/pkg/zfs" "time" "gorm.io/gorm" @@ -30,7 +31,7 @@ func NewZfsService(db *gorm.DB) zfsServiceInterfaces.ZfsServiceInterface { } func (s *Service) StoreStats() { - d := s.GetTotalIODelay() + d := zfs.GetTotalIODelay() db.StoreAndTrimRecords(s.DB, &infoModels.IODelay{Delay: d}, 128) } diff --git a/pkg/exe/executer.go b/pkg/exe/executer.go new file mode 100644 index 00000000..541e71eb --- /dev/null +++ b/pkg/exe/executer.go @@ -0,0 +1,9 @@ +package exe + +import ( + "io" +) + +type Executor interface { + Run(stdin io.Reader, stdout io.Writer, stderr io.Writer, cmd string, args ...string) error +} diff --git a/pkg/exe/local.go b/pkg/exe/local.go new file mode 100644 index 00000000..b9929ba5 --- /dev/null +++ b/pkg/exe/local.go @@ -0,0 +1,26 @@ +package exe + +import ( + "io" + "os/exec" +) + +func NewLocalExecutor() Executor { + return &localExec{} +} + +type localExec struct{} + +func (l *localExec) Run(stdin io.Reader, stdout io.Writer, stderr io.Writer, cmd string, args ...string) error { + c := exec.Command(cmd, args...) + if stdin != nil { + c.Stdin = stdin + } + if stdout != nil { + c.Stdout = stdout + } + if stderr != nil { + c.Stderr = stderr + } + return c.Run() +} diff --git a/pkg/utils/strings.go b/pkg/utils/strings.go index a3e92f73..4af79858 100644 --- a/pkg/utils/strings.go +++ b/pkg/utils/strings.go @@ -76,7 +76,12 @@ func StringInSlice(a string, list []string) bool { } func StringToUint64(s string) uint64 { - r, _ := strconv.ParseUint(s, 10, 64) + r, error := strconv.ParseUint(s, 10, 64) + + if error != nil { + return 0 + } + return r } @@ -130,3 +135,37 @@ func BytesToSize(toType string, bytes float64) float64 { return bytes } } + +/* + * from zfs diff`s escape function: + * + * Prints a file name out a character at a time. If the character is + * not in the range of what we consider "printable" ASCII, display it + * as an escaped 3-digit octal value. ASCII values less than a space + * are all control characters and we declare the upper end as the + * DELete character. This also is the last 7-bit ASCII character. + * We choose to treat all 8-bit ASCII as not printable for this + * application. + */ +func UnescapeFilepath(path string) (string, error) { + buf := make([]byte, 0, len(path)) + llen := len(path) + for i := 0; i < llen; { + if path[i] == '\\' { + if llen < i+4 { + return "", fmt.Errorf("invalid octal code: too short") + } + octalCode := path[(i + 1):(i + 4)] + val, err := strconv.ParseUint(octalCode, 8, 8) + if err != nil { + return "", fmt.Errorf("invalid octal code: %w", err) + } + buf = append(buf, byte(val)) + i += 4 + } else { + buf = append(buf, path[i]) + i++ + } + } + return string(buf), nil +} diff --git a/pkg/utils/zfs.go b/pkg/utils/zfs.go deleted file mode 100644 index 5739d55f..00000000 --- a/pkg/utils/zfs.go +++ /dev/null @@ -1,68 +0,0 @@ -// SPDX-License-Identifier: BSD-2-Clause -// -// Copyright (c) 2025 The FreeBSD Foundation. -// -// This software was developed by Hayzam Sherif -// of Alchemilla Ventures Pvt. Ltd. , -// under sponsorship from the FreeBSD Foundation. - -package utils - -import ( - "strings" - zfsServiceInterfaces "sylve/internal/interfaces/services/zfs" -) - -func ParseZpoolListOutput(pools string, vdevs string) (*zfsServiceInterfaces.Zpool, error) { - poolSlice := strings.Split(strings.TrimSpace(pools), "\n") - vdevSlice := strings.Split(strings.TrimSpace(vdevs), "\n") - - zpool := &zfsServiceInterfaces.Zpool{} - - for _, pool := range poolSlice { - if pool == "" { - continue - } - - parts := strings.Fields(pool) - if len(parts) < 9 { - continue - } - - zpool.Name = parts[0] - zpool.Health = parts[1] - zpool.Allocated = StringToUint64(parts[2]) - zpool.Size = StringToUint64(parts[3]) - zpool.Free = StringToUint64(parts[4]) - zpool.ReadOnly = parts[5] == "on" - zpool.Freeing = StringToUint64(parts[6]) - zpool.Leaked = StringToUint64(parts[7]) - zpool.DedupRatio = StringToFloat64(parts[8]) - - for _, vdev := range vdevSlice { - if strings.HasPrefix(vdev, zpool.Name) { - continue - } - - vdevParts := strings.Fields(vdev) - - if len(vdevParts) < 7 { - continue - } - - vdev := zfsServiceInterfaces.Vdev{} - - vdev.Name = vdevParts[0] - vdev.Alloc = StringToUint64(vdevParts[1]) - vdev.Free = StringToUint64(vdevParts[2]) - vdev.Operations.Read = StringToUint64(vdevParts[3]) - vdev.Operations.Write = StringToUint64(vdevParts[4]) - vdev.Bandwidth.Read = StringToUint64(vdevParts[5]) - vdev.Bandwidth.Write = StringToUint64(vdevParts[6]) - - zpool.Vdevs = append(zpool.Vdevs, vdev) - } - } - - return zpool, nil -} diff --git a/pkg/zfs/LICENSE b/pkg/zfs/LICENSE new file mode 100644 index 00000000..ca06c310 --- /dev/null +++ b/pkg/zfs/LICENSE @@ -0,0 +1,203 @@ +Parts of the code in this package are derived from the go-zfs package which has been modified to add some more functionality and also tighter integration with Sylve. Please refer to the license of the original go-zfs project below: + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2014, OmniTI Computer Consulting, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/pkg/zfs/dataset.go b/pkg/zfs/dataset.go new file mode 100644 index 00000000..76b7d5f6 --- /dev/null +++ b/pkg/zfs/dataset.go @@ -0,0 +1,311 @@ +package zfs + +import ( + "errors" + "fmt" + "io" + "strconv" + "strings" +) + +type Dataset struct { + z *zfs `json:"-"` + Name string `json:"name"` + Origin string `json:"origin"` + Used uint64 `json:"used"` + Avail uint64 `json:"avail"` + Mountpoint string `json:"mountpoint"` + Compression string `json:"compression"` + Type string `json:"type"` + Written uint64 `json:"written"` + Volsize uint64 `json:"volsize"` + Logicalused uint64 `json:"logicalused"` + Usedbydataset uint64 `json:"usedbydataset"` + Quota uint64 `json:"quota"` + Referenced uint64 `json:"referenced"` + + props map[string]string `json:"props"` +} + +func (d *Dataset) Clone(dest string, properties map[string]string) (*Dataset, error) { + if d.Type != DatasetSnapshot { + return nil, errors.New("can only clone snapshots") + } + args := make([]string, 2, 4) + args[0] = "clone" + args[1] = "-p" + if properties != nil { + args = append(args, propsSlice(properties)...) + } + args = append(args, []string{d.Name, dest}...) + if err := d.z.do(args...); err != nil { + return nil, err + } + return d.z.GetDataset(dest) +} + +func (d *Dataset) Unmount(force bool) (*Dataset, error) { + if d.Type == DatasetSnapshot { + return nil, errors.New("cannot unmount snapshots") + } + args := make([]string, 1, 3) + args[0] = "umount" + if force { + args = append(args, "-f") + } + args = append(args, d.Name) + if err := d.z.do(args...); err != nil { + return nil, err + } + return d.z.GetDataset(d.Name) +} + +func (d *Dataset) Mount(overlay bool, options []string) (*Dataset, error) { + if d.Type == DatasetSnapshot { + return nil, errors.New("cannot mount snapshots") + } + args := make([]string, 1, 5) + args[0] = "mount" + if overlay { + args = append(args, "-O") + } + if options != nil { + args = append(args, "-o") + args = append(args, strings.Join(options, ",")) + } + args = append(args, d.Name) + if err := d.z.do(args...); err != nil { + return nil, err + } + return d.z.GetDataset(d.Name) +} + +func (d *Dataset) Destroy(flags DestroyFlag) error { + args := make([]string, 1, 3) + args[0] = "destroy" + if flags&DestroyRecursive != 0 { + args = append(args, "-r") + } + + if flags&DestroyRecursiveClones != 0 { + args = append(args, "-R") + } + + if flags&DestroyDeferDeletion != 0 { + args = append(args, "-d") + } + + if flags&DestroyForceUmount != 0 { + args = append(args, "-f") + } + + args = append(args, d.Name) + err := d.z.do(args...) + return err +} + +func (d *Dataset) SetProperty(key, val string) error { + prop := strings.Join([]string{key, val}, "=") + if err := d.z.do("set", prop, d.Name); err != nil { + return err + } + d.props[strings.ToLower(key)] = val + return nil +} + +func (d *Dataset) SetProperties(keyValPairs ...string) error { + if len(keyValPairs) == 0 { + return nil + } + if len(keyValPairs)%2 != 0 { + return errors.New("keyValPairs must be an even number of strings") + } + args := []string{"set"} + props := make(map[string]string) + for i := 0; i < len(keyValPairs); i += 2 { + props[strings.ToLower(keyValPairs[i])] = keyValPairs[i+1] + args = append(args, strings.Join(keyValPairs[i:i+2], "=")) + } + args = append(args, d.Name) + if err := d.z.do(args...); err != nil { + return err + } + for k, v := range props { + d.props[k] = v + } + return nil +} + +func (d *Dataset) GetProperty(key string) (string, error) { + if v, ok := d.props[strings.ToLower(key)]; ok { + return v, nil + } + // custom properties does not return error + if strings.Contains(key, ":") { + return "-", nil + } + out, err := d.z.doOutput("get", "-H", "-p", key, d.Name) + if err != nil { + return "", err + } + + return out[0][2], nil +} + +func (d *Dataset) GetProperties(keys ...string) ([]string, error) { + if len(keys) == 0 { + return nil, nil + } + props, failed := make([]string, 0, len(keys)), false + for _, v := range keys { + val, ok := d.props[strings.ToLower(v)] + if failed = !ok && !strings.Contains(v, ":"); failed { + props = make([]string, 0, len(keys)) + break + } + if val == "" { + val = "-" + } + props = append(props, val) + } + if !failed { + return props, nil + } + out, err := d.z.doOutput("get", "-H", "-p", strings.Join(keys, ","), d.Name) + if err != nil { + return nil, err + } + for _, v := range out { + props = append(props, v[2]) + } + return props, nil +} + +func (d *Dataset) GetAllProperties() (map[string]string, error) { + out, err := d.z.doOutput("get", "-H", "-p", "all", d.Name) + if err != nil { + return nil, err + } + props := make(map[string]string) + for _, v := range out { + props[v[1]] = v[2] + } + return props, nil +} + +func (d *Dataset) Rename(name string, createParent, recursiveRenameSnapshots bool) (*Dataset, error) { + args := make([]string, 3, 5) + args[0] = "rename" + args[1] = d.Name + args[2] = name + if createParent { + args = append(args, "-p") + } + if recursiveRenameSnapshots { + args = append(args, "-r") + } + if err := d.z.do(args...); err != nil { + return d, err + } + + return d.z.GetDataset(name) +} + +func (d *Dataset) Snapshots() ([]*Dataset, error) { + return d.z.Snapshots(d.Name) +} + +func (d *Dataset) SendSnapshot(output io.Writer) error { + if d.Type != DatasetSnapshot { + return errors.New("can only send snapshots") + } + _, err := d.z.run(nil, output, "zfs", "send", d.Name) + return err +} + +func (d *Dataset) IncrementalSend(baseSnapshot *Dataset, output io.Writer) error { + if d.Type != DatasetSnapshot || baseSnapshot.Type != DatasetSnapshot { + return errors.New("can only send snapshots") + } + _, err := d.z.run(nil, output, "zfs", "send", "-i", baseSnapshot.Name, d.Name) + return err +} + +func (d *Dataset) Snapshot(name string, recursive bool) (*Dataset, error) { + args := make([]string, 1, 4) + args[0] = "snapshot" + if recursive { + args = append(args, "-r") + } + snapName := fmt.Sprintf("%s@%s", d.Name, name) + args = append(args, snapName) + if err := d.z.do(args...); err != nil { + return nil, err + } + return d.z.GetDataset(snapName) +} + +func (d *Dataset) Rollback(destroyMoreRecent bool) error { + if d.Type != DatasetSnapshot { + return errors.New("can only rollback snapshots") + } + + args := make([]string, 1, 3) + args[0] = "rollback" + if destroyMoreRecent { + args = append(args, "-r") + } + args = append(args, d.Name) + + err := d.z.do(args...) + return err +} + +func (d *Dataset) Children(depth uint64) ([]*Dataset, error) { + args := []string{"list"} + if depth > 0 { + args = append(args, "-d") + args = append(args, strconv.FormatUint(depth, 10)) + } else { + args = append(args, "-r") + } + args = append(args, "-t", "all", "-p", "-o", "all") + args = append(args, d.Name) + + out, err := d.z.doOutput(args...) + if err != nil { + return nil, err + } + + if len(out) == 0 { + return nil, nil + } + + var datasets []*Dataset + name := "" + var ds *Dataset + for _, line := range out[1:] { + if name != line[0] { + name = line[0] + ds = &Dataset{z: d.z, Name: name, props: make(map[string]string)} + datasets = append(datasets, ds) + } + if err := ds.parseProps([][]string{out[0], line}); err != nil { + return nil, err + } + } + return datasets[1:], nil +} + +func (d *Dataset) Diff(snapshot string) ([]*InodeChange, error) { + args := []string{"diff", "-FH", snapshot, d.Name} + out, err := d.z.doOutput(args...) + if err != nil { + return nil, err + } + inodeChanges, err := parseInodeChanges(out) + if err != nil { + return nil, err + } + return inodeChanges, nil +} diff --git a/pkg/zfs/default.go b/pkg/zfs/default.go new file mode 100644 index 00000000..8a16fa2b --- /dev/null +++ b/pkg/zfs/default.go @@ -0,0 +1,66 @@ +package zfs + +import ( + "fmt" + "sylve/pkg/exe" +) + +var z ZFS = &zfs{exec: exe.NewLocalExecutor(), sudo: false} + +func SetDefault(zfs ZFS) { + if zfs != nil { + z = zfs + } +} + +func Datasets(filter string) ([]*Dataset, error) { + return z.Datasets(filter) +} + +func Snapshots(filter string) ([]*Dataset, error) { + return z.Snapshots(filter) +} + +func GetZpool(name string) (*Zpool, error) { + return z.GetZpool(name) +} + +func ListZpools() ([]*Zpool, error) { + return z.ListZpools() +} + +func GetPoolIODelay(poolName string) (float64, error) { + return z.GetPoolIODelay(poolName) +} + +func GetTotalIODelay() float64 { + return z.GetTotalIODelay() +} + +func DestroyPool(poolName string) error { + var pools []*Zpool + pools, err := ListZpools() + if err != nil { + return err + } + + var found *Zpool + + for _, pool := range pools { + if pool.Name == poolName { + found = pool + break + } + } + + if found == nil { + return fmt.Errorf("error_getting_pool: pool %s not found", poolName) + } + + err = found.Destroy() + if err != nil { + return fmt.Errorf("failed to destroy pool: %w", err) + } + + return nil +} diff --git a/pkg/zfs/error.go b/pkg/zfs/error.go new file mode 100644 index 00000000..e1228a2f --- /dev/null +++ b/pkg/zfs/error.go @@ -0,0 +1,13 @@ +package zfs + +import "fmt" + +type Error struct { + Err error + Debug string + Stderr string +} + +func (e Error) Error() string { + return fmt.Sprintf("%s: %q => %s", e.Err, e.Debug, e.Stderr) +} diff --git a/pkg/zfs/helpers.go b/pkg/zfs/helpers.go new file mode 100644 index 00000000..1695a178 --- /dev/null +++ b/pkg/zfs/helpers.go @@ -0,0 +1,205 @@ +package zfs + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "sylve/pkg/utils" +) + +const ( + _ = iota + BlockDevice InodeType = iota + CharacterDevice + Directory + Door + NamedPipe + SymbolicLink + EventPort + Socket + File +) + +const ( + _ = iota + Removed ChangeType = iota + Created + Modified + Renamed +) + +var ( + dsPropList = []string{"name", "origin", "used", "avail", "mountpoint", "compression", "type", "volsize", "quota", "referenced", "written", "logicalused", "usedbydataset"} + zpoolPropList = []string{"name", "health", "allocated", "size", "free", "readonly", "dedupratio", "fragmentation", "freeing", "leaked"} + zpoolPropListOptions = strings.Join(zpoolPropList, ",") + zpoolArgs = []string{"get", "-Hp", zpoolPropListOptions} + + zpoolVdevArgs = []string{"list", "-HPpv"} +) + +var changeTypeMap = map[string]ChangeType{ + "-": Removed, + "+": Created, + "M": Modified, + "R": Renamed, +} + +var inodeTypeMap = map[string]InodeType{ + "B": BlockDevice, + "C": CharacterDevice, + "/": Directory, + ">": Door, + "|": NamedPipe, + "@": SymbolicLink, + "P": EventPort, + "=": Socket, + "F": File, +} + +var referenceCountRegex = regexp.MustCompile(`\(([+-]\d+?)\)`) + +func parseInodeChange(line []string) (*InodeChange, error) { + llen := len(line) + if llen < 1 { + return nil, fmt.Errorf("empty line passed") + } + + changeType := changeTypeMap[line[0]] + if changeType == 0 { + return nil, fmt.Errorf("unknown change type '%s'", line[0]) + } + + switch changeType { + case Renamed: + if llen != 4 { + return nil, fmt.Errorf("mismatching number of fields: expect 4, got: %d", llen) + } + case Modified: + if llen != 4 && llen != 3 { + return nil, fmt.Errorf("mismatching number of fields: expect 3..4, got: %d", llen) + } + default: + if llen != 3 { + return nil, fmt.Errorf("mismatching number of fields: expect 3, got: %d", llen) + } + } + + inodeType := inodeTypeMap[line[1]] + if inodeType == 0 { + return nil, fmt.Errorf("unknown inode type '%s'", line[1]) + } + + path, err := utils.UnescapeFilepath(line[2]) + if err != nil { + return nil, fmt.Errorf("failed to parse filename: %w", err) + } + + var newPath string + var referenceCount int + switch changeType { + case Renamed: + newPath, err = utils.UnescapeFilepath(line[3]) + if err != nil { + return nil, fmt.Errorf("failed to parse filename: %w", err) + } + case Modified: + if llen == 4 { + referenceCount, err = parseReferenceCount(line[3]) + if err != nil { + return nil, fmt.Errorf("failed to parse reference count: %w", err) + } + } + default: + newPath = "" + } + + return &InodeChange{ + Change: changeType, + Type: inodeType, + Path: path, + NewPath: newPath, + ReferenceCountChange: referenceCount, + }, nil +} + +func parseInodeChanges(lines [][]string) ([]*InodeChange, error) { + changes := make([]*InodeChange, len(lines)) + + for i, line := range lines { + c, err := parseInodeChange(line) + if err != nil { + return nil, fmt.Errorf("failed to parse line %d of zfs diff: %w, got: '%s'", i, err, line) + } + changes[i] = c + } + return changes, nil +} + +func setString(field *string, value string) { + v := "" + if value != "-" { + v = value + } + *field = v +} + +func setUint(field *uint64, value string) error { + var v uint64 + if value != "-" { + var err error + v, err = strconv.ParseUint(value, 10, 64) + if err != nil { + return err + } + } + *field = v + return nil +} + +func propsSlice(properties map[string]string) []string { + args := make([]string, 0, len(properties)*3) + for k, v := range properties { + args = append(args, "-o") + args = append(args, fmt.Sprintf("%s=%s", k, v)) + } + return args +} + +func parseReferenceCount(field string) (int, error) { + matches := referenceCountRegex.FindStringSubmatch(field) + if matches == nil { + return 0, fmt.Errorf("regexp does not match") + } + return strconv.Atoi(matches[1]) +} + +func ParseTimeUnit(value string) uint64 { + if value == "-" { + return 0 + } + + re := regexp.MustCompile(`([\d.]+)([a-zA-Z]*)`) + matches := re.FindStringSubmatch(value) + + if len(matches) != 3 { + return 0 + } + + num, err := strconv.ParseFloat(matches[1], 64) + if err != nil { + return 0 + } + + unit := matches[2] + switch unit { + case "us": + return uint64(num) + case "ms": + return uint64(num * 1000) + case "s": + return uint64(num * 1000000) + default: + return uint64(num) + } +} diff --git a/pkg/zfs/utils.go b/pkg/zfs/utils.go new file mode 100644 index 00000000..ed401b14 --- /dev/null +++ b/pkg/zfs/utils.go @@ -0,0 +1,144 @@ +package zfs + +import ( + "bytes" + "errors" + "io" + "runtime" + "strings" +) + +func (z *zfs) listByType(t, filter string) ([]*Dataset, error) { + args := []string{"list", "-rp", "-t", t, "-o", "all"} + + if filter != "" { + args = append(args, filter) + } + out, err := z.doOutput(args...) + if err != nil { + return nil, err + } + + if len(out) == 0 { + return nil, nil + } + + var datasets []*Dataset + + name := "" + var ds *Dataset + for _, line := range out[1:] { + if name != line[0] { + name = line[0] + ds = &Dataset{z: z, Name: name, props: make(map[string]string)} + datasets = append(datasets, ds) + } + if err := ds.parseProps([][]string{out[0], line}); err != nil { + return nil, err + } + } + + return datasets, nil +} + +func (d *Dataset) parseProps(out [][]string) error { + var err error + + if len(out) != 2 { + return errors.New("output does not match what is expected on this platform") + } + for i, v := range out[0] { + val := "-" + if i < len(out[1]) { + val = out[1][i] + } + d.props[strings.ToLower(v)] = val + } + + if len(d.props) <= len(dsPropList) { + return errors.New("output does not match what is expected on this platform") + } + setString(&d.Name, d.props["name"]) + setString(&d.Origin, d.props["origin"]) + + if err = setUint(&d.Used, d.props["used"]); err != nil { + return err + } + if err = setUint(&d.Avail, d.props["avail"]); err != nil { + return err + } + + setString(&d.Mountpoint, d.props["mountpoint"]) + setString(&d.Compression, d.props["compress"]) + setString(&d.Type, d.props["type"]) + + if err = setUint(&d.Volsize, d.props["volsize"]); err != nil { + return err + } + if err = setUint(&d.Quota, d.props["quota"]); err != nil { + return err + } + if err = setUint(&d.Referenced, d.props["refer"]); err != nil { + return err + } + + if runtime.GOOS == "solaris" { + return nil + } + + if err = setUint(&d.Written, d.props["written"]); err != nil { + return err + } + if err = setUint(&d.Logicalused, d.props["lused"]); err != nil { + return err + } + if err = setUint(&d.Usedbydataset, d.props["usedds"]); err != nil { + return err + } + return nil +} + +func (z *zfs) run(in io.Reader, out io.Writer, cmd string, args ...string) ([][]string, error) { + var stdout, stderr bytes.Buffer + + if z.sudo { + args = append([]string{cmd}, args...) + cmd = "sudo" + } + + cmdOut := out + if cmdOut == nil { + cmdOut = &stdout + } + + // id := uuid.New().String() + joinedArgs := strings.Join(args, " ") + + // z.logger.Log([]string{"ID:" + id, "START", joinedArgs}) + if err := z.exec.Run(in, cmdOut, &stderr, cmd, args...); err != nil { + return nil, &Error{ + Err: err, + Debug: strings.Join([]string{cmd, joinedArgs}, " "), + Stderr: stderr.String(), + } + } + + // z.logger.Log([]string{"ID:" + id, "FINISH"}) + + // assume if you passed in something for stdout, that you know what to do with it + if out != nil { + return nil, nil + } + + lines := strings.Split(stdout.String(), "\n") + + // last line is always blank + lines = lines[0 : len(lines)-1] + output := make([][]string, len(lines)) + + for i, l := range lines { + output[i] = strings.Fields(l) + } + + return output, nil +} diff --git a/pkg/zfs/zfs.go b/pkg/zfs/zfs.go new file mode 100644 index 00000000..48bb8076 --- /dev/null +++ b/pkg/zfs/zfs.go @@ -0,0 +1,134 @@ +package zfs + +import ( + "io" + "strconv" + "sylve/pkg/exe" +) + +type InodeType int +type ChangeType int +type DestroyFlag int + +const ( + DatasetFilesystem = "filesystem" + DatasetSnapshot = "snapshot" + DatasetVolume = "volume" +) + +const ( + DestroyDefault DestroyFlag = 1 << iota + DestroyRecursive = 1 << iota + DestroyRecursiveClones = 1 << iota + DestroyDeferDeletion = 1 << iota + DestroyForceUmount = 1 << iota +) + +type zfs struct { + exec exe.Executor + sudo bool +} + +type InodeChange struct { + Change ChangeType + Type InodeType + Path string + NewPath string + ReferenceCountChange int +} + +type ZFS interface { + Datasets(filter string) ([]*Dataset, error) + GetDataset(name string) (*Dataset, error) + CreateFilesystem(name string, properties map[string]string) (*Dataset, error) + Filesystems(filter string) ([]*Dataset, error) + CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) + Volumes(filter string) ([]*Dataset, error) + Snapshots(filter string) ([]*Dataset, error) + ReceiveSnapshot(input io.Reader, name string, force ...bool) (*Dataset, error) + + ListZpools() ([]*Zpool, error) + GetZpool(name string) (*Zpool, error) + CreateZpool(name string, properties map[string]string, args ...string) (*Zpool, error) + GetPoolIODelay(poolName string) (float64, error) + GetTotalIODelay() float64 +} + +func (z *zfs) do(arg ...string) error { + _, err := z.doOutput(arg...) + return err +} + +func (z *zfs) doOutput(arg ...string) ([][]string, error) { + return z.run(nil, nil, "zfs", arg...) +} + +func (z *zfs) Datasets(filter string) ([]*Dataset, error) { + return z.listByType("all", filter) +} + +func (z *zfs) Snapshots(filter string) ([]*Dataset, error) { + return z.listByType(DatasetSnapshot, filter) +} + +func (z *zfs) Filesystems(filter string) ([]*Dataset, error) { + return z.listByType(DatasetFilesystem, filter) +} + +func (z *zfs) Volumes(filter string) ([]*Dataset, error) { + return z.listByType(DatasetVolume, filter) +} + +func (z *zfs) GetDataset(name string) (*Dataset, error) { + out, err := z.doOutput("list", "-p", "-o", "all", name) + if err != nil { + return nil, err + } + + ds := &Dataset{z: z, Name: name, props: make(map[string]string)} + return ds, ds.parseProps(out) +} + +func (z *zfs) ReceiveSnapshot(input io.Reader, name string, force ...bool) (*Dataset, error) { + args := []string{"receive"} + if len(force) > 0 && force[0] { + args = append(args, "-F") + } + args = append(args, name) + if _, err := z.run(input, nil, "zfs", args...); err != nil { + return nil, err + } + return z.GetDataset(name) +} + +func (z *zfs) CreateVolume(name string, size uint64, properties map[string]string) (*Dataset, error) { + args := make([]string, 4, 5) + args[0] = "create" + args[1] = "-p" + args[2] = "-V" + args[3] = strconv.FormatUint(size, 10) + if properties != nil { + args = append(args, propsSlice(properties)...) + } + args = append(args, name) + if err := z.do(args...); err != nil { + return nil, err + } + return z.GetDataset(name) +} + +// https://openzfs.github.io/openzfs-docs/man/7/zfsprops.7.html. +func (z *zfs) CreateFilesystem(name string, properties map[string]string) (*Dataset, error) { + args := make([]string, 1, 4) + args[0] = "create" + + if properties != nil { + args = append(args, propsSlice(properties)...) + } + + args = append(args, name) + if err := z.do(args...); err != nil { + return nil, err + } + return z.GetDataset(name) +} diff --git a/pkg/zfs/zpool.go b/pkg/zfs/zpool.go new file mode 100644 index 00000000..49e434f9 --- /dev/null +++ b/pkg/zfs/zpool.go @@ -0,0 +1,276 @@ +package zfs + +import ( + "fmt" + "strconv" + "strings" + "sylve/pkg/utils" +) + +type RW struct { + Read uint64 `json:"read"` + Write uint64 `json:"write"` +} + +type VdevDevice struct { + Name string `json:"name"` + Size uint64 `json:"size"` +} + +type Vdev struct { + Name string `json:"name"` + Alloc uint64 `json:"alloc"` + Free uint64 `json:"free"` + Operations RW `json:"operations"` + Bandwidth RW `json:"bandwidth"` + VdevDevices []VdevDevice `json:"devices"` +} + +type Zpool struct { + z *zfs `json:"-"` + Name string `json:"name"` + Health string `json:"health"` + Allocated uint64 `json:"allocated"` + Size uint64 `json:"size"` + Free uint64 `json:"free"` + Fragmentation uint64 `json:"fragmentation"` + ReadOnly bool `json:"readOnly"` + Freeing uint64 `json:"freeing"` + Leaked uint64 `json:"leaked"` + DedupRatio float64 `json:"dedupRatio"` + Vdevs []Vdev `json:"vdevs"` +} + +func (z *zfs) zpool(arg ...string) error { + _, err := z.zpoolOutput(arg...) + return err +} + +func (z *zfs) zpoolOutput(arg ...string) ([][]string, error) { + return z.run(nil, nil, "zpool", arg...) +} + +func (z *Zpool) parseLine(line []string) error { + prop := line[1] + val := line[2] + + var err error + + switch prop { + case "name": + setString(&z.Name, val) + case "health": + setString(&z.Health, val) + case "allocated": + err = setUint(&z.Allocated, val) + case "size": + err = setUint(&z.Size, val) + case "free": + err = setUint(&z.Free, val) + case "fragmentation": + // Trim trailing "%" before parsing uint + i := strings.Index(val, "%") + if i < 0 { + i = len(val) + } + err = setUint(&z.Fragmentation, val[:i]) + case "readonly": + z.ReadOnly = val == "on" + case "freeing": + err = setUint(&z.Freeing, val) + case "leaked": + err = setUint(&z.Leaked, val) + case "dedupratio": + // Trim trailing "x" before parsing float64 + z.DedupRatio, err = strconv.ParseFloat(val[:len(val)-1], 64) + } + return err +} + +func (z *zfs) GetZpool(name string) (*Zpool, error) { + args := zpoolArgs + args = append(args, name) + out, err := z.zpoolOutput(args...) + if err != nil { + return nil, err + } + + pool := &Zpool{z: z, Name: name} + for _, line := range out { + if err := pool.parseLine(line); err != nil { + return nil, err + } + } + + vdevOut, err := z.zpoolOutput(append(zpoolVdevArgs, name)...) + if err != nil { + return nil, err + } + + var vdevPtrs []*Vdev + var currentVdev *Vdev + + for i, line := range vdevOut { + name := line[0] + + if i == 0 && name == pool.Name { + continue + } + + if strings.HasPrefix(name, "mirror") || strings.HasPrefix(name, "raidz") { + currentVdev = &Vdev{ + Name: name, + Alloc: utils.StringToUint64(line[1]), + Free: utils.StringToUint64(line[3]), + Operations: RW{Read: utils.StringToUint64(line[5]), Write: utils.StringToUint64(line[6])}, + Bandwidth: RW{Read: utils.StringToUint64(line[7]), Write: utils.StringToUint64(line[8])}, + } + vdevPtrs = append(vdevPtrs, currentVdev) + } else if strings.HasPrefix(name, "/dev/") { + device := VdevDevice{ + Name: name, + Size: utils.StringToUint64(line[1]), + } + + if currentVdev != nil { + currentVdev.VdevDevices = append(currentVdev.VdevDevices, device) + } else { + vdev := &Vdev{ + Name: name, + Alloc: utils.StringToUint64(line[1]), + Free: utils.StringToUint64(line[2]), + Operations: RW{Read: utils.StringToUint64(line[5]), Write: utils.StringToUint64(line[6])}, + Bandwidth: RW{Read: utils.StringToUint64(line[7]), Write: utils.StringToUint64(line[8])}, + VdevDevices: []VdevDevice{ + device, + }, + } + vdevPtrs = append(vdevPtrs, vdev) + } + } else { + currentVdev = nil + } + } + + var vdevs []Vdev + for _, v := range vdevPtrs { + vdevs = append(vdevs, *v) + } + pool.Vdevs = vdevs + + return pool, nil +} + +func (z *Zpool) Datasets() ([]*Dataset, error) { + return z.z.Datasets(z.Name) +} + +func (z *Zpool) Snapshots() ([]*Dataset, error) { + return z.z.Snapshots(z.Name) +} + +func (z *zfs) CreateZpool(name string, properties map[string]string, args ...string) (*Zpool, error) { + cli := make([]string, 1, 4) + cli[0] = "create" + if properties != nil { + cli = append(cli, propsSlice(properties)...) + } + cli = append(cli, name) + cli = append(cli, args...) + if err := z.zpool(cli...); err != nil { + return nil, err + } + + return &Zpool{z: z, Name: name}, nil +} + +func (z *Zpool) Destroy() error { + err := z.z.zpool("destroy", z.Name) + return err +} + +func (z *zfs) ListZpools() ([]*Zpool, error) { + args := []string{"list", "-Ho", "name"} + out, err := z.zpoolOutput(args...) + if err != nil { + return nil, err + } + + var pools []*Zpool + + for _, line := range out { + z, err := z.GetZpool(line[0]) + if err != nil { + return nil, err + } + pools = append(pools, z) + } + return pools, nil +} + +func (z *zfs) GetPoolIODelay(poolName string) (float64, error) { + pool, err := z.GetZpool(poolName) + if err != nil { + return 0.0, err + } + + rows, err := z.zpoolOutput("iostat", "-l", "-H", "-v", pool.Name, "1", "2") + if err != nil { + return 0.0, err + } + + var sampleIndices []int + for i, row := range rows { + if len(row) > 0 && row[0] == poolName { + sampleIndices = append(sampleIndices, i) + } + } + + if len(sampleIndices) < 2 { + return 0.0, fmt.Errorf("not enough samples for pool %s", poolName) + } + + secondSampleRow := rows[sampleIndices[1]] + if len(secondSampleRow) < 9 { + return 0.0, fmt.Errorf("not enough fields in iostat output") + } + + readOps := utils.StringToUint64(secondSampleRow[3]) + writeOps := utils.StringToUint64(secondSampleRow[4]) + if (readOps + writeOps) == 0 { + return 0.0, nil + } + + readWait := ParseTimeUnit(secondSampleRow[7]) + writeWait := ParseTimeUnit(secondSampleRow[8]) + + totalWait := (readOps * readWait) + (writeOps * writeWait) + avgWait := totalWait / (readOps + writeOps) + delayPercentage := (float64(avgWait) / 1_000_000.0) * 100 + + return delayPercentage, nil +} + +func (z *zfs) GetTotalIODelay() float64 { + pools, err := z.ListZpools() + if err != nil { + return 0.0 + } + + var totalDelay float64 + count := 0 + + for _, pool := range pools { + delay, _ := GetPoolIODelay(pool.Name) + if delay > 0 { + totalDelay += delay + count++ + } + } + + if count == 0 { + return 0.0 + } + + return totalDelay / float64(count) +} diff --git a/web/package-lock.json b/web/package-lock.json index 72ba2863..be9355e0 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -14,6 +14,7 @@ "@layerstack/svelte-stores": "^1.0.0", "@svelte-put/shortcut": "^4.1.0", "@sveltestack/svelte-query": "^1.6.0", + "@thisux/sveltednd": "^0.0.20", "adze": "^2.2.1", "axios": "^1.8.2", "d3-array": "^3.2.4", @@ -2002,6 +2003,47 @@ "node": ">=4" } }, + "node_modules/@thisux/sveltednd": { + "version": "0.0.20", + "resolved": "https://registry.npmjs.org/@thisux/sveltednd/-/sveltednd-0.0.20.tgz", + "integrity": "sha512-VE0HopIlHIvNOfSZ1SsiIzl1AmTU0VSd2Nz2Q2nIdqIUsT/UP4hiLw2ed2jFqTKTbqC+kbCYL9vuBpkYnEw+Kw==", + "license": "MIT", + "dependencies": { + "@thisux/sveltednd": "^0.0.18" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, + "node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd": { + "version": "0.0.18", + "resolved": "https://registry.npmjs.org/@thisux/sveltednd/-/sveltednd-0.0.18.tgz", + "integrity": "sha512-MO+iR9ZRHApvtwgWujmqwUbhv9mKs0jLOmeG4tn98GmU1/9wVOi02jrNqeggKg3c8dXz6pMmUHfOfmcq7WwYZg==", + "license": "MIT", + "dependencies": { + "@thisux/sveltednd": "^0.0.17" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, + "node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd": { + "version": "0.0.17", + "resolved": "https://registry.npmjs.org/@thisux/sveltednd/-/sveltednd-0.0.17.tgz", + "integrity": "sha512-lRninjw439phhA8xAHqCpMAX0hnwFMdbXW4M0XJgAdnGxeum+QsLiIC4P3HnkNXAygsVKUqxRcbS84CxDZ9hPw==", + "license": "MIT", + "dependencies": { + "@thisux/sveltednd": "^0.0.14" + }, + "peerDependencies": { + "svelte": "^5.0.0" + } + }, + "node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd/node_modules/@thisux/sveltednd": { + "version": "0.0.14", + "resolved": "https://registry.npmjs.org/@thisux/sveltednd/-/sveltednd-0.0.14.tgz", + "integrity": "sha512-Vbq69SU3HUomPg6oCXtb89OG89hka0YIkdaErYibn3waK7tYE66IcQxD/Fzg8YNW3EVsXoA9kc7kW5EUBCSQGg==" + }, "node_modules/@types/cookie": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.6.0.tgz", diff --git a/web/package.json b/web/package.json index 37521c86..0582040b 100644 --- a/web/package.json +++ b/web/package.json @@ -58,6 +58,7 @@ "@layerstack/svelte-stores": "^1.0.0", "@svelte-put/shortcut": "^4.1.0", "@sveltestack/svelte-query": "^1.6.0", + "@thisux/sveltednd": "^0.0.20", "adze": "^2.2.1", "axios": "^1.8.2", "d3-array": "^3.2.4", diff --git a/web/src/lib/api/zfs/pool.ts b/web/src/lib/api/zfs/pool.ts index 51bbfa05..47258016 100644 --- a/web/src/lib/api/zfs/pool.ts +++ b/web/src/lib/api/zfs/pool.ts @@ -37,10 +37,14 @@ export async function createPool( raid: string, options: Record ) { - return await apiRequest('/zfs/pool', APIResponseSchema, 'POST', { + return await apiRequest('/zfs/pools', APIResponseSchema, 'POST', { name, vdevs, raid, options }); } + +export async function deletePool(name: string) { + return await apiRequest(`/zfs/pools/${name}`, APIResponseSchema, 'DELETE'); +} diff --git a/web/src/lib/components/Skeleton/LeftPanel.svelte b/web/src/lib/components/Skeleton/LeftPanel.svelte index bfddd0bf..10ba42bd 100644 --- a/web/src/lib/components/Skeleton/LeftPanel.svelte +++ b/web/src/lib/components/Skeleton/LeftPanel.svelte @@ -1,25 +1,15 @@ -
-
- -
-
-
-
- - - - {#each keys as key} - {#if visibleColumns.value[key]} - - {/if} - {/each} - - + - - {#each table.rows as row, index} - - {#each keys as key} - {#if visibleColumns.value[key]} - - {/if} - {/each} - - {/each} - -
- - open ? handleContextMenuOpen(key) : handleContextMenuClose()} - > - - - - - Toggle Columns - - {#each keys as columnKey} - { - toggleColumnVisibility(columnKey); - }} - > - {columnKey} - - {/each} - - Reset Columns - - -
- {row[key]} -
-
-
-
-
- - - -
- - zfs + close()}> + +
+ + Create ZFS Pool + close()} > - - Close +
-
-
-
- - -
-
- -
- console.log(event.detail)} - /> -
-
+
+
+ + +
+
+ +
- - {#if selected.length > 0} -
- {#each selected as item} - {#if item.label.includes('harddisk')} -
- -
- {:else} -
- -
- {/if} - {/each} -
- -
-
- - - - - - - - {#each raid as fruit} - {fruit.label} - {/each} - - - - -
-
- - - - - - - - {#each compression as fruit} - {fruit.label} - {/each} - - - - -
-
- - - - - - - - {#each ashift as fruit} - {fruit.label} - {/each} - - - - -
-
-
- - -
- - {#if advancedChecked} -
- {#each pairs as pair, index} -
- - - - - {#if pairs.length > 1} - - {/if} -
- {/each} -
-
- -
- {/if} - {/if}
- -
-
- -
+ +
+ +
- - diff --git a/web/src/routes/[node]/storage/zfs/+page.ts b/web/src/routes/[node]/storage/zfs/+page.ts new file mode 100644 index 00000000..792974a7 --- /dev/null +++ b/web/src/routes/[node]/storage/zfs/+page.ts @@ -0,0 +1,17 @@ +import { listDisks } from '$lib/api/disk/disk'; +import { getPools } from '$lib/api/zfs/pool'; +import { simplifyDisks } from '$lib/utils/disk'; +import { cachedFetch } from '$lib/utils/http'; + +export async function load() { + const cacheDuration = 3600 * 1000; + const [disks, pools] = await Promise.all([ + cachedFetch('disks', async () => simplifyDisks(await listDisks()), cacheDuration), + cachedFetch('pools', getPools, cacheDuration) + ]); + + return { + disks, + pools + }; +}