diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index be823611c..24ed1e850 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -102,7 +102,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. cipherKey = util.GenCipherKey() } - err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { + err = doTraverseBfsAndSaving(commandEnv, writer, path, *verbose, true, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if !entry.Entry.IsDirectory { ext := filepath.Ext(entry.Entry.Name) if encrypted, encErr := util.Encrypt([]byte(entry.Entry.Name), cipherKey); encErr == nil { @@ -147,7 +147,12 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. } -func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error { +// skipSystemLog excludes the metadata-log subtree (SystemLogDir) from the +// traversal. fs.meta.save sets it so the export doesn't carry the whole change +// log; volume.fsck and fs.verify must leave it false so the live log-file +// chunks are accounted for — otherwise fsck reports them as orphans and could +// delete referenced needles. +func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, path string, verbose bool, skipSystemLog bool, genFn func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) error, saveFn func(outputChan chan interface{}) error) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -199,11 +204,14 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, } defer cancel() + systemLogChildPrefix := filer.SystemLogDir + "/" err := filer_pb.TraverseBfs(ctx, filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) error { - parent := string(parentPath) - if parent == filer.SystemLogDir || strings.HasPrefix(parent, filer.SystemLogDir+"/") { - return nil + if skipSystemLog { + parent := string(parentPath) + if parent == filer.SystemLogDir || strings.HasPrefix(parent, systemLogChildPrefix) { + return nil + } } protoMessage := &filer_pb.FullEntry{ diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.go index d53d077de..7e06a111f 100644 --- a/weed/shell/command_fs_verify.go +++ b/weed/shell/command_fs_verify.go @@ -280,7 +280,7 @@ func (c *commandFsVerify) verifyEntry(path string, chunks []*filer_pb.FileChunk, func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount uint64, errCount uint64, err error) { timeNowAtSec := time.Now().Unix() - return fileCount, errCount, doTraverseBfsAndSaving(c.env, c.writer, path, false, + return fileCount, errCount, doTraverseBfsAndSaving(c.env, c.writer, path, false, false, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if c.modifyTimeAgoAtSec > 0 { if entry.Entry.Attributes != nil && c.modifyTimeAgoAtSec < timeNowAtSec-entry.Entry.Attributes.Mtime { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 3b85f8610..3f653221d 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -297,7 +297,7 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m } }() - return doTraverseBfsAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, + return doTraverseBfsAndSaving(c.env, c.writer, c.getCollectFilerFilePath(), false, false, func(ctx context.Context, entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { if *c.verbose && entry.Entry.IsDirectory { fmt.Fprintf(c.writer, "checking directory %s\n", util.NewFullPath(entry.Dir, entry.Entry.Name))