fsutil

package module
v0.0.0-...-b61e79c Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 9, 2026 License: MIT Imports: 25 Imported by: 316

README

PkgGoDev CI Status Go Report Card Codecov

Incremental file directory sync tools in golang.

BENCH_FILE_SIZE=10000 docker buildx bake bench-root
...
#17 0.303 + CGO_ENABLED=0 xx-go test -benchmem '-bench=.' '-run=^$' .
#17 0.303 + tee /tmp/fsutil.log
#17 1.527 BenchmarkWalker/depth_1_target-32                28356             42258 ns/op            9234 B/op        174 allocs/op
#17 3.166 BenchmarkWalker/depth_1_doublestar_target-32             28647             42038 ns/op            9282 B/op        175 allocs/op
#17 4.865 BenchmarkWalker/depth_2_star_target-32                    1184           1009371 ns/op          200654 B/op       3971 allocs/op
#17 6.342 BenchmarkWalker/depth_2_doublestar_target-32              1148           1007115 ns/op          195891 B/op       3908 allocs/op
#17 9.339 BenchmarkWalker/depth_3_star_star_target-32                 39          28146516 ns/op         5221363 B/op     100915 allocs/op
#17 13.54 BenchmarkWalker/depth_3_doublestar_target-32                40          28496829 ns/op         5206464 B/op     100828 allocs/op
#17 17.99 BenchmarkWalker/depth_4_star_star_star_target-32            26          48224854 ns/op         6571213 B/op     119421 allocs/op
#17 25.32 BenchmarkWalker/depth_4_doublestar_target-32                24          45061931 ns/op         6488522 B/op     119315 allocs/op
#17 28.67 BenchmarkWalker/depth_5_star_star_star_star_target-32                       54          22124864 ns/op         2476377 B/op      42818 allocs/op
#17 32.59 BenchmarkWalker/depth_5_doublestar_target-32                                49          21479412 ns/op         2460690 B/op      42699 allocs/op
#17 35.09 BenchmarkWalker/depth_6_star_star_star_star_star_target-32                  28          38307776 ns/op         3998884 B/op      67772 allocs/op
#17 38.05 BenchmarkWalker/depth_6_doublestar_target-32                                31          38242074 ns/op         3980841 B/op      67634 allocs/op
#17 42.92 BenchmarkWalker/depth_6_doublestar_exclude_star_star_doublestar-32                2925            393602 ns/op           47439 B/op       1006 allocs/op
#17 45.99 + cd bench
#17 45.99 + CGO_ENABLED=0 xx-go test -benchmem '-bench=.' '-run=^$' .
#17 45.99 + tee /tmp/bench.log
#17 46.84 BenchmarkCopyWithTar10-32                          283           4291776 ns/op          906824 B/op        843 allocs/op
#17 50.05 BenchmarkCopyWithTar50-32                           50          24077499 ns/op         4999874 B/op       4514 allocs/op
#17 54.00 BenchmarkCopyWithTar200-32                          15          74350687 ns/op        18757006 B/op      15347 allocs/op
#17 56.31 BenchmarkCopyWithTar1000-32                          5         259188166 ns/op        72393427 B/op      55045 allocs/op
#17 61.31 BenchmarkCPA10-32                                  339           3496169 ns/op            7102 B/op         77 allocs/op
#17 64.68 BenchmarkCPA50-32                                   74          15808000 ns/op            7102 B/op         77 allocs/op
#17 67.26 BenchmarkCPA200-32                                  26          45892546 ns/op            7101 B/op         77 allocs/op
#17 70.08 BenchmarkCPA1000-32                                  8         147602854 ns/op            7103 B/op         77 allocs/op
#17 72.60 BenchmarkDiffCopy10-32                             392           3045072 ns/op          219789 B/op       1123 allocs/op
#17 76.15 BenchmarkDiffCopy50-32                              82          14024073 ns/op         1253831 B/op       5460 allocs/op
#17 78.83 BenchmarkDiffCopy200-32                             27          41259003 ns/op         4683591 B/op      18640 allocs/op
#17 81.60 BenchmarkDiffCopy1000-32                             8         125244042 ns/op        17285147 B/op      67649 allocs/op
#17 83.90 BenchmarkDiffCopyProto10-32                        412           2934103 ns/op          232184 B/op       1143 allocs/op
#17 87.46 BenchmarkDiffCopyProto50-32                         80          13809955 ns/op         1273311 B/op       5565 allocs/op
#17 90.02 BenchmarkDiffCopyProto200-32                        30          41169665 ns/op         4697476 B/op      18995 allocs/op
#17 93.05 BenchmarkDiffCopyProto1000-32                        8         127126319 ns/op        17334920 B/op      68883 allocs/op
#17 95.37 BenchmarkIncrementalDiffCopy10-32                 1540            779568 ns/op          119068 B/op       1015 allocs/op
#17 97.84 BenchmarkIncrementalDiffCopy50-32                  782           1513121 ns/op          455740 B/op       4285 allocs/op
#17 99.97 BenchmarkIncrementalDiffCopy200-32                 271           4248983 ns/op         1329875 B/op      13603 allocs/op
#17 102.3 BenchmarkIncrementalDiffCopy1000-32                 84          14027390 ns/op         4552790 B/op      46849 allocs/op
#17 105.4 BenchmarkIncrementalDiffCopy5000-32                 14          73269136 ns/op        24500669 B/op     266772 allocs/op
#17 110.6 BenchmarkIncrementalDiffCopy10000-32                 9         128400623 ns/op        43706410 B/op     472982 allocs/op
#17 114.4 BenchmarkIncrementalCopyWithTar10-32               396           2946217 ns/op          915096 B/op        826 allocs/op
#17 116.3 BenchmarkIncrementalCopyWithTar50-32                69          16767967 ns/op         5093048 B/op       4472 allocs/op
#17 117.7 BenchmarkIncrementalCopyWithTar200-32               19          62520307 ns/op        19081658 B/op      15270 allocs/op
#17 119.7 BenchmarkIncrementalCopyWithTar1000-32               5         239712851 ns/op        73113704 B/op      54938 allocs/op
#17 122.8 BenchmarkIncrementalRsync10-32                      26          43773727 ns/op            6608 B/op         69 allocs/op
#17 124.1 BenchmarkIncrementalRsync50-32                      25          45985011 ns/op            6608 B/op         69 allocs/op
#17 125.6 BenchmarkIncrementalRsync200-32                     22          49976819 ns/op            6608 B/op         69 allocs/op
#17 127.3 BenchmarkIncrementalRsync1000-32                    19          63618139 ns/op            6600 B/op         69 allocs/op
#17 130.6 BenchmarkIncrementalRsync5000-32                     8         132002745 ns/op            6608 B/op         69 allocs/op
#17 136.3 BenchmarkIncrementalRsync10000-32                    6         187247351 ns/op            6608 B/op         69 allocs/op
#17 140.3 BenchmarkRsync10-32                                 25          46054741 ns/op            6606 B/op         69 allocs/op
#17 141.6 BenchmarkRsync50-32                                 19          58922835 ns/op            6605 B/op         69 allocs/op
#17 143.2 BenchmarkRsync200-32                                13          91176938 ns/op            6606 B/op         69 allocs/op
#17 145.5 BenchmarkRsync1000-32                                6         198319527 ns/op            6606 B/op         69 allocs/op
#17 147.6 BenchmarkGnuTar10-32                               268           4489528 ns/op           14192 B/op        151 allocs/op
#17 150.8 BenchmarkGnuTar50-32                                54          20528041 ns/op           14192 B/op        151 allocs/op
#17 152.9 BenchmarkGnuTar200-32                               19          60394926 ns/op           14192 B/op        151 allocs/op
#17 155.4 BenchmarkGnuTar1000-32                               6         198630328 ns/op           14192 B/op        151 allocs/op

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ComparePath

func ComparePath(p1, p2 string) int
func FollowLinks(fs FS, paths []string) ([]string, error)

func Receive

func Receive(ctx context.Context, conn Stream, dest string, opt ReceiveOpt) error

func ReceiveRoot

func ReceiveRoot(ctx context.Context, conn Stream, dest Root, opt ReceiveOpt) error

func Send

func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool)) error

func Stat

func Stat(path string) (*types.Stat, error)

func Walk

func Walk(ctx context.Context, p string, opt *FilterOpt, fn filepath.WalkFunc) error

func WalkDir

func WalkDir(ctx context.Context, p string, opt *FilterOpt, fn gofs.WalkDirFunc) error

func WriteTar

func WriteTar(ctx context.Context, fs FS, w io.Writer) error

Types

type ChangeFunc

type ChangeFunc func(ChangeKind, string, os.FileInfo, error) error

ChangeFunc is the type of function called for each change computed during a directory changes calculation.

type ChangeKind

type ChangeKind int

ChangeKind is the type of modification that a change is making.

const (
	// ChangeKindAdd represents an addition of
	// a file
	ChangeKindAdd ChangeKind = iota

	// ChangeKindModify represents a change to
	// an existing file
	ChangeKindModify

	// ChangeKindDelete represents a delete of
	// a file
	ChangeKindDelete
)

func (ChangeKind) String

func (k ChangeKind) String() string

type ContentHasher

type ContentHasher func(*types.Stat) (hash.Hash, error)

type DiffType

type DiffType int
const (
	DiffMetadata DiffType = iota
	DiffNone
	DiffContent
)

type Dir

type Dir struct {
	Stat *types.Stat
	FS   FS
}

type DirEntryInfo

type DirEntryInfo struct {
	*types.Stat
	// contains filtered or unexported fields
}

func (*DirEntryInfo) Info

func (s *DirEntryInfo) Info() (gofs.FileInfo, error)

func (*DirEntryInfo) IsDir

func (s *DirEntryInfo) IsDir() bool

func (*DirEntryInfo) Name

func (s *DirEntryInfo) Name() string

func (*DirEntryInfo) Type

func (s *DirEntryInfo) Type() gofs.FileMode

type DiskWriter

type DiskWriter struct {
	// contains filtered or unexported fields
}

func NewDiskWriter

func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error)

func (*DiskWriter) HandleChange

func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*DiskWriter) Wait

func (dw *DiskWriter) Wait(ctx context.Context) error

type DiskWriterOpt

type DiskWriterOpt struct {
	AsyncDataCb   WriteToFunc
	SyncDataCb    WriteToFunc
	NotifyCb      func(ChangeKind, string, os.FileInfo, error) error
	ContentHasher ContentHasher
	Filter        FilterFunc
}

type FS

type FS interface {
	Walk(context.Context, string, gofs.WalkDirFunc) error
	Open(string) (io.ReadCloser, error)
}

func NewFS

func NewFS(root string) (FS, error)

NewFS creates a new FS from a root directory on the host filesystem.

func NewFilterFS

func NewFilterFS(fs FS, opt *FilterOpt) (FS, error)

NewFilterFS creates a new FS that filters the given FS using the given FilterOpt.

The returned FS will not contain any paths that do not match the provided include and exclude patterns, or that are are excluded using the mapping function.

The FS is assumed to be a snapshot of the filesystem at the time of the call to NewFilterFS. If the underlying filesystem changes, calls to the underlying FS may be inconsistent.

func NewRootFS

func NewRootFS(root Root) FS

NewRootFS creates a new FS from a filesystem root.

func SubDirFS

func SubDirFS(dirs []Dir) (FS, error)

func WithHardlinkReset

func WithHardlinkReset(fs FS) FS

WithHardlinkReset returns a FS that fixes hardlinks for FS that has been filtered so that original hardlink sources might be missing

type FilterFunc

type FilterFunc func(string, *types.Stat) bool

type FilterOpt

type FilterOpt struct {
	// IncludePatterns requires that the path matches at least one of the
	// specified patterns.
	IncludePatterns []string

	// ExcludePatterns requires that the path does not match any of the
	// specified patterns.
	ExcludePatterns []string

	// FollowPaths contains symlinks that are resolved into IncludePatterns
	// at the time of the call to NewFilterFS.
	FollowPaths []string

	// Map is called for each path that is included in the result.
	// The function can modify the stat info for each element, while the result
	// of the function controls both how Walk continues.
	Map MapFunc
}

type HandleChangeFn

type HandleChangeFn func(ChangeKind, string, os.FileInfo, error) error
type Hardlinks struct {
	// contains filtered or unexported fields
}

func (*Hardlinks) HandleChange

func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) error

type MapFunc

type MapFunc func(string, *types.Stat) MapResult

type MapResult

type MapResult int

The result of the walk function controls both how WalkDir continues and whether the path is kept.

const (
	// Keep the current path and continue.
	MapResultKeep MapResult = iota

	// Exclude the current path and continue.
	MapResultExclude

	// Exclude the current path, and skip the rest of the dir.
	// If path is a dir, skip the current directory.
	// If path is a file, skip the rest of the parent directory.
	// (This matches the semantics of fs.SkipDir.)
	MapResultSkipDir
)

type ReceiveOpt

type ReceiveOpt struct {
	NotifyHashed  ChangeFunc
	ContentHasher ContentHasher
	ProgressCb    func(int, bool)
	Merge         bool
	Filter        FilterFunc
	Differ        DiffType
	MetadataOnly  FilterFunc
}

type Root

type Root interface {
	Close() error
	FS() gofs.FS
	Remove(string) error
	RemoveAll(string) error
	Lstat(string) (os.FileInfo, error)
	Stat(string) (os.FileInfo, error)
	Mkdir(string, os.FileMode) error
	Symlink(string, string) error
	Link(string, string) error
	OpenRoot(string) (*os.Root, error)
	OpenFile(string, int, os.FileMode) (*os.File, error)
	Readlink(string) (string, error)
	Rename(string, string) error
	Lchown(string, int, int) error
	Chmod(string, os.FileMode) error
	Chtimes(string, time.Time, time.Time) error
	RootXattr
	RootMknod
	RootLChtimes
}

func NewRoot

func NewRoot(osroot *os.Root) Root

type RootDiskWriter

type RootDiskWriter struct {
	// contains filtered or unexported fields
}

func NewRootDiskWriter

func NewRootDiskWriter(ctx context.Context, dest Root, opt DiskWriterOpt) (*RootDiskWriter, error)

func (*RootDiskWriter) HandleChange

func (dw *RootDiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

func (*RootDiskWriter) Wait

func (dw *RootDiskWriter) Wait(ctx context.Context) error

type RootLChtimes

type RootLChtimes interface {
	LChtimes(name string, mtime time.Time) error
}

type RootMknod

type RootMknod interface {
	Mknod(name string, mode uint32, dev int) error
}

type RootXattr

type RootXattr interface {
	LSetxattr(name, key string, value []byte, flags int) error
}

type StatInfo

type StatInfo struct {
	*types.Stat
}

func (*StatInfo) IsDir

func (s *StatInfo) IsDir() bool

func (*StatInfo) ModTime

func (s *StatInfo) ModTime() time.Time

func (*StatInfo) Mode

func (s *StatInfo) Mode() os.FileMode

func (*StatInfo) Name

func (s *StatInfo) Name() string

func (*StatInfo) Size

func (s *StatInfo) Size() int64

func (*StatInfo) Sys

func (s *StatInfo) Sys() any

type Stream

type Stream interface {
	RecvMsg(any) error
	SendMsg(m any) error
	Context() context.Context
}

type Validator

type Validator struct {
	// contains filtered or unexported fields
}

func (*Validator) HandleChange

func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error)

type WriteToFunc

type WriteToFunc func(context.Context, string, io.WriteCloser) error

Directories

Path Synopsis
bench module
cmd
receive command
send command
walk command

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL