profile
viewpoint
If you are wondering where the data of this site comes from, please visit https://api.github.com/users/ktock/events. GitMemory does not store any data, but only uses NGINX to cache data for a period of time. The idea behind GitMemory is simply to give users a better reading experience.

moby/buildkit 4031

concurrent, cache-efficient, and Dockerfile-agnostic builder toolkit

containerd/stargz-snapshotter 361

Fast container image distribution plugin with lazy pulling

ktock/container-bootfs 17

Container image converter aiming to minimize image size and speed up boot time dramatically with block-level de-dupliction and lazy-pull technology.

ktock/containerd 1

An open and reliable container runtime

ktock/stargz-snapshotter 1

Fast docker image distribution plugin for containerd, based on CRFS/stargz

ktock/buildkit 0

concurrent, cache-efficient, and Dockerfile-agnostic builder toolkit

ktock/common 0

Location for shared common files in github.com/containers repos.

ktock/continuity 0

A transport-agnostic, filesystem metadata manifest system

Pull request review commentmoby/buildkit

Compute diff from the upper directory of overlayfs-based snapshotter

+// +build linux++package cache++import (+	"bytes"+	"context"+	"fmt"+	"io"+	"io/ioutil"+	"os"+	"path/filepath"+	"strings"+	"syscall"++	"github.com/containerd/containerd/archive"+	ctdcompression "github.com/containerd/containerd/archive/compression"+	"github.com/containerd/containerd/content"+	"github.com/containerd/containerd/errdefs"+	"github.com/containerd/containerd/mount"+	"github.com/containerd/continuity/devices"+	"github.com/containerd/continuity/fs"+	"github.com/containerd/continuity/sysx"+	digest "github.com/opencontainers/go-digest"+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"+	"github.com/pkg/errors"+	"golang.org/x/sys/unix"+)++var emptyDesc = ocispec.Descriptor{}++// computeOverlayBlob provides overlayfs-specialized method to compute+// diff between lower and upper snapshot. If the passed mounts cannot+// be computed (e.g. because the mounts aren't overlayfs), it returns+// an error.+func (sr *immutableRef) computeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string) (_ ocispec.Descriptor, err error) {+	upperdir, err := getOverlayUpperdir(lower, upper)+	if err != nil {+		return emptyDesc, err+	}++	var isCompressed bool+	switch mediaType {+	case ocispec.MediaTypeImageLayer:+	case ocispec.MediaTypeImageLayerGzip:+		isCompressed = true+	default:+		return emptyDesc, fmt.Errorf("unsupported diff media type: %v", mediaType)+	}++	cw, err := sr.cm.ContentStore.Writer(ctx,+		content.WithRef(ref),+		content.WithDescriptor(ocispec.Descriptor{+			MediaType: mediaType, // most contentstore implementations just ignore this+		}))+	if err != nil {+		return emptyDesc, errors.Wrap(err, "failed to open writer")+	}+	defer func() {+		if err != nil {+			cw.Close()+		}+	}()++	var labels map[string]string+	if isCompressed {+		dgstr := digest.SHA256.Digester()+		compressed, err := ctdcompression.CompressStream(cw, ctdcompression.Gzip)+		if err != nil {+			return emptyDesc, errors.Wrap(err, "failed to get compressed stream")+		}+		err = writeOverlayUpperdir(ctx, io.MultiWriter(compressed, dgstr.Hash()), upperdir, lower)+		compressed.Close()+		if err != nil {+			return emptyDesc, errors.Wrap(err, "failed to write compressed diff")+		}+		if labels == nil {+			labels = map[string]string{}+		}+		labels[containerdUncompressed] = dgstr.Digest().String()+	} else {+		if err = writeOverlayUpperdir(ctx, cw, upperdir, lower); err != nil {+			return emptyDesc, errors.Wrap(err, "failed to write diff")+		}+	}++	var commitopts []content.Opt+	if labels != nil {+		commitopts = append(commitopts, content.WithLabels(labels))+	}+	dgst := cw.Digest()+	if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {+		if !errdefs.IsAlreadyExists(err) {+			return emptyDesc, errors.Wrap(err, "failed to commit")+		}+	}+	cinfo, err := sr.cm.ContentStore.Info(ctx, dgst)+	if err != nil {+		return emptyDesc, errors.Wrap(err, "failed to get info from content store")+	}+	if cinfo.Labels == nil {+		cinfo.Labels = make(map[string]string)+	}+	// Set uncompressed label if digest already existed without label+	if _, ok := cinfo.Labels[containerdUncompressed]; !ok {+		cinfo.Labels[containerdUncompressed] = labels[containerdUncompressed]+		if _, err := sr.cm.ContentStore.Update(ctx, cinfo, "labels."+containerdUncompressed); err != nil {+			return emptyDesc, errors.Wrap(err, "error setting uncompressed label")+		}+	}++	return ocispec.Descriptor{+		MediaType: mediaType,+		Size:      cinfo.Size,+		Digest:    cinfo.Digest,+	}, nil+}++// getOverlayUpperdir parses the passed mounts and identifies the directory+// that contains diff between upper and lower.+func getOverlayUpperdir(lower, upper []mount.Mount) (string, error) {+	var upperdir string+	if len(lower) == 1 && len(upper) == 1 {+		// We only support single mount configuration as of now.+		upperM, lowerM := upper[0], lower[0]++		// Get layer directories of lower snapshot+		var lowerlayers []string+		switch lowerM.Type {+		case "bind":+			// lower snapshot is a bind mount of one layer+			lowerlayers = []string{lowerM.Source}+		case "overlay":+			// lower snapshot is an overlay mount of multiple layers+			lowerlayers = getOverlayLayers(lowerM)+		default:+			return "", fmt.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type)+		}++		// Get layer directories of upper snapshot+		if upperM.Type != "overlay" {+			return "", fmt.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type)+		}+		upperlayers := getOverlayLayers(upperM)++		// Check if the diff directory can be determined+		if len(upperlayers) != len(lowerlayers)+1 {+			return "", fmt.Errorf("cannot determine diff of more than one upper directories")+		}+		for i := 0; i < len(lowerlayers); i++ {+			if upperlayers[i] != lowerlayers[i] {+				return "", fmt.Errorf("layer %d must be common between upper and lower snapshots", i)+			}+		}+		upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff+	}+	if upperdir == "" {+		return "", fmt.Errorf("cannot determine upperdir from mount option")+	}+	return upperdir, nil+}++// getOverlayLayers returns all layer directories of an overlayfs mount.+func getOverlayLayers(m mount.Mount) []string {+	var u string+	var uFound bool+	var l []string // l[0] = bottommost+	for _, o := range m.Options {+		if strings.HasPrefix(o, "upperdir=") {+			u, uFound = strings.TrimPrefix(o, "upperdir="), true+		} else if strings.HasPrefix(o, "lowerdir=") {+			l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":")+			for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {+				l[i], l[j] = l[j], l[i] // make l[0] = bottommost+			}+		}+	}+	if uFound {+		return append(l, u)+	}+	return l+}++// writeOverlayUpperdir writes a layer tar archive into the specified writer, based on+// the diff information stored in the upperdir.+func writeOverlayUpperdir(ctx context.Context, w io.Writer, upperdir string, lower []mount.Mount) error {+	emptyLower, err := ioutil.TempDir("", "buildkit") // empty directory used for the lower of diff view+	if err != nil {+		return errors.Wrapf(err, "failed to create temp dir")+	}+	defer os.Remove(emptyLower)+	upperView := []mount.Mount{+		{+			Type:    "overlay",+			Source:  "overlay",+			Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))},+		},+	}+	return mount.WithTempMount(ctx, lower, func(lowerRoot string) error {+		return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error {+			cw := archive.NewChangeWriter(w, upperViewRoot)+			if err := overlayChanges(ctx, cw.HandleChange, upperdir, upperViewRoot, lowerRoot); err != nil {+				if err2 := cw.Close(); err2 != nil {+					return errors.Wrapf(err, "failed torecord upperdir changes (close error: %v)", err2)+				}+				return errors.Wrapf(err, "failed torecord upperdir changes")+			}+			return cw.Close()+		})+	})+}++// overlayChanges is continuty's `fs.Change`-like method but leverages overlayfs's+// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of+// the upperdir that doesn't contain whiteouts. This is used for computing+// changes under opaque directories.+func overlayChanges(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error {+	return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error {+		if err != nil {+			return err+		}++		// Rebase path+		path, err = filepath.Rel(upperdir, path)+		if err != nil {+			return err+		}+		path = filepath.Join(string(os.PathSeparator), path)++		// Skip root+		if path == string(os.PathSeparator) {+			return nil+		}++		// Check if this is a deleted entry+		isDelete, skip, err := checkDelete(upperdir, path, base, f)+		if err != nil {+			return err+		} else if skip {+			return nil+		}++		var kind fs.ChangeKind+		var skipRecord bool+		if isDelete {+			// This is a deleted entry.+			kind = fs.ChangeKindDelete+			f = nil+		} else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil {+			// File exists in the base layer. Thus this is modified.+			kind = fs.ChangeKindModify+			// Avoid including directory that hasn't been modified. If /foo/bar/baz is modified,+			// then /foo will apper here even if it's not been modified because it's the parent of bar.+			if same, err := sameDir(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same {+				skipRecord = true // Both are the same, don't record the change+			} else if err != nil {+				return err+			}+		} else if os.IsNotExist(err) {+			// File doesn't exist in the base layer. Thus this is added.+			kind = fs.ChangeKindAdd+		} else if err != nil {+			return err+		}++		if !skipRecord {+			if err := changeFn(kind, path, f, nil); err != nil {+				return err+			}+		}++		if f != nil {+			if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil {+				return err+			} else if isOpaque {+				// This is an opaque directory. Start a new walking differ to get adds/deletes of+				// this directory. We use "upperdirView" directory which doesn't contain whiteouts.+				if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path),+					func(k fs.ChangeKind, p string, f os.FileInfo, err error) error {+						return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir+					},+				); err != nil {+					return err+				}+				return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore.+			}+		}+		return nil+	})+}++// checkDelete checks if the specified file is a whiteout+func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) {+	if f.Mode()&os.ModeCharDevice != 0 {+		if _, ok := f.Sys().(*syscall.Stat_t); ok {+			maj, min, err := devices.DeviceInfo(f)+			if err != nil {+				return false, false, errors.Wrapf(err, "failed to get device info")+			}+			if maj == 0 && min == 0 {+				// This file is a whiteout (char 0/0) that indicates this is deleted from the base+				if _, err := os.Lstat(filepath.Join(base, path)); err != nil {+					if !os.IsNotExist(err) {+						return false, false, errors.Wrapf(err, "failed to lstat")+					}+					// This file doesn't exist even in the base dir.+					// We don't need whiteout. Just skip this file.+					return false, true, nil+				}+				return true, false, nil+			}+		}+	}+	return false, false, nil+}++// checkDelete checks if the specified file is an opaque directory+func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) {+	if f.IsDir() {+		for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} {+			opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey)+			if err != nil && err != unix.ENODATA {+				return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey)+			} else if len(opaque) == 1 && opaque[0] == 'y' {+				// This is an opaque whiteout directory.+				if _, err := os.Lstat(filepath.Join(base, path)); err != nil {+					if !os.IsNotExist(err) {+						return false, errors.Wrapf(err, "failed to lstat")+					}+					// This file doesn't exist even in the base dir. We don't need treat this as an opaque.+					return false, nil+				}+				return true, nil+			}+		}+	}+	return false, nil+}++// sameDir performs continity-compatible comparison of directories.+// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133

I'll open PRs in continuity to make them public functions.

ktock

comment created time in 3 days

PullRequestReviewEvent

Pull request review commentmoby/buildkit

Compute diff from the upper directory of overlayfs-based snapshotter

+// +build linux++package cache++import (+	"bytes"+	"context"+	"fmt"+	"io"+	"io/ioutil"+	"os"+	"path/filepath"+	"strings"+	"syscall"++	"github.com/containerd/containerd/archive"+	ctdcompression "github.com/containerd/containerd/archive/compression"+	"github.com/containerd/containerd/content"+	"github.com/containerd/containerd/errdefs"+	"github.com/containerd/containerd/mount"+	"github.com/containerd/continuity/devices"+	"github.com/containerd/continuity/fs"+	"github.com/containerd/continuity/sysx"+	digest "github.com/opencontainers/go-digest"+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"+	"github.com/pkg/errors"+	"golang.org/x/sys/unix"+)++var emptyDesc = ocispec.Descriptor{}++// computeOverlayBlob provides overlayfs-specialized method to compute+// diff between lower and upper snapshot. If the passed mounts cannot+// be computed (e.g. because the mounts aren't overlayfs), it returns+// an error.+func (sr *immutableRef) computeOverlayBlob(ctx context.Context, lower, upper []mount.Mount, mediaType string, ref string) (_ ocispec.Descriptor, err error) {+	upperdir, err := getOverlayUpperdir(lower, upper)+	if err != nil {+		return emptyDesc, err+	}++	var isCompressed bool+	switch mediaType {+	case ocispec.MediaTypeImageLayer:+	case ocispec.MediaTypeImageLayerGzip:+		isCompressed = true+	default:+		return emptyDesc, fmt.Errorf("unsupported diff media type: %v", mediaType)+	}++	cw, err := sr.cm.ContentStore.Writer(ctx,+		content.WithRef(ref),+		content.WithDescriptor(ocispec.Descriptor{+			MediaType: mediaType, // most contentstore implementations just ignore this+		}))+	if err != nil {+		return emptyDesc, errors.Wrap(err, "failed to open writer")+	}+	defer func() {+		if err != nil {+			cw.Close()+		}+	}()++	var labels map[string]string+	if isCompressed {+		dgstr := digest.SHA256.Digester()+		compressed, err := ctdcompression.CompressStream(cw, ctdcompression.Gzip)+		if err != nil {+			return emptyDesc, errors.Wrap(err, "failed to get compressed stream")+		}+		err = writeOverlayUpperdir(ctx, io.MultiWriter(compressed, dgstr.Hash()), upperdir, lower)+		compressed.Close()+		if err != nil {+			return emptyDesc, errors.Wrap(err, "failed to write compressed diff")+		}+		if labels == nil {+			labels = map[string]string{}+		}+		labels[containerdUncompressed] = dgstr.Digest().String()+	} else {+		if err = writeOverlayUpperdir(ctx, cw, upperdir, lower); err != nil {+			return emptyDesc, errors.Wrap(err, "failed to write diff")+		}+	}++	var commitopts []content.Opt+	if labels != nil {+		commitopts = append(commitopts, content.WithLabels(labels))+	}+	dgst := cw.Digest()+	if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {+		if !errdefs.IsAlreadyExists(err) {+			return emptyDesc, errors.Wrap(err, "failed to commit")+		}+	}+	cinfo, err := sr.cm.ContentStore.Info(ctx, dgst)+	if err != nil {+		return emptyDesc, errors.Wrap(err, "failed to get info from content store")+	}+	if cinfo.Labels == nil {+		cinfo.Labels = make(map[string]string)+	}+	// Set uncompressed label if digest already existed without label+	if _, ok := cinfo.Labels[containerdUncompressed]; !ok {+		cinfo.Labels[containerdUncompressed] = labels[containerdUncompressed]+		if _, err := sr.cm.ContentStore.Update(ctx, cinfo, "labels."+containerdUncompressed); err != nil {+			return emptyDesc, errors.Wrap(err, "error setting uncompressed label")+		}+	}++	return ocispec.Descriptor{+		MediaType: mediaType,+		Size:      cinfo.Size,+		Digest:    cinfo.Digest,+	}, nil+}++// getOverlayUpperdir parses the passed mounts and identifies the directory+// that contains diff between upper and lower.+func getOverlayUpperdir(lower, upper []mount.Mount) (string, error) {+	var upperdir string+	if len(lower) == 1 && len(upper) == 1 {+		// We only support single mount configuration as of now.+		upperM, lowerM := upper[0], lower[0]++		// Get layer directories of lower snapshot+		var lowerlayers []string+		switch lowerM.Type {+		case "bind":+			// lower snapshot is a bind mount of one layer+			lowerlayers = []string{lowerM.Source}+		case "overlay":+			// lower snapshot is an overlay mount of multiple layers+			lowerlayers = getOverlayLayers(lowerM)+		default:+			return "", fmt.Errorf("cannot get layer information from mount option (type = %q)", lowerM.Type)+		}++		// Get layer directories of upper snapshot+		if upperM.Type != "overlay" {+			return "", fmt.Errorf("upper snapshot isn't overlay mounted (type = %q)", upperM.Type)+		}+		upperlayers := getOverlayLayers(upperM)++		// Check if the diff directory can be determined+		if len(upperlayers) != len(lowerlayers)+1 {+			return "", fmt.Errorf("cannot determine diff of more than one upper directories")+		}+		for i := 0; i < len(lowerlayers); i++ {+			if upperlayers[i] != lowerlayers[i] {+				return "", fmt.Errorf("layer %d must be common between upper and lower snapshots", i)+			}+		}+		upperdir = upperlayers[len(upperlayers)-1] // get the topmost layer that indicates diff+	}+	if upperdir == "" {+		return "", fmt.Errorf("cannot determine upperdir from mount option")+	}+	return upperdir, nil+}++// getOverlayLayers returns all layer directories of an overlayfs mount.+func getOverlayLayers(m mount.Mount) []string {+	var u string+	var uFound bool+	var l []string // l[0] = bottommost+	for _, o := range m.Options {+		if strings.HasPrefix(o, "upperdir=") {+			u, uFound = strings.TrimPrefix(o, "upperdir="), true+		} else if strings.HasPrefix(o, "lowerdir=") {+			l = strings.Split(strings.TrimPrefix(o, "lowerdir="), ":")+			for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 {+				l[i], l[j] = l[j], l[i] // make l[0] = bottommost+			}+		}+	}+	if uFound {+		return append(l, u)+	}+	return l+}++// writeOverlayUpperdir writes a layer tar archive into the specified writer, based on+// the diff information stored in the upperdir.+func writeOverlayUpperdir(ctx context.Context, w io.Writer, upperdir string, lower []mount.Mount) error {+	emptyLower, err := ioutil.TempDir("", "buildkit") // empty directory used for the lower of diff view+	if err != nil {+		return errors.Wrapf(err, "failed to create temp dir")+	}+	defer os.Remove(emptyLower)+	upperView := []mount.Mount{+		{+			Type:    "overlay",+			Source:  "overlay",+			Options: []string{fmt.Sprintf("lowerdir=%s", strings.Join([]string{upperdir, emptyLower}, ":"))},+		},+	}+	return mount.WithTempMount(ctx, lower, func(lowerRoot string) error {+		return mount.WithTempMount(ctx, upperView, func(upperViewRoot string) error {+			cw := archive.NewChangeWriter(w, upperViewRoot)+			if err := overlayChanges(ctx, cw.HandleChange, upperdir, upperViewRoot, lowerRoot); err != nil {+				if err2 := cw.Close(); err2 != nil {+					return errors.Wrapf(err, "failed torecord upperdir changes (close error: %v)", err2)+				}+				return errors.Wrapf(err, "failed torecord upperdir changes")+			}+			return cw.Close()+		})+	})+}++// overlayChanges is continuty's `fs.Change`-like method but leverages overlayfs's+// "upperdir" for computing the diff. "upperdirView" is overlayfs mounted view of+// the upperdir that doesn't contain whiteouts. This is used for computing+// changes under opaque directories.+func overlayChanges(ctx context.Context, changeFn fs.ChangeFunc, upperdir, upperdirView, base string) error {+	return filepath.Walk(upperdir, func(path string, f os.FileInfo, err error) error {+		if err != nil {+			return err+		}++		// Rebase path+		path, err = filepath.Rel(upperdir, path)+		if err != nil {+			return err+		}+		path = filepath.Join(string(os.PathSeparator), path)++		// Skip root+		if path == string(os.PathSeparator) {+			return nil+		}++		// Check if this is a deleted entry+		isDelete, skip, err := checkDelete(upperdir, path, base, f)+		if err != nil {+			return err+		} else if skip {+			return nil+		}++		var kind fs.ChangeKind+		var skipRecord bool+		if isDelete {+			// This is a deleted entry.+			kind = fs.ChangeKindDelete+			f = nil+		} else if baseF, err := os.Lstat(filepath.Join(base, path)); err == nil {+			// File exists in the base layer. Thus this is modified.+			kind = fs.ChangeKindModify+			// Avoid including directory that hasn't been modified. If /foo/bar/baz is modified,+			// then /foo will apper here even if it's not been modified because it's the parent of bar.+			if same, err := sameDir(baseF, f, filepath.Join(base, path), filepath.Join(upperdirView, path)); same {+				skipRecord = true // Both are the same, don't record the change+			} else if err != nil {+				return err+			}+		} else if os.IsNotExist(err) {+			// File doesn't exist in the base layer. Thus this is added.+			kind = fs.ChangeKindAdd+		} else if err != nil {+			return err+		}++		if !skipRecord {+			if err := changeFn(kind, path, f, nil); err != nil {+				return err+			}+		}++		if f != nil {+			if isOpaque, err := checkOpaque(upperdir, path, base, f); err != nil {+				return err+			} else if isOpaque {+				// This is an opaque directory. Start a new walking differ to get adds/deletes of+				// this directory. We use "upperdirView" directory which doesn't contain whiteouts.+				if err := fs.Changes(ctx, filepath.Join(base, path), filepath.Join(upperdirView, path),+					func(k fs.ChangeKind, p string, f os.FileInfo, err error) error {+						return changeFn(k, filepath.Join(path, p), f, err) // rebase path to be based on the opaque dir+					},+				); err != nil {+					return err+				}+				return filepath.SkipDir // We completed this directory. Do not walk files under this directory anymore.+			}+		}+		return nil+	})+}++// checkDelete checks if the specified file is a whiteout+func checkDelete(upperdir string, path string, base string, f os.FileInfo) (delete, skip bool, _ error) {+	if f.Mode()&os.ModeCharDevice != 0 {+		if _, ok := f.Sys().(*syscall.Stat_t); ok {+			maj, min, err := devices.DeviceInfo(f)+			if err != nil {+				return false, false, errors.Wrapf(err, "failed to get device info")+			}+			if maj == 0 && min == 0 {+				// This file is a whiteout (char 0/0) that indicates this is deleted from the base+				if _, err := os.Lstat(filepath.Join(base, path)); err != nil {+					if !os.IsNotExist(err) {+						return false, false, errors.Wrapf(err, "failed to lstat")+					}+					// This file doesn't exist even in the base dir.+					// We don't need whiteout. Just skip this file.+					return false, true, nil+				}+				return true, false, nil+			}+		}+	}+	return false, false, nil+}++// checkDelete checks if the specified file is an opaque directory+func checkOpaque(upperdir string, path string, base string, f os.FileInfo) (isOpaque bool, _ error) {+	if f.IsDir() {+		for _, oKey := range []string{"trusted.overlay.opaque", "user.overlay.opaque"} {+			opaque, err := sysx.LGetxattr(filepath.Join(upperdir, path), oKey)+			if err != nil && err != unix.ENODATA {+				return false, errors.Wrapf(err, "failed to retrieve %s attr", oKey)+			} else if len(opaque) == 1 && opaque[0] == 'y' {+				// This is an opaque whiteout directory.+				if _, err := os.Lstat(filepath.Join(base, path)); err != nil {+					if !os.IsNotExist(err) {+						return false, errors.Wrapf(err, "failed to lstat")+					}+					// This file doesn't exist even in the base dir. We don't need treat this as an opaque.+					return false, nil+				}+				return true, nil+			}+		}+	}+	return false, nil+}++// sameDir performs continity-compatible comparison of directories.+// https://github.com/containerd/continuity/blob/v0.1.0/fs/path.go#L91-L133+// This doesn't compare files because it requires to compare their contents.+// This is what we want to avoid by this overlayfs-specialized differ.+func sameDir(f1, f2 os.FileInfo, f1fullPath, f2fullPath string) (bool, error) {+	if !f1.IsDir() || !f2.IsDir() {+		return false, nil+	}++	if os.SameFile(f1, f2) {+		return true, nil+	}++	equalStat, err := compareSysStat(f1.Sys(), f2.Sys())+	if err != nil || !equalStat {+		return equalStat, err+	}++	if eq, err := compareCapabilities(f1fullPath, f2fullPath); err != nil || !eq {

Let's address this in follow-ups.

ktock

comment created time in 3 days

PullRequestReviewEvent

issue commentcontainerd/stargz-snapshotter

Enable GPU support and net-host inside container when optimizing

Added in https://github.com/containerd/stargz-snapshotter/pull/382

rdpsin

comment created time in 4 days

issue closedcontainerd/stargz-snapshotter

Enable GPU support and net-host inside container when optimizing

Currently, networking inside a container when optimizing through the ctr-remote tool is only available through CNI-based networking plugins, not through net-host, which is often quite simpler. Furthermore, there is no option to enable GPUs inside the container when optimizing GPU-based images.

I propose changes to adds flags to the optimize command to enable both features.

The changes can be seen in: https://github.com/rdpsin/stargz-snapshotter/commit/54b9be6efbf45fd7c59f7976d854c72e421785ba

closed time in 4 days

rdpsin

push eventcontainerd/stargz-snapshotter

Rishabh Singhvi

commit sha deb5648d7333577d29a72815a3d90d8da8824906

Enable GPUs and networking in container during optimize Currently, networking inside a container when optimizing through the ctr-remote tool is only available through CNI-based networking plugins, not through `net-host`. Furthermore, there is no option to enable GPUs inside the container when optimizing GPU-based images. This commit adds flags to the optimize command to enable both features. Changes: 1) Added `net-host` flag to enable to networking inside the container, similar to when using `ctr-remote run`. 2) Added `gpus` flag to enable GPU-based image optimization. 3) Changes to documentation explaining the above made changes. Signed-off-by: Rishabh Singhvi <rdpsin@amazon.com>

view details

Kohei Tokunaga

commit sha 5ad3a11b816cb330550dca66d11c5d37271f3e62

Merge pull request #382 from rdpsin/gpu-net-host-opt Enable GPUs and networking in container during optimize

view details

push time in 4 days

PR merged containerd/stargz-snapshotter

Enable GPUs and networking in container during optimize

Currently, networking inside a container when optimizing through the ctr-remote tool is only available through CNI-based networking plugins, not through net-host. Furthermore, there is no option to enable GPUs inside the container when optimizing GPU-based images.

This commit adds flags to the optimize command to enable both features.

Changes:

  1. Added net-host flag to enable to networking inside the container, similar to when using ctr-remote run.
  2. Added gpus flag to enable GPU-based image optimization.
  3. Changes to documentation explaining the above made changes.

Signed-off-by: Rishabh Singhvi rdpsin@amazon.com

+37 -0

0 comment

2 changed files

rdpsin

pr closed time in 4 days

Pull request review commentcontainerd/stargz-snapshotter

Enable GPUs and networking in container during optimize

 The following example optimizes an image with a compression level of 1. # ctr-remote image optimize --oci --estargz-compression-level 1 ghcr.io/stargz-containers/golang:1.15.3-buster-org registry2:5000/golang:1.15.3-esgz ``` +You can enable networking inside the container using the `net-host` flag.

Thank you!

rdpsin

comment created time in 4 days

PullRequestReviewEvent
PullRequestReviewEvent

Pull request review commentcontainerd/stargz-snapshotter

Enable GPUs and networking in container during optimize

 The following example optimizes an image with a compression level of 1. # ctr-remote image optimize --oci --estargz-compression-level 1 ghcr.io/stargz-containers/golang:1.15.3-buster-org registry2:5000/golang:1.15.3-esgz ``` +You can enable networking inside the container using the `net-host` flag.

So I think rather than "You can enable networking inside the container ", "You can enable host networking for the container` seems the right description here.

rdpsin

comment created time in 4 days

PullRequestReviewEvent

pull request commentcontainerd/stargz-snapshotter

[WIP] Add fuse-manager

@ilyee Could you add changes for https://github.com/containerd/stargz-snapshotter/pull/324#pullrequestreview-690717373 ?

ilyee

comment created time in 4 days

PullRequestReviewEvent
PullRequestReviewEvent
PullRequestReviewEvent

create barnchktock/stargz-snapshotter

branch : ctd-1.5.4

created branch time in 5 days

push eventktock/containerd

ktock

commit sha b483177ee2089fd8284258840e2526370c15eab3

Support custom compressor for walking differ Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

view details

push time in 5 days

Pull request review commentcontainerd/containerd

Support custom compressor for walking differ

 type Applier interface { 	Apply(ctx context.Context, desc ocispec.Descriptor, mount []mount.Mount, opts ...ApplyOpt) (ocispec.Descriptor, error) } +// WithCompressor sets the function to be used to compress the diff+// stream.+func WithCompressor(f func(dest io.Writer, mediaType string) (io.WriteCloser, error)) Opt {+	return func(c *Config) error {+		c.Compressor = f

@fuweid I tried the following interface but it seems impossible because the writer of the content store (store.Writer()) requires to determine the mediatype of the contents before starting to write.

Compressor func(dest io.Writer) (_ io.WriteCloser, mediaType string, _ error)

Instead, I would like to make it the caller's responsibility to specify the target mediatype (config.MediaType) of the final content created by the Compressor. So I updated the patch to enforce it by fixing Differ to return an error if no media type of the target content is explicitly specified but a compressor is used. PTAL?

ktock

comment created time in 5 days

PullRequestReviewEvent

push eventktock/buildkit

ktock

commit sha de8a3421f1f9b45da6917aadefd8cc7759832ce7

Add `estargz` compression type Signed-off-by: ktock <ktokunaga.mail@gmail.com>

view details

push time in 5 days

push eventktock/containerd

ktock

commit sha 7792069b1309bfea97bcb8ba031cf69a1ca7d538

Support custom compressor for walking differ Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

view details

push time in 5 days

Pull request review commentmoby/buildkit

Compute diff from the upper directory of overlayfs-based snapshotter

 func computeBlobChain(ctx context.Context, sr *immutableRef, createIfNeeded bool 				if release != nil { 					defer release() 				}-				descr, err = sr.cm.Differ.Compare(ctx, lower, upper,-					diff.WithMediaType(mediaType),-					diff.WithReference(sr.ID()),-				)-				if err != nil {-					return nil, err+				if !isTypeWindows(sr) {+					// Try optimized diff for overlayfs+					descr, err = sr.computeOverlayBlob(ctx, lower, upper, mediaType, sr.ID())+					if err != nil {+						logrus.Errorf("failed to compute blob (%s) from diff of overlay snapshotter: %+v", sr.ID(), err)

Thank you for the review. Fixed to use Debugf instead.

ktock

comment created time in 5 days

PullRequestReviewEvent

PR closed containerd/stargz-snapshotter

Bump k8s.io/client-go from 0.21.2 to 0.21.3 dependencies go

Bumps k8s.io/client-go from 0.21.2 to 0.21.3. <details> <summary>Commits</summary> <ul> <li><a href="https://github.com/kubernetes/client-go/commit/e337077829155da74648d511bfa560a148fd965d"><code>e337077</code></a> Update dependencies to v0.21.3 tag</li> <li><a href="https://github.com/kubernetes/client-go/commit/fefd2ad85b0e0fd04f21f16411d8113df762d539"><code>fefd2ad</code></a> Merge pull request <a href="https://github-redirect.dependabot.com/kubernetes/client-go/issues/103319">#103319</a> from jpbetz/fix-102749-1.21</li> <li><a href="https://github.com/kubernetes/client-go/commit/58234552b992e32a5009d9e636b8313c1da1d725"><code>5823455</code></a> Bump SMD to v4.1.2 to pick up <a href="https://github-redirect.dependabot.com/kubernetes/client-go/issues/102749">#102749</a> fix</li> <li>See full diff in <a href="https://github.com/kubernetes/client-go/compare/v0.21.2...v0.21.3">compare view</a></li> </ul> </details> <br />

Dependabot compatibility score

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting @dependabot rebase.


<details> <summary>Dependabot commands and options</summary> <br />

You can trigger Dependabot actions by commenting on this PR:

  • @dependabot rebase will rebase this PR
  • @dependabot recreate will recreate this PR, overwriting any edits that have been made to it
  • @dependabot merge will merge this PR after your CI passes on it
  • @dependabot squash and merge will squash and merge this PR after your CI passes on it
  • @dependabot cancel merge will cancel a previously requested merge and block automerging
  • @dependabot reopen will reopen this PR if it is closed
  • @dependabot close will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
  • @dependabot ignore this major version will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
  • @dependabot ignore this minor version will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
  • @dependabot ignore this dependency will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)

</details>

+11 -11

1 comment

2 changed files

dependabot[bot]

pr closed time in 5 days

pull request commentcontainerd/stargz-snapshotter

Bump k8s.io/client-go from 0.21.2 to 0.21.3

Closing in favor of #384.

dependabot[bot]

comment created time in 5 days

PR closed containerd/stargz-snapshotter

Bump k8s.io/apimachinery from 0.21.2 to 0.21.3 dependencies go

Bumps k8s.io/apimachinery from 0.21.2 to 0.21.3. <details> <summary>Commits</summary> <ul> <li><a href="https://github.com/kubernetes/apimachinery/commit/f916759cb6b8547418dc7708876ecab5c1961448"><code>f916759</code></a> Merge pull request <a href="https://github-redirect.dependabot.com/kubernetes/apimachinery/issues/103319">#103319</a> from jpbetz/fix-102749-1.21</li> <li><a href="https://github.com/kubernetes/apimachinery/commit/12b6a22767213cd6debcafd59190e53c133024fe"><code>12b6a22</code></a> Bump SMD to v4.1.2 to pick up <a href="https://github-redirect.dependabot.com/kubernetes/apimachinery/issues/102749">#102749</a> fix</li> <li><a href="https://github.com/kubernetes/apimachinery/commit/41e4141c92b63512d3e84bc5564c08c2308b96fa"><code>41e4141</code></a> sync: remove Godeps/</li> <li>See full diff in <a href="https://github.com/kubernetes/apimachinery/compare/v0.21.2...v0.21.3">compare view</a></li> </ul> </details> <br />

Dependabot compatibility score

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting @dependabot rebase.


<details> <summary>Dependabot commands and options</summary> <br />

You can trigger Dependabot actions by commenting on this PR:

  • @dependabot rebase will rebase this PR
  • @dependabot recreate will recreate this PR, overwriting any edits that have been made to it
  • @dependabot merge will merge this PR after your CI passes on it
  • @dependabot squash and merge will squash and merge this PR after your CI passes on it
  • @dependabot cancel merge will cancel a previously requested merge and block automerging
  • @dependabot reopen will reopen this PR if it is closed
  • @dependabot close will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually
  • @dependabot ignore this major version will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself)
  • @dependabot ignore this minor version will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself)
  • @dependabot ignore this dependency will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)

</details>

+5 -3

1 comment

2 changed files

dependabot[bot]

pr closed time in 5 days

pull request commentcontainerd/stargz-snapshotter

Bump k8s.io/apimachinery from 0.21.2 to 0.21.3

Closing in favor of #384.

dependabot[bot]

comment created time in 5 days