Switch to golang/dep.

vendor/github.com/docker/distribution/registry/doc.go

@@ -0,0 +1,2 @@
+// Package registry provides the main entrypoints for running a registry.
+package registry

vendor/github.com/docker/distribution/registry/registry.go

@@ -0,0 +1,356 @@
+package registry
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	"rsc.io/letsencrypt"
+
+	log "github.com/Sirupsen/logrus"
+	logstash "github.com/bshuster-repo/logrus-logstash-hook"
+	"github.com/bugsnag/bugsnag-go"
+	"github.com/docker/distribution/configuration"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/health"
+	"github.com/docker/distribution/registry/handlers"
+	"github.com/docker/distribution/registry/listener"
+	"github.com/docker/distribution/uuid"
+	"github.com/docker/distribution/version"
+	gorhandlers "github.com/gorilla/handlers"
+	"github.com/spf13/cobra"
+	"github.com/yvasiyarov/gorelic"
+)
+
+// ServeCmd is a cobra command for running the registry.
+var ServeCmd = &cobra.Command{
+	Use:   "serve <config>",
+	Short: "`serve` stores and distributes Docker images",
+	Long:  "`serve` stores and distributes Docker images.",
+	Run: func(cmd *cobra.Command, args []string) {
+
+		// setup context
+		ctx := context.WithVersion(context.Background(), version.Version)
+
+		config, err := resolveConfiguration(args)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
+			cmd.Usage()
+			os.Exit(1)
+		}
+
+		if config.HTTP.Debug.Addr != "" {
+			go func(addr string) {
+				log.Infof("debug server listening %v", addr)
+				if err := http.ListenAndServe(addr, nil); err != nil {
+					log.Fatalf("error listening on debug interface: %v", err)
+				}
+			}(config.HTTP.Debug.Addr)
+		}
+
+		registry, err := NewRegistry(ctx, config)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		if err = registry.ListenAndServe(); err != nil {
+			log.Fatalln(err)
+		}
+	},
+}
+
+// A Registry represents a complete instance of the registry.
+// TODO(aaronl): It might make sense for Registry to become an interface.
+type Registry struct {
+	config *configuration.Configuration
+	app    *handlers.App
+	server *http.Server
+}
+
+// NewRegistry creates a new registry from a context and configuration struct.
+func NewRegistry(ctx context.Context, config *configuration.Configuration) (*Registry, error) {
+	var err error
+	ctx, err = configureLogging(ctx, config)
+	if err != nil {
+		return nil, fmt.Errorf("error configuring logger: %v", err)
+	}
+
+	// inject a logger into the uuid library. warns us if there is a problem
+	// with uuid generation under low entropy.
+	uuid.Loggerf = context.GetLogger(ctx).Warnf
+
+	app := handlers.NewApp(ctx, config)
+	// TODO(aaronl): The global scope of the health checks means NewRegistry
+	// can only be called once per process.
+	app.RegisterHealthChecks()
+	handler := configureReporting(app)
+	handler = alive("/", handler)
+	handler = health.Handler(handler)
+	handler = panicHandler(handler)
+	if !config.Log.AccessLog.Disabled {
+		handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler)
+	}
+
+	server := &http.Server{
+		Handler: handler,
+	}
+
+	return &Registry{
+		app:    app,
+		config: config,
+		server: server,
+	}, nil
+}
+
+// ListenAndServe runs the registry's HTTP server.
+func (registry *Registry) ListenAndServe() error {
+	config := registry.config
+
+	ln, err := listener.NewListener(config.HTTP.Net, config.HTTP.Addr)
+	if err != nil {
+		return err
+	}
+
+	if config.HTTP.TLS.Certificate != "" || config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
+		tlsConf := &tls.Config{
+			ClientAuth:               tls.NoClientCert,
+			NextProtos:               nextProtos(config),
+			MinVersion:               tls.VersionTLS10,
+			PreferServerCipherSuites: true,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		if config.HTTP.TLS.LetsEncrypt.CacheFile != "" {
+			if config.HTTP.TLS.Certificate != "" {
+				return fmt.Errorf("cannot specify both certificate and Let's Encrypt")
+			}
+			var m letsencrypt.Manager
+			if err := m.CacheFile(config.HTTP.TLS.LetsEncrypt.CacheFile); err != nil {
+				return err
+			}
+			if !m.Registered() {
+				if err := m.Register(config.HTTP.TLS.LetsEncrypt.Email, nil); err != nil {
+					return err
+				}
+			}
+			tlsConf.GetCertificate = m.GetCertificate
+		} else {
+			tlsConf.Certificates = make([]tls.Certificate, 1)
+			tlsConf.Certificates[0], err = tls.LoadX509KeyPair(config.HTTP.TLS.Certificate, config.HTTP.TLS.Key)
+			if err != nil {
+				return err
+			}
+		}
+
+		if len(config.HTTP.TLS.ClientCAs) != 0 {
+			pool := x509.NewCertPool()
+
+			for _, ca := range config.HTTP.TLS.ClientCAs {
+				caPem, err := ioutil.ReadFile(ca)
+				if err != nil {
+					return err
+				}
+
+				if ok := pool.AppendCertsFromPEM(caPem); !ok {
+					return fmt.Errorf("Could not add CA to pool")
+				}
+			}
+
+			for _, subj := range pool.Subjects() {
+				context.GetLogger(registry.app).Debugf("CA Subject: %s", string(subj))
+			}
+
+			tlsConf.ClientAuth = tls.RequireAndVerifyClientCert
+			tlsConf.ClientCAs = pool
+		}
+
+		ln = tls.NewListener(ln, tlsConf)
+		context.GetLogger(registry.app).Infof("listening on %v, tls", ln.Addr())
+	} else {
+		context.GetLogger(registry.app).Infof("listening on %v", ln.Addr())
+	}
+
+	return registry.server.Serve(ln)
+}
+
+func configureReporting(app *handlers.App) http.Handler {
+	var handler http.Handler = app
+
+	if app.Config.Reporting.Bugsnag.APIKey != "" {
+		bugsnagConfig := bugsnag.Configuration{
+			APIKey: app.Config.Reporting.Bugsnag.APIKey,
+			// TODO(brianbland): provide the registry version here
+			// AppVersion: "2.0",
+		}
+		if app.Config.Reporting.Bugsnag.ReleaseStage != "" {
+			bugsnagConfig.ReleaseStage = app.Config.Reporting.Bugsnag.ReleaseStage
+		}
+		if app.Config.Reporting.Bugsnag.Endpoint != "" {
+			bugsnagConfig.Endpoint = app.Config.Reporting.Bugsnag.Endpoint
+		}
+		bugsnag.Configure(bugsnagConfig)
+
+		handler = bugsnag.Handler(handler)
+	}
+
+	if app.Config.Reporting.NewRelic.LicenseKey != "" {
+		agent := gorelic.NewAgent()
+		agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey
+		if app.Config.Reporting.NewRelic.Name != "" {
+			agent.NewrelicName = app.Config.Reporting.NewRelic.Name
+		}
+		agent.CollectHTTPStat = true
+		agent.Verbose = app.Config.Reporting.NewRelic.Verbose
+		agent.Run()
+
+		handler = agent.WrapHTTPHandler(handler)
+	}
+
+	return handler
+}
+
+// configureLogging prepares the context with a logger using the
+// configuration.
+func configureLogging(ctx context.Context, config *configuration.Configuration) (context.Context, error) {
+	if config.Log.Level == "" && config.Log.Formatter == "" {
+		// If no config for logging is set, fallback to deprecated "Loglevel".
+		log.SetLevel(logLevel(config.Loglevel))
+		ctx = context.WithLogger(ctx, context.GetLogger(ctx))
+		return ctx, nil
+	}
+
+	log.SetLevel(logLevel(config.Log.Level))
+
+	formatter := config.Log.Formatter
+	if formatter == "" {
+		formatter = "text" // default formatter
+	}
+
+	switch formatter {
+	case "json":
+		log.SetFormatter(&log.JSONFormatter{
+			TimestampFormat: time.RFC3339Nano,
+		})
+	case "text":
+		log.SetFormatter(&log.TextFormatter{
+			TimestampFormat: time.RFC3339Nano,
+		})
+	case "logstash":
+		log.SetFormatter(&logstash.LogstashFormatter{
+			TimestampFormat: time.RFC3339Nano,
+		})
+	default:
+		// just let the library use default on empty string.
+		if config.Log.Formatter != "" {
+			return ctx, fmt.Errorf("unsupported logging formatter: %q", config.Log.Formatter)
+		}
+	}
+
+	if config.Log.Formatter != "" {
+		log.Debugf("using %q logging formatter", config.Log.Formatter)
+	}
+
+	if len(config.Log.Fields) > 0 {
+		// build up the static fields, if present.
+		var fields []interface{}
+		for k := range config.Log.Fields {
+			fields = append(fields, k)
+		}
+
+		ctx = context.WithValues(ctx, config.Log.Fields)
+		ctx = context.WithLogger(ctx, context.GetLogger(ctx, fields...))
+	}
+
+	return ctx, nil
+}
+
+func logLevel(level configuration.Loglevel) log.Level {
+	l, err := log.ParseLevel(string(level))
+	if err != nil {
+		l = log.InfoLevel
+		log.Warnf("error parsing level %q: %v, using %q	", level, err, l)
+	}
+
+	return l
+}
+
+// panicHandler add an HTTP handler to web app. The handler recover the happening
+// panic. logrus.Panic transmits panic message to pre-config log hooks, which is
+// defined in config.yml.
+func panicHandler(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Panic(fmt.Sprintf("%v", err))
+			}
+		}()
+		handler.ServeHTTP(w, r)
+	})
+}
+
+// alive simply wraps the handler with a route that always returns an http 200
+// response when the path is matched. If the path is not matched, the request
+// is passed to the provided handler. There is no guarantee of anything but
+// that the server is up. Wrap with other handlers (such as health.Handler)
+// for greater affect.
+func alive(path string, handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == path {
+			w.Header().Set("Cache-Control", "no-cache")
+			w.WriteHeader(http.StatusOK)
+			return
+		}
+
+		handler.ServeHTTP(w, r)
+	})
+}
+
+func resolveConfiguration(args []string) (*configuration.Configuration, error) {
+	var configurationPath string
+
+	if len(args) > 0 {
+		configurationPath = args[0]
+	} else if os.Getenv("REGISTRY_CONFIGURATION_PATH") != "" {
+		configurationPath = os.Getenv("REGISTRY_CONFIGURATION_PATH")
+	}
+
+	if configurationPath == "" {
+		return nil, fmt.Errorf("configuration path unspecified")
+	}
+
+	fp, err := os.Open(configurationPath)
+	if err != nil {
+		return nil, err
+	}
+
+	defer fp.Close()
+
+	config, err := configuration.Parse(fp)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s: %v", configurationPath, err)
+	}
+
+	return config, nil
+}
+
+func nextProtos(config *configuration.Configuration) []string {
+	switch config.HTTP.HTTP2.Disabled {
+	case true:
+		return []string{"http/1.1"}
+	default:
+		return []string{"h2", "http/1.1"}
+	}
+}

vendor/github.com/docker/distribution/registry/root.go

@@ -0,0 +1,84 @@
+package registry
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage"
+	"github.com/docker/distribution/registry/storage/driver/factory"
+	"github.com/docker/distribution/version"
+	"github.com/docker/libtrust"
+	"github.com/spf13/cobra"
+)
+
+var showVersion bool
+
+func init() {
+	RootCmd.AddCommand(ServeCmd)
+	RootCmd.AddCommand(GCCmd)
+	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
+	RootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show the version and exit")
+}
+
+// RootCmd is the main command for the 'registry' binary.
+var RootCmd = &cobra.Command{
+	Use:   "registry",
+	Short: "`registry`",
+	Long:  "`registry`",
+	Run: func(cmd *cobra.Command, args []string) {
+		if showVersion {
+			version.PrintVersion()
+			return
+		}
+		cmd.Usage()
+	},
+}
+
+var dryRun bool
+
+// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
+var GCCmd = &cobra.Command{
+	Use:   "garbage-collect <config>",
+	Short: "`garbage-collect` deletes layers not referenced by any manifests",
+	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
+	Run: func(cmd *cobra.Command, args []string) {
+		config, err := resolveConfiguration(args)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
+			cmd.Usage()
+			os.Exit(1)
+		}
+
+		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
+			os.Exit(1)
+		}
+
+		ctx := context.Background()
+		ctx, err = configureLogging(ctx, config)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
+			os.Exit(1)
+		}
+
+		k, err := libtrust.GenerateECP256PrivateKey()
+		if err != nil {
+			fmt.Fprint(os.Stderr, err)
+			os.Exit(1)
+		}
+
+		registry, err := storage.NewRegistry(ctx, driver, storage.Schema1SigningKey(k))
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
+			os.Exit(1)
+		}
+
+		err = storage.MarkAndSweep(ctx, driver, registry, dryRun)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
+			os.Exit(1)
+		}
+	},
+}

vendor/github.com/docker/distribution/registry/storage/blobcachemetrics.go

@@ -0,0 +1,60 @@
+package storage
+
+import (
+	"expvar"
+	"sync/atomic"
+
+	"github.com/docker/distribution/registry/storage/cache"
+)
+
+type blobStatCollector struct {
+	metrics cache.Metrics
+}
+
+func (bsc *blobStatCollector) Hit() {
+	atomic.AddUint64(&bsc.metrics.Requests, 1)
+	atomic.AddUint64(&bsc.metrics.Hits, 1)
+}
+
+func (bsc *blobStatCollector) Miss() {
+	atomic.AddUint64(&bsc.metrics.Requests, 1)
+	atomic.AddUint64(&bsc.metrics.Misses, 1)
+}
+
+func (bsc *blobStatCollector) Metrics() cache.Metrics {
+	return bsc.metrics
+}
+
+// blobStatterCacheMetrics keeps track of cache metrics for blob descriptor
+// cache requests. Note this is kept globally and made available via expvar.
+// For more detailed metrics, its recommend to instrument a particular cache
+// implementation.
+var blobStatterCacheMetrics cache.MetricsTracker = &blobStatCollector{}
+
+func init() {
+	registry := expvar.Get("registry")
+	if registry == nil {
+		registry = expvar.NewMap("registry")
+	}
+
+	cache := registry.(*expvar.Map).Get("cache")
+	if cache == nil {
+		cache = &expvar.Map{}
+		cache.(*expvar.Map).Init()
+		registry.(*expvar.Map).Set("cache", cache)
+	}
+
+	storage := cache.(*expvar.Map).Get("storage")
+	if storage == nil {
+		storage = &expvar.Map{}
+		storage.(*expvar.Map).Init()
+		cache.(*expvar.Map).Set("storage", storage)
+	}
+
+	storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} {
+		// no need for synchronous access: the increments are atomic and
+		// during reading, we don't care if the data is up to date. The
+		// numbers will always *eventually* be reported correctly.
+		return blobStatterCacheMetrics
+	}))
+}

vendor/github.com/docker/distribution/registry/storage/blobserver.go

@@ -0,0 +1,78 @@
+package storage
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+// TODO(stevvooe): This should configurable in the future.
+const blobCacheControlMaxAge = 365 * 24 * time.Hour
+
+// blobServer simply serves blobs from a driver instance using a path function
+// to identify paths and a descriptor service to fill in metadata.
+type blobServer struct {
+	driver   driver.StorageDriver
+	statter  distribution.BlobStatter
+	pathFn   func(dgst digest.Digest) (string, error)
+	redirect bool // allows disabling URLFor redirects
+}
+
+func (bs *blobServer) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	path, err := bs.pathFn(desc.Digest)
+	if err != nil {
+		return err
+	}
+
+	if bs.redirect {
+		redirectURL, err := bs.driver.URLFor(ctx, path, map[string]interface{}{"method": r.Method})
+		switch err.(type) {
+		case nil:
+			// Redirect to storage URL.
+			http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect)
+			return err
+
+		case driver.ErrUnsupportedMethod:
+			// Fallback to serving the content directly.
+		default:
+			// Some unexpected error.
+			return err
+		}
+	}
+
+	br, err := newFileReader(ctx, bs.driver, path, desc.Size)
+	if err != nil {
+		return err
+	}
+	defer br.Close()
+
+	w.Header().Set("ETag", fmt.Sprintf(`"%s"`, desc.Digest)) // If-None-Match handled by ServeContent
+	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%.f", blobCacheControlMaxAge.Seconds()))
+
+	if w.Header().Get("Docker-Content-Digest") == "" {
+		w.Header().Set("Docker-Content-Digest", desc.Digest.String())
+	}
+
+	if w.Header().Get("Content-Type") == "" {
+		// Set the content type if not already set.
+		w.Header().Set("Content-Type", desc.MediaType)
+	}
+
+	if w.Header().Get("Content-Length") == "" {
+		// Set the content length if not already set.
+		w.Header().Set("Content-Length", fmt.Sprint(desc.Size))
+	}
+
+	http.ServeContent(w, r, desc.Digest.String(), time.Time{}, br)
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/blobstore.go

@@ -0,0 +1,223 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+// blobStore implements the read side of the blob store interface over a
+// driver without enforcing per-repository membership. This object is
+// intentionally a leaky abstraction, providing utility methods that support
+// creating and traversing backend links.
+type blobStore struct {
+	driver  driver.StorageDriver
+	statter distribution.BlobStatter
+}
+
+var _ distribution.BlobProvider = &blobStore{}
+
+// Get implements the BlobReadService.Get call.
+func (bs *blobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	bp, err := bs.path(dgst)
+	if err != nil {
+		return nil, err
+	}
+
+	p, err := bs.driver.GetContent(ctx, bp)
+	if err != nil {
+		switch err.(type) {
+		case driver.PathNotFoundError:
+			return nil, distribution.ErrBlobUnknown
+		}
+
+		return nil, err
+	}
+
+	return p, err
+}
+
+func (bs *blobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := bs.path(desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+
+	return newFileReader(ctx, bs.driver, path, desc.Size)
+}
+
+// Put stores the content p in the blob store, calculating the digest. If the
+// content is already present, only the digest will be returned. This should
+// only be used for small objects, such as manifests. This implemented as a convenience for other Put implementations
+func (bs *blobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	dgst := digest.FromBytes(p)
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err == nil {
+		// content already present
+		return desc, nil
+	} else if err != distribution.ErrBlobUnknown {
+		context.GetLogger(ctx).Errorf("blobStore: error stating content (%v): %v", dgst, err)
+		// real error, return it
+		return distribution.Descriptor{}, err
+	}
+
+	bp, err := bs.path(dgst)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// TODO(stevvooe): Write out mediatype here, as well.
+	return distribution.Descriptor{
+		Size: int64(len(p)),
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}, bs.driver.PutContent(ctx, bp, p)
+}
+
+func (bs *blobStore) Enumerate(ctx context.Context, ingester func(dgst digest.Digest) error) error {
+
+	specPath, err := pathFor(blobsPathSpec{})
+	if err != nil {
+		return err
+	}
+
+	err = Walk(ctx, bs.driver, specPath, func(fileInfo driver.FileInfo) error {
+		// skip directories
+		if fileInfo.IsDir() {
+			return nil
+		}
+
+		currentPath := fileInfo.Path()
+		// we only want to parse paths that end with /data
+		_, fileName := path.Split(currentPath)
+		if fileName != "data" {
+			return nil
+		}
+
+		digest, err := digestFromPath(currentPath)
+		if err != nil {
+			return err
+		}
+
+		return ingester(digest)
+	})
+	return err
+}
+
+// path returns the canonical path for the blob identified by digest. The blob
+// may or may not exist.
+func (bs *blobStore) path(dgst digest.Digest) (string, error) {
+	bp, err := pathFor(blobDataPathSpec{
+		digest: dgst,
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return bp, nil
+}
+
+// link links the path to the provided digest by writing the digest into the
+// target file. Caller must ensure that the blob actually exists.
+func (bs *blobStore) link(ctx context.Context, path string, dgst digest.Digest) error {
+	// The contents of the "link" file are the exact string contents of the
+	// digest, which is specified in that package.
+	return bs.driver.PutContent(ctx, path, []byte(dgst))
+}
+
+// readlink returns the linked digest at path.
+func (bs *blobStore) readlink(ctx context.Context, path string) (digest.Digest, error) {
+	content, err := bs.driver.GetContent(ctx, path)
+	if err != nil {
+		return "", err
+	}
+
+	linked, err := digest.Parse(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return linked, nil
+}
+
+// resolve reads the digest link at path and returns the blob store path.
+func (bs *blobStore) resolve(ctx context.Context, path string) (string, error) {
+	dgst, err := bs.readlink(ctx, path)
+	if err != nil {
+		return "", err
+	}
+
+	return bs.path(dgst)
+}
+
+type blobStatter struct {
+	driver driver.StorageDriver
+}
+
+var _ distribution.BlobDescriptorService = &blobStatter{}
+
+// Stat implements BlobStatter.Stat by returning the descriptor for the blob
+// in the main blob store. If this method returns successfully, there is
+// strong guarantee that the blob exists and is available.
+func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	path, err := pathFor(blobDataPathSpec{
+		digest: dgst,
+	})
+
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	fi, err := bs.driver.Stat(ctx, path)
+	if err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return distribution.Descriptor{}, distribution.ErrBlobUnknown
+		default:
+			return distribution.Descriptor{}, err
+		}
+	}
+
+	if fi.IsDir() {
+		// NOTE(stevvooe): This represents a corruption situation. Somehow, we
+		// calculated a blob path and then detected a directory. We log the
+		// error and then error on the side of not knowing about the blob.
+		context.GetLogger(ctx).Warnf("blob path should not be a directory: %q", path)
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	// TODO(stevvooe): Add method to resolve the mediatype. We can store and
+	// cache a "global" media type for the blob, even if a specific repo has a
+	// mediatype that overrides the main one.
+
+	return distribution.Descriptor{
+		Size: fi.Size(),
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}, nil
+}
+
+func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
+	return distribution.ErrUnsupported
+}
+
+func (bs *blobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	return distribution.ErrUnsupported
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter.go

@@ -0,0 +1,400 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+var (
+	errResumableDigestNotAvailable = errors.New("resumable digest not available")
+)
+
+const (
+	// digestSha256Empty is the canonical sha256 digest of empty data
+	digestSha256Empty = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+)
+
+// blobWriter is used to control the various aspects of resumable
+// blob upload.
+type blobWriter struct {
+	ctx       context.Context
+	blobStore *linkedBlobStore
+
+	id        string
+	startedAt time.Time
+	digester  digest.Digester
+	written   int64 // track the contiguous write
+
+	fileWriter storagedriver.FileWriter
+	driver     storagedriver.StorageDriver
+	path       string
+
+	resumableDigestEnabled bool
+	committed              bool
+}
+
+var _ distribution.BlobWriter = &blobWriter{}
+
+// ID returns the identifier for this upload.
+func (bw *blobWriter) ID() string {
+	return bw.id
+}
+
+func (bw *blobWriter) StartedAt() time.Time {
+	return bw.startedAt
+}
+
+// Commit marks the upload as completed, returning a valid descriptor. The
+// final size and digest are checked against the first descriptor provided.
+func (bw *blobWriter) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) {
+	context.GetLogger(ctx).Debug("(*blobWriter).Commit")
+
+	if err := bw.fileWriter.Commit(); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	bw.Close()
+	desc.Size = bw.Size()
+
+	canonical, err := bw.validateBlob(ctx, desc)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.moveBlob(ctx, canonical); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.blobStore.linkBlob(ctx, canonical, desc.Digest); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.removeResources(ctx); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	err = bw.blobStore.blobAccessController.SetDescriptor(ctx, canonical.Digest, canonical)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	bw.committed = true
+	return canonical, nil
+}
+
+// Cancel the blob upload process, releasing any resources associated with
+// the writer and canceling the operation.
+func (bw *blobWriter) Cancel(ctx context.Context) error {
+	context.GetLogger(ctx).Debug("(*blobWriter).Cancel")
+	if err := bw.fileWriter.Cancel(); err != nil {
+		return err
+	}
+
+	if err := bw.Close(); err != nil {
+		context.GetLogger(ctx).Errorf("error closing blobwriter: %s", err)
+	}
+
+	if err := bw.removeResources(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (bw *blobWriter) Size() int64 {
+	return bw.fileWriter.Size()
+}
+
+func (bw *blobWriter) Write(p []byte) (int, error) {
+	// Ensure that the current write offset matches how many bytes have been
+	// written to the digester. If not, we need to update the digest state to
+	// match the current write position.
+	if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return 0, err
+	}
+
+	n, err := io.MultiWriter(bw.fileWriter, bw.digester.Hash()).Write(p)
+	bw.written += int64(n)
+
+	return n, err
+}
+
+func (bw *blobWriter) ReadFrom(r io.Reader) (n int64, err error) {
+	// Ensure that the current write offset matches how many bytes have been
+	// written to the digester. If not, we need to update the digest state to
+	// match the current write position.
+	if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return 0, err
+	}
+
+	nn, err := io.Copy(io.MultiWriter(bw.fileWriter, bw.digester.Hash()), r)
+	bw.written += nn
+
+	return nn, err
+}
+
+func (bw *blobWriter) Close() error {
+	if bw.committed {
+		return errors.New("blobwriter close after commit")
+	}
+
+	if err := bw.storeHashState(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return err
+	}
+
+	return bw.fileWriter.Close()
+}
+
+// validateBlob checks the data against the digest, returning an error if it
+// does not match. The canonical descriptor is returned.
+func (bw *blobWriter) validateBlob(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) {
+	var (
+		verified, fullHash bool
+		canonical          digest.Digest
+	)
+
+	if desc.Digest == "" {
+		// if no descriptors are provided, we have nothing to validate
+		// against. We don't really want to support this for the registry.
+		return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{
+			Reason: fmt.Errorf("cannot validate against empty digest"),
+		}
+	}
+
+	var size int64
+
+	// Stat the on disk file
+	if fi, err := bw.driver.Stat(ctx, bw.path); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// NOTE(stevvooe): We really don't care if the file is
+			// not actually present for the reader. We now assume
+			// that the desc length is zero.
+			desc.Size = 0
+		default:
+			// Any other error we want propagated up the stack.
+			return distribution.Descriptor{}, err
+		}
+	} else {
+		if fi.IsDir() {
+			return distribution.Descriptor{}, fmt.Errorf("unexpected directory at upload location %q", bw.path)
+		}
+
+		size = fi.Size()
+	}
+
+	if desc.Size > 0 {
+		if desc.Size != size {
+			return distribution.Descriptor{}, distribution.ErrBlobInvalidLength
+		}
+	} else {
+		// if provided 0 or negative length, we can assume caller doesn't know or
+		// care about length.
+		desc.Size = size
+	}
+
+	// TODO(stevvooe): This section is very meandering. Need to be broken down
+	// to be a lot more clear.
+
+	if err := bw.resumeDigest(ctx); err == nil {
+		canonical = bw.digester.Digest()
+
+		if canonical.Algorithm() == desc.Digest.Algorithm() {
+			// Common case: client and server prefer the same canonical digest
+			// algorithm - currently SHA256.
+			verified = desc.Digest == canonical
+		} else {
+			// The client wants to use a different digest algorithm. They'll just
+			// have to be patient and wait for us to download and re-hash the
+			// uploaded content using that digest algorithm.
+			fullHash = true
+		}
+	} else if err == errResumableDigestNotAvailable {
+		// Not using resumable digests, so we need to hash the entire layer.
+		fullHash = true
+	} else {
+		return distribution.Descriptor{}, err
+	}
+
+	if fullHash {
+		// a fantastic optimization: if the the written data and the size are
+		// the same, we don't need to read the data from the backend. This is
+		// because we've written the entire file in the lifecycle of the
+		// current instance.
+		if bw.written == size && digest.Canonical == desc.Digest.Algorithm() {
+			canonical = bw.digester.Digest()
+			verified = desc.Digest == canonical
+		}
+
+		// If the check based on size fails, we fall back to the slowest of
+		// paths. We may be able to make the size-based check a stronger
+		// guarantee, so this may be defensive.
+		if !verified {
+			digester := digest.Canonical.Digester()
+			verifier := desc.Digest.Verifier()
+
+			// Read the file from the backend driver and validate it.
+			fr, err := newFileReader(ctx, bw.driver, bw.path, desc.Size)
+			if err != nil {
+				return distribution.Descriptor{}, err
+			}
+			defer fr.Close()
+
+			tr := io.TeeReader(fr, digester.Hash())
+
+			if _, err := io.Copy(verifier, tr); err != nil {
+				return distribution.Descriptor{}, err
+			}
+
+			canonical = digester.Digest()
+			verified = verifier.Verified()
+		}
+	}
+
+	if !verified {
+		context.GetLoggerWithFields(ctx,
+			map[interface{}]interface{}{
+				"canonical": canonical,
+				"provided":  desc.Digest,
+			}, "canonical", "provided").
+			Errorf("canonical digest does match provided digest")
+		return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{
+			Digest: desc.Digest,
+			Reason: fmt.Errorf("content does not match digest"),
+		}
+	}
+
+	// update desc with canonical hash
+	desc.Digest = canonical
+
+	if desc.MediaType == "" {
+		desc.MediaType = "application/octet-stream"
+	}
+
+	return desc, nil
+}
+
+// moveBlob moves the data into its final, hash-qualified destination,
+// identified by dgst. The layer should be validated before commencing the
+// move.
+func (bw *blobWriter) moveBlob(ctx context.Context, desc distribution.Descriptor) error {
+	blobPath, err := pathFor(blobDataPathSpec{
+		digest: desc.Digest,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// Check for existence
+	if _, err := bw.blobStore.driver.Stat(ctx, blobPath); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			break // ensure that it doesn't exist.
+		default:
+			return err
+		}
+	} else {
+		// If the path exists, we can assume that the content has already
+		// been uploaded, since the blob storage is content-addressable.
+		// While it may be corrupted, detection of such corruption belongs
+		// elsewhere.
+		return nil
+	}
+
+	// If no data was received, we may not actually have a file on disk. Check
+	// the size here and write a zero-length file to blobPath if this is the
+	// case. For the most part, this should only ever happen with zero-length
+	// blobs.
+	if _, err := bw.blobStore.driver.Stat(ctx, bw.path); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// HACK(stevvooe): This is slightly dangerous: if we verify above,
+			// get a hash, then the underlying file is deleted, we risk moving
+			// a zero-length blob into a nonzero-length blob location. To
+			// prevent this horrid thing, we employ the hack of only allowing
+			// to this happen for the digest of an empty blob.
+			if desc.Digest == digestSha256Empty {
+				return bw.blobStore.driver.PutContent(ctx, blobPath, []byte{})
+			}
+
+			// We let this fail during the move below.
+			logrus.
+				WithField("upload.id", bw.ID()).
+				WithField("digest", desc.Digest).Warnf("attempted to move zero-length content with non-zero digest")
+		default:
+			return err // unrelated error
+		}
+	}
+
+	// TODO(stevvooe): We should also write the mediatype when executing this move.
+
+	return bw.blobStore.driver.Move(ctx, bw.path, blobPath)
+}
+
+// removeResources should clean up all resources associated with the upload
+// instance. An error will be returned if the clean up cannot proceed. If the
+// resources are already not present, no error will be returned.
+func (bw *blobWriter) removeResources(ctx context.Context) error {
+	dataPath, err := pathFor(uploadDataPathSpec{
+		name: bw.blobStore.repository.Named().Name(),
+		id:   bw.id,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// Resolve and delete the containing directory, which should include any
+	// upload related files.
+	dirPath := path.Dir(dataPath)
+	if err := bw.blobStore.driver.Delete(ctx, dirPath); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			break // already gone!
+		default:
+			// This should be uncommon enough such that returning an error
+			// should be okay. At this point, the upload should be mostly
+			// complete, but perhaps the backend became unaccessible.
+			context.GetLogger(ctx).Errorf("unable to delete layer upload resources %q: %v", dirPath, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bw *blobWriter) Reader() (io.ReadCloser, error) {
+	// todo(richardscothern): Change to exponential backoff, i=0.5, e=2, n=4
+	try := 1
+	for try <= 5 {
+		_, err := bw.driver.Stat(bw.ctx, bw.path)
+		if err == nil {
+			break
+		}
+		switch err.(type) {
+		case storagedriver.PathNotFoundError:
+			context.GetLogger(bw.ctx).Debugf("Nothing found on try %d, sleeping...", try)
+			time.Sleep(1 * time.Second)
+			try++
+		default:
+			return nil, err
+		}
+	}
+
+	readCloser, err := bw.driver.Reader(bw.ctx, bw.path, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return readCloser, nil
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter_nonresumable.go

@@ -0,0 +1,17 @@
+// +build noresumabledigest
+
+package storage
+
+import (
+	"github.com/docker/distribution/context"
+)
+
+// resumeHashAt is a noop when resumable digest support is disabled.
+func (bw *blobWriter) resumeDigest(ctx context.Context) error {
+	return errResumableDigestNotAvailable
+}
+
+// storeHashState is a noop when resumable digest support is disabled.
+func (bw *blobWriter) storeHashState(ctx context.Context) error {
+	return errResumableDigestNotAvailable
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter_resumable.go

@@ -0,0 +1,145 @@
+// +build !noresumabledigest
+
+package storage
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/stevvooe/resumable"
+
+	// register resumable hashes with import
+	_ "github.com/stevvooe/resumable/sha256"
+	_ "github.com/stevvooe/resumable/sha512"
+)
+
+// resumeDigest attempts to restore the state of the internal hash function
+// by loading the most recent saved hash state equal to the current size of the blob.
+func (bw *blobWriter) resumeDigest(ctx context.Context) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
+	h, ok := bw.digester.Hash().(resumable.Hash)
+	if !ok {
+		return errResumableDigestNotAvailable
+	}
+	offset := bw.fileWriter.Size()
+	if offset == int64(h.Len()) {
+		// State of digester is already at the requested offset.
+		return nil
+	}
+
+	// List hash states from storage backend.
+	var hashStateMatch hashStateEntry
+	hashStates, err := bw.getStoredHashStates(ctx)
+	if err != nil {
+		return fmt.Errorf("unable to get stored hash states with offset %d: %s", offset, err)
+	}
+
+	// Find the highest stored hashState with offset equal to
+	// the requested offset.
+	for _, hashState := range hashStates {
+		if hashState.offset == offset {
+			hashStateMatch = hashState
+			break // Found an exact offset match.
+		}
+	}
+
+	if hashStateMatch.offset == 0 {
+		// No need to load any state, just reset the hasher.
+		h.Reset()
+	} else {
+		storedState, err := bw.driver.GetContent(ctx, hashStateMatch.path)
+		if err != nil {
+			return err
+		}
+
+		if err = h.Restore(storedState); err != nil {
+			return err
+		}
+	}
+
+	// Mind the gap.
+	if gapLen := offset - int64(h.Len()); gapLen > 0 {
+		return errResumableDigestNotAvailable
+	}
+
+	return nil
+}
+
+type hashStateEntry struct {
+	offset int64
+	path   string
+}
+
+// getStoredHashStates returns a slice of hashStateEntries for this upload.
+func (bw *blobWriter) getStoredHashStates(ctx context.Context) ([]hashStateEntry, error) {
+	uploadHashStatePathPrefix, err := pathFor(uploadHashStatePathSpec{
+		name: bw.blobStore.repository.Named().String(),
+		id:   bw.id,
+		alg:  bw.digester.Digest().Algorithm(),
+		list: true,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	paths, err := bw.blobStore.driver.List(ctx, uploadHashStatePathPrefix)
+	if err != nil {
+		if _, ok := err.(storagedriver.PathNotFoundError); !ok {
+			return nil, err
+		}
+		// Treat PathNotFoundError as no entries.
+		paths = nil
+	}
+
+	hashStateEntries := make([]hashStateEntry, 0, len(paths))
+
+	for _, p := range paths {
+		pathSuffix := path.Base(p)
+		// The suffix should be the offset.
+		offset, err := strconv.ParseInt(pathSuffix, 0, 64)
+		if err != nil {
+			logrus.Errorf("unable to parse offset from upload state path %q: %s", p, err)
+		}
+
+		hashStateEntries = append(hashStateEntries, hashStateEntry{offset: offset, path: p})
+	}
+
+	return hashStateEntries, nil
+}
+
+func (bw *blobWriter) storeHashState(ctx context.Context) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
+	h, ok := bw.digester.Hash().(resumable.Hash)
+	if !ok {
+		return errResumableDigestNotAvailable
+	}
+
+	uploadHashStatePath, err := pathFor(uploadHashStatePathSpec{
+		name:   bw.blobStore.repository.Named().String(),
+		id:     bw.id,
+		alg:    bw.digester.Digest().Algorithm(),
+		offset: int64(h.Len()),
+	})
+
+	if err != nil {
+		return err
+	}
+
+	hashState, err := h.State()
+	if err != nil {
+		return err
+	}
+
+	return bw.driver.PutContent(ctx, uploadHashStatePath, hashState)
+}

vendor/github.com/docker/distribution/registry/storage/catalog.go

@@ -0,0 +1,153 @@
+package storage
+
+import (
+	"errors"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// errFinishedWalk signals an early exit to the walk when the current query
+// is satisfied.
+var errFinishedWalk = errors.New("finished walk")
+
+// Returns a list, or partial list, of repositories in the registry.
+// Because it's a quite expensive operation, it should only be used when building up
+// an initial set of repositories.
+func (reg *registry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) {
+	var foundRepos []string
+
+	if len(repos) == 0 {
+		return 0, errors.New("no space in slice")
+	}
+
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return 0, err
+	}
+
+	err = Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error {
+		err := handleRepository(fileInfo, root, last, func(repoPath string) error {
+			foundRepos = append(foundRepos, repoPath)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		// if we've filled our array, no need to walk any further
+		if len(foundRepos) == len(repos) {
+			return errFinishedWalk
+		}
+
+		return nil
+	})
+
+	n = copy(repos, foundRepos)
+
+	switch err {
+	case nil:
+		// nil means that we completed walk and didn't fill buffer. No more
+		// records are available.
+		err = io.EOF
+	case errFinishedWalk:
+		// more records are available.
+		err = nil
+	}
+
+	return n, err
+}
+
+// Enumerate applies ingester to each repository
+func (reg *registry) Enumerate(ctx context.Context, ingester func(string) error) error {
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return err
+	}
+
+	err = Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error {
+		return handleRepository(fileInfo, root, "", ingester)
+	})
+
+	return err
+}
+
+// lessPath returns true if one path a is less than path b.
+//
+// A component-wise comparison is done, rather than the lexical comparison of
+// strings.
+func lessPath(a, b string) bool {
+	// we provide this behavior by making separator always sort first.
+	return compareReplaceInline(a, b, '/', '\x00') < 0
+}
+
+// compareReplaceInline modifies runtime.cmpstring to replace old with new
+// during a byte-wise comparison.
+func compareReplaceInline(s1, s2 string, old, new byte) int {
+	// TODO(stevvooe): We are missing an optimization when the s1 and s2 have
+	// the exact same slice header. It will make the code unsafe but can
+	// provide some extra performance.
+
+	l := len(s1)
+	if len(s2) < l {
+		l = len(s2)
+	}
+
+	for i := 0; i < l; i++ {
+		c1, c2 := s1[i], s2[i]
+		if c1 == old {
+			c1 = new
+		}
+
+		if c2 == old {
+			c2 = new
+		}
+
+		if c1 < c2 {
+			return -1
+		}
+
+		if c1 > c2 {
+			return +1
+		}
+	}
+
+	if len(s1) < len(s2) {
+		return -1
+	}
+
+	if len(s1) > len(s2) {
+		return +1
+	}
+
+	return 0
+}
+
+// handleRepository calls function fn with a repository path if fileInfo
+// has a path of a repository under root and that it is lexographically
+// after last. Otherwise, it will return ErrSkipDir. This should be used
+// with Walk to do handling with repositories in a storage.
+func handleRepository(fileInfo driver.FileInfo, root, last string, fn func(repoPath string) error) error {
+	filePath := fileInfo.Path()
+
+	// lop the base path off
+	repo := filePath[len(root)+1:]
+
+	_, file := path.Split(repo)
+	if file == "_layers" {
+		repo = strings.TrimSuffix(repo, "/_layers")
+		if lessPath(last, repo) {
+			if err := fn(repo); err != nil {
+				return err
+			}
+		}
+		return ErrSkipDir
+	} else if strings.HasPrefix(file, "_") {
+		return ErrSkipDir
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/doc.go

@@ -0,0 +1,3 @@
+// Package storage contains storage services for use in the registry
+// application. It should be considered an internal package, as of Go 1.4.
+package storage

vendor/github.com/docker/distribution/registry/storage/filereader.go

@@ -0,0 +1,177 @@
+package storage
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// TODO(stevvooe): Set an optimal buffer size here. We'll have to
+// understand the latency characteristics of the underlying network to
+// set this correctly, so we may want to leave it to the driver. For
+// out of process drivers, we'll have to optimize this buffer size for
+// local communication.
+const fileReaderBufferSize = 4 << 20
+
+// remoteFileReader provides a read seeker interface to files stored in
+// storagedriver. Used to implement part of layer interface and will be used
+// to implement read side of LayerUpload.
+type fileReader struct {
+	driver storagedriver.StorageDriver
+
+	ctx context.Context
+
+	// identifying fields
+	path string
+	size int64 // size is the total size, must be set.
+
+	// mutable fields
+	rc     io.ReadCloser // remote read closer
+	brd    *bufio.Reader // internal buffered io
+	offset int64         // offset is the current read offset
+	err    error         // terminal error, if set, reader is closed
+}
+
+// newFileReader initializes a file reader for the remote file. The reader
+// takes on the size and path that must be determined externally with a stat
+// call. The reader operates optimistically, assuming that the file is already
+// there.
+func newFileReader(ctx context.Context, driver storagedriver.StorageDriver, path string, size int64) (*fileReader, error) {
+	return &fileReader{
+		ctx:    ctx,
+		driver: driver,
+		path:   path,
+		size:   size,
+	}, nil
+}
+
+func (fr *fileReader) Read(p []byte) (n int, err error) {
+	if fr.err != nil {
+		return 0, fr.err
+	}
+
+	rd, err := fr.reader()
+	if err != nil {
+		return 0, err
+	}
+
+	n, err = rd.Read(p)
+	fr.offset += int64(n)
+
+	// Simulate io.EOR error if we reach filesize.
+	if err == nil && fr.offset >= fr.size {
+		err = io.EOF
+	}
+
+	return n, err
+}
+
+func (fr *fileReader) Seek(offset int64, whence int) (int64, error) {
+	if fr.err != nil {
+		return 0, fr.err
+	}
+
+	var err error
+	newOffset := fr.offset
+
+	switch whence {
+	case os.SEEK_CUR:
+		newOffset += int64(offset)
+	case os.SEEK_END:
+		newOffset = fr.size + int64(offset)
+	case os.SEEK_SET:
+		newOffset = int64(offset)
+	}
+
+	if newOffset < 0 {
+		err = fmt.Errorf("cannot seek to negative position")
+	} else {
+		if fr.offset != newOffset {
+			fr.reset()
+		}
+
+		// No problems, set the offset.
+		fr.offset = newOffset
+	}
+
+	return fr.offset, err
+}
+
+func (fr *fileReader) Close() error {
+	return fr.closeWithErr(fmt.Errorf("fileReader: closed"))
+}
+
+// reader prepares the current reader at the lrs offset, ensuring its buffered
+// and ready to go.
+func (fr *fileReader) reader() (io.Reader, error) {
+	if fr.err != nil {
+		return nil, fr.err
+	}
+
+	if fr.rc != nil {
+		return fr.brd, nil
+	}
+
+	// If we don't have a reader, open one up.
+	rc, err := fr.driver.Reader(fr.ctx, fr.path, fr.offset)
+	if err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// NOTE(stevvooe): If the path is not found, we simply return a
+			// reader that returns io.EOF. However, we do not set fr.rc,
+			// allowing future attempts at getting a reader to possibly
+			// succeed if the file turns up later.
+			return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		default:
+			return nil, err
+		}
+	}
+
+	fr.rc = rc
+
+	if fr.brd == nil {
+		fr.brd = bufio.NewReaderSize(fr.rc, fileReaderBufferSize)
+	} else {
+		fr.brd.Reset(fr.rc)
+	}
+
+	return fr.brd, nil
+}
+
+// resetReader resets the reader, forcing the read method to open up a new
+// connection and rebuild the buffered reader. This should be called when the
+// offset and the reader will become out of sync, such as during a seek
+// operation.
+func (fr *fileReader) reset() {
+	if fr.err != nil {
+		return
+	}
+	if fr.rc != nil {
+		fr.rc.Close()
+		fr.rc = nil
+	}
+}
+
+func (fr *fileReader) closeWithErr(err error) error {
+	if fr.err != nil {
+		return fr.err
+	}
+
+	fr.err = err
+
+	// close and release reader chain
+	if fr.rc != nil {
+		fr.rc.Close()
+	}
+
+	fr.rc = nil
+	fr.brd = nil
+
+	return fr.err
+}

vendor/github.com/docker/distribution/registry/storage/garbagecollect.go

@@ -0,0 +1,114 @@
+package storage
+
+import (
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+func emit(format string, a ...interface{}) {
+	fmt.Printf(format+"\n", a...)
+}
+
+// MarkAndSweep performs a mark and sweep of registry data
+func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dryRun bool) error {
+	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
+	if !ok {
+		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
+	}
+
+	// mark
+	markSet := make(map[digest.Digest]struct{})
+	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
+		emit(repoName)
+
+		var err error
+		named, err := reference.WithName(repoName)
+		if err != nil {
+			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
+		}
+		repository, err := registry.Repository(ctx, named)
+		if err != nil {
+			return fmt.Errorf("failed to construct repository: %v", err)
+		}
+
+		manifestService, err := repository.Manifests(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to construct manifest service: %v", err)
+		}
+
+		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
+		if !ok {
+			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
+		}
+
+		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
+			// Mark the manifest's blob
+			emit("%s: marking manifest %s ", repoName, dgst)
+			markSet[dgst] = struct{}{}
+
+			manifest, err := manifestService.Get(ctx, dgst)
+			if err != nil {
+				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
+			}
+
+			descriptors := manifest.References()
+			for _, descriptor := range descriptors {
+				markSet[descriptor.Digest] = struct{}{}
+				emit("%s: marking blob %s", repoName, descriptor.Digest)
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			// In certain situations such as unfinished uploads, deleting all
+			// tags in S3 or removing the _manifests folder manually, this
+			// error may be of type PathNotFound.
+			//
+			// In these cases we can continue marking other manifests safely.
+			if _, ok := err.(driver.PathNotFoundError); ok {
+				return nil
+			}
+		}
+
+		return err
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to mark: %v", err)
+	}
+
+	// sweep
+	blobService := registry.Blobs()
+	deleteSet := make(map[digest.Digest]struct{})
+	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
+		// check if digest is in markSet. If not, delete it!
+		if _, ok := markSet[dgst]; !ok {
+			deleteSet[dgst] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error enumerating blobs: %v", err)
+	}
+	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
+	// Construct vacuum
+	vacuum := NewVacuum(ctx, storageDriver)
+	for dgst := range deleteSet {
+		emit("blob eligible for deletion: %s", dgst)
+		if dryRun {
+			continue
+		}
+		err = vacuum.RemoveBlob(string(dgst))
+		if err != nil {
+			return fmt.Errorf("failed to delete blob %s: %v", dgst, err)
+		}
+	}
+
+	return err
+}

vendor/github.com/docker/distribution/registry/storage/linkedblobstore.go

@@ -0,0 +1,470 @@
+package storage
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"time"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/uuid"
+	"github.com/opencontainers/go-digest"
+)
+
+// linkPathFunc describes a function that can resolve a link based on the
+// repository name and digest.
+type linkPathFunc func(name string, dgst digest.Digest) (string, error)
+
+// linkedBlobStore provides a full BlobService that namespaces the blobs to a
+// given repository. Effectively, it manages the links in a given repository
+// that grant access to the global blob store.
+type linkedBlobStore struct {
+	*blobStore
+	registry               *registry
+	blobServer             distribution.BlobServer
+	blobAccessController   distribution.BlobDescriptorService
+	repository             distribution.Repository
+	ctx                    context.Context // only to be used where context can't come through method args
+	deleteEnabled          bool
+	resumableDigestEnabled bool
+
+	// linkPathFns specifies one or more path functions allowing one to
+	// control the repository blob link set to which the blob store
+	// dispatches. This is required because manifest and layer blobs have not
+	// yet been fully merged. At some point, this functionality should be
+	// removed the blob links folder should be merged. The first entry is
+	// treated as the "canonical" link location and will be used for writes.
+	linkPathFns []linkPathFunc
+
+	// linkDirectoryPathSpec locates the root directories in which one might find links
+	linkDirectoryPathSpec pathSpec
+}
+
+var _ distribution.BlobStore = &linkedBlobStore{}
+
+func (lbs *linkedBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	return lbs.blobAccessController.Stat(ctx, dgst)
+}
+
+func (lbs *linkedBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.blobStore.Get(ctx, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.blobStore.Open(ctx, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return err
+	}
+
+	if canonical.MediaType != "" {
+		// Set the repository local content type.
+		w.Header().Set("Content-Type", canonical.MediaType)
+	}
+
+	return lbs.blobServer.ServeBlob(ctx, w, r, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	dgst := digest.FromBytes(p)
+	// Place the data in the blob store first.
+	desc, err := lbs.blobStore.Put(ctx, mediaType, p)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting into main store: %v", err)
+		return distribution.Descriptor{}, err
+	}
+
+	if err := lbs.blobAccessController.SetDescriptor(ctx, dgst, desc); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// TODO(stevvooe): Write out mediatype if incoming differs from what is
+	// returned by Put above. Note that we should allow updates for a given
+	// repository.
+
+	return desc, lbs.linkBlob(ctx, desc)
+}
+
+type optionFunc func(interface{}) error
+
+func (f optionFunc) Apply(v interface{}) error {
+	return f(v)
+}
+
+// WithMountFrom returns a BlobCreateOption which designates that the blob should be
+// mounted from the given canonical reference.
+func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption {
+	return optionFunc(func(v interface{}) error {
+		opts, ok := v.(*distribution.CreateOptions)
+		if !ok {
+			return fmt.Errorf("unexpected options type: %T", v)
+		}
+
+		opts.Mount.ShouldMount = true
+		opts.Mount.From = ref
+
+		return nil
+	})
+}
+
+// Writer begins a blob write session, returning a handle.
+func (lbs *linkedBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
+	context.GetLogger(ctx).Debug("(*linkedBlobStore).Writer")
+
+	var opts distribution.CreateOptions
+
+	for _, option := range options {
+		err := option.Apply(&opts)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if opts.Mount.ShouldMount {
+		desc, err := lbs.mount(ctx, opts.Mount.From, opts.Mount.From.Digest(), opts.Mount.Stat)
+		if err == nil {
+			// Mount successful, no need to initiate an upload session
+			return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc}
+		}
+	}
+
+	uuid := uuid.Generate().String()
+	startedAt := time.Now().UTC()
+
+	path, err := pathFor(uploadDataPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   uuid,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	startedAtPath, err := pathFor(uploadStartedAtPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   uuid,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Write a startedat file for this upload
+	if err := lbs.blobStore.driver.PutContent(ctx, startedAtPath, []byte(startedAt.Format(time.RFC3339))); err != nil {
+		return nil, err
+	}
+
+	return lbs.newBlobUpload(ctx, uuid, path, startedAt, false)
+}
+
+func (lbs *linkedBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
+	context.GetLogger(ctx).Debug("(*linkedBlobStore).Resume")
+
+	startedAtPath, err := pathFor(uploadStartedAtPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   id,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	startedAtBytes, err := lbs.blobStore.driver.GetContent(ctx, startedAtPath)
+	if err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return nil, distribution.ErrBlobUploadUnknown
+		default:
+			return nil, err
+		}
+	}
+
+	startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes))
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := pathFor(uploadDataPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   id,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.newBlobUpload(ctx, id, path, startedAt, true)
+}
+
+func (lbs *linkedBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	if !lbs.deleteEnabled {
+		return distribution.ErrUnsupported
+	}
+
+	// Ensure the blob is available for deletion
+	_, err := lbs.blobAccessController.Stat(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	err = lbs.blobAccessController.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lbs *linkedBlobStore) Enumerate(ctx context.Context, ingestor func(digest.Digest) error) error {
+	rootPath, err := pathFor(lbs.linkDirectoryPathSpec)
+	if err != nil {
+		return err
+	}
+	err = Walk(ctx, lbs.blobStore.driver, rootPath, func(fileInfo driver.FileInfo) error {
+		// exit early if directory...
+		if fileInfo.IsDir() {
+			return nil
+		}
+		filePath := fileInfo.Path()
+
+		// check if it's a link
+		_, fileName := path.Split(filePath)
+		if fileName != "link" {
+			return nil
+		}
+
+		// read the digest found in link
+		digest, err := lbs.blobStore.readlink(ctx, filePath)
+		if err != nil {
+			return err
+		}
+
+		// ensure this conforms to the linkPathFns
+		_, err = lbs.Stat(ctx, digest)
+		if err != nil {
+			// we expect this error to occur so we move on
+			if err == distribution.ErrBlobUnknown {
+				return nil
+			}
+			return err
+		}
+
+		err = ingestor(digest)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lbs *linkedBlobStore) mount(ctx context.Context, sourceRepo reference.Named, dgst digest.Digest, sourceStat *distribution.Descriptor) (distribution.Descriptor, error) {
+	var stat distribution.Descriptor
+	if sourceStat == nil {
+		// look up the blob info from the sourceRepo if not already provided
+		repo, err := lbs.registry.Repository(ctx, sourceRepo)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		stat, err = repo.Blobs(ctx).Stat(ctx, dgst)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+	} else {
+		// use the provided blob info
+		stat = *sourceStat
+	}
+
+	desc := distribution.Descriptor{
+		Size: stat.Size,
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}
+	return desc, lbs.linkBlob(ctx, desc)
+}
+
+// newBlobUpload allocates a new upload controller with the given state.
+func (lbs *linkedBlobStore) newBlobUpload(ctx context.Context, uuid, path string, startedAt time.Time, append bool) (distribution.BlobWriter, error) {
+	fw, err := lbs.driver.Writer(ctx, path, append)
+	if err != nil {
+		return nil, err
+	}
+
+	bw := &blobWriter{
+		ctx:        ctx,
+		blobStore:  lbs,
+		id:         uuid,
+		startedAt:  startedAt,
+		digester:   digest.Canonical.Digester(),
+		fileWriter: fw,
+		driver:     lbs.driver,
+		path:       path,
+		resumableDigestEnabled: lbs.resumableDigestEnabled,
+	}
+
+	return bw, nil
+}
+
+// linkBlob links a valid, written blob into the registry under the named
+// repository for the upload controller.
+func (lbs *linkedBlobStore) linkBlob(ctx context.Context, canonical distribution.Descriptor, aliases ...digest.Digest) error {
+	dgsts := append([]digest.Digest{canonical.Digest}, aliases...)
+
+	// TODO(stevvooe): Need to write out mediatype for only canonical hash
+	// since we don't care about the aliases. They are generally unused except
+	// for tarsum but those versions don't care about mediatype.
+
+	// Don't make duplicate links.
+	seenDigests := make(map[digest.Digest]struct{}, len(dgsts))
+
+	// only use the first link
+	linkPathFn := lbs.linkPathFns[0]
+
+	for _, dgst := range dgsts {
+		if _, seen := seenDigests[dgst]; seen {
+			continue
+		}
+		seenDigests[dgst] = struct{}{}
+
+		blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+		if err != nil {
+			return err
+		}
+
+		if err := lbs.blobStore.link(ctx, blobLinkPath, canonical.Digest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type linkedBlobStatter struct {
+	*blobStore
+	repository distribution.Repository
+
+	// linkPathFns specifies one or more path functions allowing one to
+	// control the repository blob link set to which the blob store
+	// dispatches. This is required because manifest and layer blobs have not
+	// yet been fully merged. At some point, this functionality should be
+	// removed an the blob links folder should be merged. The first entry is
+	// treated as the "canonical" link location and will be used for writes.
+	linkPathFns []linkPathFunc
+}
+
+var _ distribution.BlobDescriptorService = &linkedBlobStatter{}
+
+func (lbs *linkedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	var (
+		found  bool
+		target digest.Digest
+	)
+
+	// try the many link path functions until we get success or an error that
+	// is not PathNotFoundError.
+	for _, linkPathFn := range lbs.linkPathFns {
+		var err error
+		target, err = lbs.resolveWithLinkFunc(ctx, dgst, linkPathFn)
+
+		if err == nil {
+			found = true
+			break // success!
+		}
+
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			// do nothing, just move to the next linkPathFn
+		default:
+			return distribution.Descriptor{}, err
+		}
+	}
+
+	if !found {
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	if target != dgst {
+		// Track when we are doing cross-digest domain lookups. ie, sha512 to sha256.
+		context.GetLogger(ctx).Warnf("looking up blob with canonical target: %v -> %v", dgst, target)
+	}
+
+	// TODO(stevvooe): Look up repository local mediatype and replace that on
+	// the returned descriptor.
+
+	return lbs.blobStore.statter.Stat(ctx, target)
+}
+
+func (lbs *linkedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) (err error) {
+	// clear any possible existence of a link described in linkPathFns
+	for _, linkPathFn := range lbs.linkPathFns {
+		blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+		if err != nil {
+			return err
+		}
+
+		err = lbs.blobStore.driver.Delete(ctx, blobLinkPath)
+		if err != nil {
+			switch err := err.(type) {
+			case driver.PathNotFoundError:
+				continue // just ignore this error and continue
+			default:
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// resolveTargetWithFunc allows us to read a link to a resource with different
+// linkPathFuncs to let us try a few different paths before returning not
+// found.
+func (lbs *linkedBlobStatter) resolveWithLinkFunc(ctx context.Context, dgst digest.Digest, linkPathFn linkPathFunc) (digest.Digest, error) {
+	blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+	if err != nil {
+		return "", err
+	}
+
+	return lbs.blobStore.readlink(ctx, blobLinkPath)
+}
+
+func (lbs *linkedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	// The canonical descriptor for a blob is set at the commit phase of upload
+	return nil
+}
+
+// blobLinkPath provides the path to the blob link, also known as layers.
+func blobLinkPath(name string, dgst digest.Digest) (string, error) {
+	return pathFor(layerLinkPathSpec{name: name, digest: dgst})
+}
+
+// manifestRevisionLinkPath provides the path to the manifest revision link.
+func manifestRevisionLinkPath(name string, dgst digest.Digest) (string, error) {
+	return pathFor(manifestRevisionLinkPathSpec{name: name, revision: dgst})
+}

vendor/github.com/docker/distribution/registry/storage/manifestlisthandler.go

@@ -0,0 +1,92 @@
+package storage
+
+import (
+	"fmt"
+
+	"encoding/json"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/opencontainers/go-digest"
+)
+
+// manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
+type manifestListHandler struct {
+	repository distribution.Repository
+	blobStore  distribution.BlobStore
+	ctx        context.Context
+}
+
+var _ ManifestHandler = &manifestListHandler{}
+
+func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal")
+
+	var m manifestlist.DeserializedManifestList
+	if err := json.Unmarshal(content, &m); err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (ms *manifestListHandler) Put(ctx context.Context, manifestList distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Put")
+
+	m, ok := manifestList.(*manifestlist.DeserializedManifestList)
+	if !ok {
+		return "", fmt.Errorf("wrong type put to manifestListHandler: %T", manifestList)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt, payload, err := m.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. As a policy, the registry only tries to
+// store valid content, leaving trust policies of that content up to
+// consumers.
+func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst manifestlist.DeserializedManifestList, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if !skipDependencyVerification {
+		// This manifest service is different from the blob service
+		// returned by Blob. It uses a linked blob store to ensure that
+		// only manifests are accessible.
+
+		manifestService, err := ms.repository.Manifests(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, manifestDescriptor := range mnfst.References() {
+			exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
+			if err != nil && err != distribution.ErrBlobUnknown {
+				errs = append(errs, err)
+			}
+			if err != nil || !exists {
+				// On error here, we always append unknown blob errors.
+				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
+			}
+		}
+	}
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/manifeststore.go

@@ -0,0 +1,141 @@
+package storage
+
+import (
+	"fmt"
+
+	"encoding/json"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/opencontainers/go-digest"
+)
+
+// A ManifestHandler gets and puts manifests of a particular type.
+type ManifestHandler interface {
+	// Unmarshal unmarshals the manifest from a byte slice.
+	Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error)
+
+	// Put creates or updates the given manifest returning the manifest digest.
+	Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error)
+}
+
+// SkipLayerVerification allows a manifest to be Put before its
+// layers are on the filesystem
+func SkipLayerVerification() distribution.ManifestServiceOption {
+	return skipLayerOption{}
+}
+
+type skipLayerOption struct{}
+
+func (o skipLayerOption) Apply(m distribution.ManifestService) error {
+	if ms, ok := m.(*manifestStore); ok {
+		ms.skipDependencyVerification = true
+		return nil
+	}
+	return fmt.Errorf("skip layer verification only valid for manifestStore")
+}
+
+type manifestStore struct {
+	repository *repository
+	blobStore  *linkedBlobStore
+	ctx        context.Context
+
+	skipDependencyVerification bool
+
+	schema1Handler      ManifestHandler
+	schema2Handler      ManifestHandler
+	manifestListHandler ManifestHandler
+}
+
+var _ distribution.ManifestService = &manifestStore{}
+
+func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")
+
+	_, err := ms.blobStore.Stat(ms.ctx, dgst)
+	if err != nil {
+		if err == distribution.ErrBlobUnknown {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
+
+	// TODO(stevvooe): Need to check descriptor from above to ensure that the
+	// mediatype is as we expect for the manifest store.
+
+	content, err := ms.blobStore.Get(ctx, dgst)
+	if err != nil {
+		if err == distribution.ErrBlobUnknown {
+			return nil, distribution.ErrManifestUnknownRevision{
+				Name:     ms.repository.Named().Name(),
+				Revision: dgst,
+			}
+		}
+
+		return nil, err
+	}
+
+	var versioned manifest.Versioned
+	if err = json.Unmarshal(content, &versioned); err != nil {
+		return nil, err
+	}
+
+	switch versioned.SchemaVersion {
+	case 1:
+		return ms.schema1Handler.Unmarshal(ctx, dgst, content)
+	case 2:
+		// This can be an image manifest or a manifest list
+		switch versioned.MediaType {
+		case schema2.MediaTypeManifest:
+			return ms.schema2Handler.Unmarshal(ctx, dgst, content)
+		case manifestlist.MediaTypeManifestList:
+			return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
+		default:
+			return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)}
+		}
+	}
+
+	return nil, fmt.Errorf("unrecognized manifest schema version %d", versioned.SchemaVersion)
+}
+
+func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")
+
+	switch manifest.(type) {
+	case *schema1.SignedManifest:
+		return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification)
+	case *schema2.DeserializedManifest:
+		return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification)
+	case *manifestlist.DeserializedManifestList:
+		return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification)
+	}
+
+	return "", fmt.Errorf("unrecognized manifest type %T", manifest)
+}
+
+// Delete removes the revision of the specified manifest.
+func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
+	return ms.blobStore.Delete(ctx, dgst)
+}
+
+func (ms *manifestStore) Enumerate(ctx context.Context, ingester func(digest.Digest) error) error {
+	err := ms.blobStore.Enumerate(ctx, func(dgst digest.Digest) error {
+		err := ingester(dgst)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}

vendor/github.com/docker/distribution/registry/storage/paths.go

@@ -0,0 +1,490 @@
+package storage
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/opencontainers/go-digest"
+)
+
+const (
+	storagePathVersion = "v2"                // fixed storage layout version
+	storagePathRoot    = "/docker/registry/" // all driver paths have a prefix
+
+	// TODO(stevvooe): Get rid of the "storagePathRoot". Initially, we though
+	// storage path root would configurable for all drivers through this
+	// package. In reality, we've found it simpler to do this on a per driver
+	// basis.
+)
+
+// pathFor maps paths based on "object names" and their ids. The "object
+// names" mapped by are internal to the storage system.
+//
+// The path layout in the storage backend is roughly as follows:
+//
+//		<root>/v2
+//			-> repositories/
+// 				-><name>/
+// 					-> _manifests/
+// 						revisions
+//							-> <manifest digest path>
+//								-> link
+// 						tags/<tag>
+//							-> current/link
+// 							-> index
+//								-> <algorithm>/<hex digest>/link
+// 					-> _layers/
+// 						<layer links to blob store>
+// 					-> _uploads/<id>
+// 						data
+// 						startedat
+// 						hashstates/<algorithm>/<offset>
+//			-> blob/<algorithm>
+//				<split directory content addressable storage>
+//
+// The storage backend layout is broken up into a content-addressable blob
+// store and repositories. The content-addressable blob store holds most data
+// throughout the backend, keyed by algorithm and digests of the underlying
+// content. Access to the blob store is controlled through links from the
+// repository to blobstore.
+//
+// A repository is made up of layers, manifests and tags. The layers component
+// is just a directory of layers which are "linked" into a repository. A layer
+// can only be accessed through a qualified repository name if it is linked in
+// the repository. Uploads of layers are managed in the uploads directory,
+// which is key by upload id. When all data for an upload is received, the
+// data is moved into the blob store and the upload directory is deleted.
+// Abandoned uploads can be garbage collected by reading the startedat file
+// and removing uploads that have been active for longer than a certain time.
+//
+// The third component of the repository directory is the manifests store,
+// which is made up of a revision store and tag store. Manifests are stored in
+// the blob store and linked into the revision store.
+// While the registry can save all revisions of a manifest, no relationship is
+// implied as to the ordering of changes to a manifest. The tag store provides
+// support for name, tag lookups of manifests, using "current/link" under a
+// named tag directory. An index is maintained to support deletions of all
+// revisions of a given manifest tag.
+//
+// We cover the path formats implemented by this path mapper below.
+//
+//	Manifests:
+//
+// 	manifestRevisionsPathSpec:      <root>/v2/repositories/<name>/_manifests/revisions/
+// 	manifestRevisionPathSpec:      <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/
+// 	manifestRevisionLinkPathSpec:  <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/link
+//
+//	Tags:
+//
+// 	manifestTagsPathSpec:                  <root>/v2/repositories/<name>/_manifests/tags/
+// 	manifestTagPathSpec:                   <root>/v2/repositories/<name>/_manifests/tags/<tag>/
+// 	manifestTagCurrentPathSpec:            <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
+// 	manifestTagIndexPathSpec:              <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
+// 	manifestTagIndexEntryPathSpec:         <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/
+// 	manifestTagIndexEntryLinkPathSpec:     <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
+//
+// 	Blobs:
+//
+// 	layerLinkPathSpec:            <root>/v2/repositories/<name>/_layers/<algorithm>/<hex digest>/link
+//
+//	Uploads:
+//
+// 	uploadDataPathSpec:             <root>/v2/repositories/<name>/_uploads/<id>/data
+// 	uploadStartedAtPathSpec:        <root>/v2/repositories/<name>/_uploads/<id>/startedat
+// 	uploadHashStatePathSpec:        <root>/v2/repositories/<name>/_uploads/<id>/hashstates/<algorithm>/<offset>
+//
+//	Blob Store:
+//
+//	blobsPathSpec:                  <root>/v2/blobs/
+// 	blobPathSpec:                   <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>
+// 	blobDataPathSpec:               <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
+// 	blobMediaTypePathSpec:               <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
+//
+// For more information on the semantic meaning of each path and their
+// contents, please see the path spec documentation.
+func pathFor(spec pathSpec) (string, error) {
+
+	// Switch on the path object type and return the appropriate path. At
+	// first glance, one may wonder why we don't use an interface to
+	// accomplish this. By keep the formatting separate from the pathSpec, we
+	// keep separate the path generation componentized. These specs could be
+	// passed to a completely different mapper implementation and generate a
+	// different set of paths.
+	//
+	// For example, imagine migrating from one backend to the other: one could
+	// build a filesystem walker that converts a string path in one version,
+	// to an intermediate path object, than can be consumed and mapped by the
+	// other version.
+
+	rootPrefix := []string{storagePathRoot, storagePathVersion}
+	repoPrefix := append(rootPrefix, "repositories")
+
+	switch v := spec.(type) {
+
+	case manifestRevisionsPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_manifests", "revisions")...), nil
+
+	case manifestRevisionPathSpec:
+		components, err := digestPathComponents(v.revision, false)
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(append(append(repoPrefix, v.name, "_manifests", "revisions"), components...)...), nil
+	case manifestRevisionLinkPathSpec:
+		root, err := pathFor(manifestRevisionPathSpec{
+			name:     v.name,
+			revision: v.revision,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "link"), nil
+	case manifestTagsPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_manifests", "tags")...), nil
+	case manifestTagPathSpec:
+		root, err := pathFor(manifestTagsPathSpec{
+			name: v.name,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, v.tag), nil
+	case manifestTagCurrentPathSpec:
+		root, err := pathFor(manifestTagPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "current", "link"), nil
+	case manifestTagIndexPathSpec:
+		root, err := pathFor(manifestTagPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "index"), nil
+	case manifestTagIndexEntryLinkPathSpec:
+		root, err := pathFor(manifestTagIndexEntryPathSpec{
+			name:     v.name,
+			tag:      v.tag,
+			revision: v.revision,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "link"), nil
+	case manifestTagIndexEntryPathSpec:
+		root, err := pathFor(manifestTagIndexPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		components, err := digestPathComponents(v.revision, false)
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, path.Join(components...)), nil
+	case layerLinkPathSpec:
+		components, err := digestPathComponents(v.digest, false)
+		if err != nil {
+			return "", err
+		}
+
+		// TODO(stevvooe): Right now, all blobs are linked under "_layers". If
+		// we have future migrations, we may want to rename this to "_blobs".
+		// A migration strategy would simply leave existing items in place and
+		// write the new paths, commit a file then delete the old files.
+
+		blobLinkPathComponents := append(repoPrefix, v.name, "_layers")
+
+		return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil
+	case blobsPathSpec:
+		blobsPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(blobsPathPrefix...), nil
+	case blobPathSpec:
+		components, err := digestPathComponents(v.digest, true)
+		if err != nil {
+			return "", err
+		}
+
+		blobPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(append(blobPathPrefix, components...)...), nil
+	case blobDataPathSpec:
+		components, err := digestPathComponents(v.digest, true)
+		if err != nil {
+			return "", err
+		}
+
+		components = append(components, "data")
+		blobPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(append(blobPathPrefix, components...)...), nil
+
+	case uploadDataPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "data")...), nil
+	case uploadStartedAtPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "startedat")...), nil
+	case uploadHashStatePathSpec:
+		offset := fmt.Sprintf("%d", v.offset)
+		if v.list {
+			offset = "" // Limit to the prefix for listing offsets.
+		}
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil
+	case repositoriesRootPathSpec:
+		return path.Join(repoPrefix...), nil
+	default:
+		// TODO(sday): This is an internal error. Ensure it doesn't escape (panic?).
+		return "", fmt.Errorf("unknown path spec: %#v", v)
+	}
+}
+
+// pathSpec is a type to mark structs as path specs. There is no
+// implementation because we'd like to keep the specs and the mappers
+// decoupled.
+type pathSpec interface {
+	pathSpec()
+}
+
+// manifestRevisionsPathSpec describes the directory path for
+// a manifest revision.
+type manifestRevisionsPathSpec struct {
+	name string
+}
+
+func (manifestRevisionsPathSpec) pathSpec() {}
+
+// manifestRevisionPathSpec describes the components of the directory path for
+// a manifest revision.
+type manifestRevisionPathSpec struct {
+	name     string
+	revision digest.Digest
+}
+
+func (manifestRevisionPathSpec) pathSpec() {}
+
+// manifestRevisionLinkPathSpec describes the path components required to look
+// up the data link for a revision of a manifest. If this file is not present,
+// the manifest blob is not available in the given repo. The contents of this
+// file should just be the digest.
+type manifestRevisionLinkPathSpec struct {
+	name     string
+	revision digest.Digest
+}
+
+func (manifestRevisionLinkPathSpec) pathSpec() {}
+
+// manifestTagsPathSpec describes the path elements required to point to the
+// manifest tags directory.
+type manifestTagsPathSpec struct {
+	name string
+}
+
+func (manifestTagsPathSpec) pathSpec() {}
+
+// manifestTagPathSpec describes the path elements required to point to the
+// manifest tag links files under a repository. These contain a blob id that
+// can be used to look up the data and signatures.
+type manifestTagPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagPathSpec) pathSpec() {}
+
+// manifestTagCurrentPathSpec describes the link to the current revision for a
+// given tag.
+type manifestTagCurrentPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagCurrentPathSpec) pathSpec() {}
+
+// manifestTagCurrentPathSpec describes the link to the index of revisions
+// with the given tag.
+type manifestTagIndexPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagIndexPathSpec) pathSpec() {}
+
+// manifestTagIndexEntryPathSpec contains the entries of the index by revision.
+type manifestTagIndexEntryPathSpec struct {
+	name     string
+	tag      string
+	revision digest.Digest
+}
+
+func (manifestTagIndexEntryPathSpec) pathSpec() {}
+
+// manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a
+// manifest with given tag within the index.
+type manifestTagIndexEntryLinkPathSpec struct {
+	name     string
+	tag      string
+	revision digest.Digest
+}
+
+func (manifestTagIndexEntryLinkPathSpec) pathSpec() {}
+
+// blobLinkPathSpec specifies a path for a blob link, which is a file with a
+// blob id. The blob link will contain a content addressable blob id reference
+// into the blob store. The format of the contents is as follows:
+//
+// 	<algorithm>:<hex digest of layer data>
+//
+// The following example of the file contents is more illustrative:
+//
+// 	sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36
+//
+// This  indicates that there is a blob with the id/digest, calculated via
+// sha256 that can be fetched from the blob store.
+type layerLinkPathSpec struct {
+	name   string
+	digest digest.Digest
+}
+
+func (layerLinkPathSpec) pathSpec() {}
+
+// blobAlgorithmReplacer does some very simple path sanitization for user
+// input. Paths should be "safe" before getting this far due to strict digest
+// requirements but we can add further path conversion here, if needed.
+var blobAlgorithmReplacer = strings.NewReplacer(
+	"+", "/",
+	".", "/",
+	";", "/",
+)
+
+// blobsPathSpec contains the path for the blobs directory
+type blobsPathSpec struct{}
+
+func (blobsPathSpec) pathSpec() {}
+
+// blobPathSpec contains the path for the registry global blob store.
+type blobPathSpec struct {
+	digest digest.Digest
+}
+
+func (blobPathSpec) pathSpec() {}
+
+// blobDataPathSpec contains the path for the registry global blob store. For
+// now, this contains layer data, exclusively.
+type blobDataPathSpec struct {
+	digest digest.Digest
+}
+
+func (blobDataPathSpec) pathSpec() {}
+
+// uploadDataPathSpec defines the path parameters of the data file for
+// uploads.
+type uploadDataPathSpec struct {
+	name string
+	id   string
+}
+
+func (uploadDataPathSpec) pathSpec() {}
+
+// uploadDataPathSpec defines the path parameters for the file that stores the
+// start time of an uploads. If it is missing, the upload is considered
+// unknown. Admittedly, the presence of this file is an ugly hack to make sure
+// we have a way to cleanup old or stalled uploads that doesn't rely on driver
+// FileInfo behavior. If we come up with a more clever way to do this, we
+// should remove this file immediately and rely on the startetAt field from
+// the client to enforce time out policies.
+type uploadStartedAtPathSpec struct {
+	name string
+	id   string
+}
+
+func (uploadStartedAtPathSpec) pathSpec() {}
+
+// uploadHashStatePathSpec defines the path parameters for the file that stores
+// the hash function state of an upload at a specific byte offset. If `list` is
+// set, then the path mapper will generate a list prefix for all hash state
+// offsets for the upload identified by the name, id, and alg.
+type uploadHashStatePathSpec struct {
+	name   string
+	id     string
+	alg    digest.Algorithm
+	offset int64
+	list   bool
+}
+
+func (uploadHashStatePathSpec) pathSpec() {}
+
+// repositoriesRootPathSpec returns the root of repositories
+type repositoriesRootPathSpec struct {
+}
+
+func (repositoriesRootPathSpec) pathSpec() {}
+
+// digestPathComponents provides a consistent path breakdown for a given
+// digest. For a generic digest, it will be as follows:
+//
+// 	<algorithm>/<hex digest>
+//
+// If multilevel is true, the first two bytes of the digest will separate
+// groups of digest folder. It will be as follows:
+//
+// 	<algorithm>/<first two bytes of digest>/<full digest>
+//
+func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) {
+	if err := dgst.Validate(); err != nil {
+		return nil, err
+	}
+
+	algorithm := blobAlgorithmReplacer.Replace(string(dgst.Algorithm()))
+	hex := dgst.Hex()
+	prefix := []string{algorithm}
+
+	var suffix []string
+
+	if multilevel {
+		suffix = append(suffix, hex[:2])
+	}
+
+	suffix = append(suffix, hex)
+
+	return append(prefix, suffix...), nil
+}
+
+// Reconstructs a digest from a path
+func digestFromPath(digestPath string) (digest.Digest, error) {
+
+	digestPath = strings.TrimSuffix(digestPath, "/data")
+	dir, hex := path.Split(digestPath)
+	dir = path.Dir(dir)
+	dir, next := path.Split(dir)
+
+	// next is either the algorithm OR the first two characters in the hex string
+	var algo string
+	if next == hex[:2] {
+		algo = path.Base(dir)
+	} else {
+		algo = next
+	}
+
+	dgst := digest.NewDigestFromHex(algo, hex)
+	return dgst, dgst.Validate()
+}

vendor/github.com/docker/distribution/registry/storage/purgeuploads.go

@@ -0,0 +1,139 @@
+package storage
+
+import (
+	"path"
+	"strings"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/context"
+	storageDriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/uuid"
+)
+
+// uploadData stored the location of temporary files created during a layer upload
+// along with the date the upload was started
+type uploadData struct {
+	containingDir string
+	startedAt     time.Time
+}
+
+func newUploadData() uploadData {
+	return uploadData{
+		containingDir: "",
+		// default to far in future to protect against missing startedat
+		startedAt: time.Now().Add(time.Duration(10000 * time.Hour)),
+	}
+}
+
+// PurgeUploads deletes files from the upload directory
+// created before olderThan.  The list of files deleted and errors
+// encountered are returned
+func PurgeUploads(ctx context.Context, driver storageDriver.StorageDriver, olderThan time.Time, actuallyDelete bool) ([]string, []error) {
+	log.Infof("PurgeUploads starting: olderThan=%s, actuallyDelete=%t", olderThan, actuallyDelete)
+	uploadData, errors := getOutstandingUploads(ctx, driver)
+	var deleted []string
+	for _, uploadData := range uploadData {
+		if uploadData.startedAt.Before(olderThan) {
+			var err error
+			log.Infof("Upload files in %s have older date (%s) than purge date (%s).  Removing upload directory.",
+				uploadData.containingDir, uploadData.startedAt, olderThan)
+			if actuallyDelete {
+				err = driver.Delete(ctx, uploadData.containingDir)
+			}
+			if err == nil {
+				deleted = append(deleted, uploadData.containingDir)
+			} else {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	log.Infof("Purge uploads finished.  Num deleted=%d, num errors=%d", len(deleted), len(errors))
+	return deleted, errors
+}
+
+// getOutstandingUploads walks the upload directory, collecting files
+// which could be eligible for deletion.  The only reliable way to
+// classify the age of a file is with the date stored in the startedAt
+// file, so gather files by UUID with a date from startedAt.
+func getOutstandingUploads(ctx context.Context, driver storageDriver.StorageDriver) (map[string]uploadData, []error) {
+	var errors []error
+	uploads := make(map[string]uploadData, 0)
+
+	inUploadDir := false
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return uploads, append(errors, err)
+	}
+
+	err = Walk(ctx, driver, root, func(fileInfo storageDriver.FileInfo) error {
+		filePath := fileInfo.Path()
+		_, file := path.Split(filePath)
+		if file[0] == '_' {
+			// Reserved directory
+			inUploadDir = (file == "_uploads")
+
+			if fileInfo.IsDir() && !inUploadDir {
+				return ErrSkipDir
+			}
+
+		}
+
+		uuid, isContainingDir := uuidFromPath(filePath)
+		if uuid == "" {
+			// Cannot reliably delete
+			return nil
+		}
+		ud, ok := uploads[uuid]
+		if !ok {
+			ud = newUploadData()
+		}
+		if isContainingDir {
+			ud.containingDir = filePath
+		}
+		if file == "startedat" {
+			if t, err := readStartedAtFile(driver, filePath); err == nil {
+				ud.startedAt = t
+			} else {
+				errors = pushError(errors, filePath, err)
+			}
+
+		}
+
+		uploads[uuid] = ud
+		return nil
+	})
+
+	if err != nil {
+		errors = pushError(errors, root, err)
+	}
+	return uploads, errors
+}
+
+// uuidFromPath extracts the upload UUID from a given path
+// If the UUID is the last path component, this is the containing
+// directory for all upload files
+func uuidFromPath(path string) (string, bool) {
+	components := strings.Split(path, "/")
+	for i := len(components) - 1; i >= 0; i-- {
+		if u, err := uuid.Parse(components[i]); err == nil {
+			return u.String(), i == len(components)-1
+		}
+	}
+	return "", false
+}
+
+// readStartedAtFile reads the date from an upload's startedAtFile
+func readStartedAtFile(driver storageDriver.StorageDriver, path string) (time.Time, error) {
+	// todo:(richardscothern) - pass in a context
+	startedAtBytes, err := driver.GetContent(context.Background(), path)
+	if err != nil {
+		return time.Now(), err
+	}
+	startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes))
+	if err != nil {
+		return time.Now(), err
+	}
+	return startedAt, nil
+}

vendor/github.com/docker/distribution/registry/storage/registry.go

@@ -0,0 +1,306 @@
+package storage
+
+import (
+	"regexp"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/libtrust"
+)
+
+// registry is the top-level implementation of Registry for use in the storage
+// package. All instances should descend from this object.
+type registry struct {
+	blobStore                    *blobStore
+	blobServer                   *blobServer
+	statter                      *blobStatter // global statter service.
+	blobDescriptorCacheProvider  cache.BlobDescriptorCacheProvider
+	deleteEnabled                bool
+	resumableDigestEnabled       bool
+	schema1SigningKey            libtrust.PrivateKey
+	blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory
+	manifestURLs                 manifestURLs
+}
+
+// manifestURLs holds regular expressions for controlling manifest URL whitelisting
+type manifestURLs struct {
+	allow *regexp.Regexp
+	deny  *regexp.Regexp
+}
+
+// RegistryOption is the type used for functional options for NewRegistry.
+type RegistryOption func(*registry) error
+
+// EnableRedirect is a functional option for NewRegistry. It causes the backend
+// blob server to attempt using (StorageDriver).URLFor to serve all blobs.
+func EnableRedirect(registry *registry) error {
+	registry.blobServer.redirect = true
+	return nil
+}
+
+// EnableDelete is a functional option for NewRegistry. It enables deletion on
+// the registry.
+func EnableDelete(registry *registry) error {
+	registry.deleteEnabled = true
+	return nil
+}
+
+// DisableDigestResumption is a functional option for NewRegistry. It should be
+// used if the registry is acting as a caching proxy.
+func DisableDigestResumption(registry *registry) error {
+	registry.resumableDigestEnabled = false
+	return nil
+}
+
+// ManifestURLsAllowRegexp is a functional option for NewRegistry.
+func ManifestURLsAllowRegexp(r *regexp.Regexp) RegistryOption {
+	return func(registry *registry) error {
+		registry.manifestURLs.allow = r
+		return nil
+	}
+}
+
+// ManifestURLsDenyRegexp is a functional option for NewRegistry.
+func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption {
+	return func(registry *registry) error {
+		registry.manifestURLs.deny = r
+		return nil
+	}
+}
+
+// Schema1SigningKey returns a functional option for NewRegistry. It sets the
+// key for signing  all schema1 manifests.
+func Schema1SigningKey(key libtrust.PrivateKey) RegistryOption {
+	return func(registry *registry) error {
+		registry.schema1SigningKey = key
+		return nil
+	}
+}
+
+// BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the
+// factory to create BlobDescriptorServiceFactory middleware.
+func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption {
+	return func(registry *registry) error {
+		registry.blobDescriptorServiceFactory = factory
+		return nil
+	}
+}
+
+// BlobDescriptorCacheProvider returns a functional option for
+// NewRegistry. It creates a cached blob statter for use by the
+// registry.
+func BlobDescriptorCacheProvider(blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider) RegistryOption {
+	// TODO(aaronl): The duplication of statter across several objects is
+	// ugly, and prevents us from using interface types in the registry
+	// struct. Ideally, blobStore and blobServer should be lazily
+	// initialized, and use the current value of
+	// blobDescriptorCacheProvider.
+	return func(registry *registry) error {
+		if blobDescriptorCacheProvider != nil {
+			statter := cache.NewCachedBlobStatter(blobDescriptorCacheProvider, registry.statter)
+			registry.blobStore.statter = statter
+			registry.blobServer.statter = statter
+			registry.blobDescriptorCacheProvider = blobDescriptorCacheProvider
+		}
+		return nil
+	}
+}
+
+// NewRegistry creates a new registry instance from the provided driver. The
+// resulting registry may be shared by multiple goroutines but is cheap to
+// allocate. If the Redirect option is specified, the backend blob server will
+// attempt to use (StorageDriver).URLFor to serve all blobs.
+func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, options ...RegistryOption) (distribution.Namespace, error) {
+	// create global statter
+	statter := &blobStatter{
+		driver: driver,
+	}
+
+	bs := &blobStore{
+		driver:  driver,
+		statter: statter,
+	}
+
+	registry := &registry{
+		blobStore: bs,
+		blobServer: &blobServer{
+			driver:  driver,
+			statter: statter,
+			pathFn:  bs.path,
+		},
+		statter:                statter,
+		resumableDigestEnabled: true,
+	}
+
+	for _, option := range options {
+		if err := option(registry); err != nil {
+			return nil, err
+		}
+	}
+
+	return registry, nil
+}
+
+// Scope returns the namespace scope for a registry. The registry
+// will only serve repositories contained within this scope.
+func (reg *registry) Scope() distribution.Scope {
+	return distribution.GlobalScope
+}
+
+// Repository returns an instance of the repository tied to the registry.
+// Instances should not be shared between goroutines but are cheap to
+// allocate. In general, they should be request scoped.
+func (reg *registry) Repository(ctx context.Context, canonicalName reference.Named) (distribution.Repository, error) {
+	var descriptorCache distribution.BlobDescriptorService
+	if reg.blobDescriptorCacheProvider != nil {
+		var err error
+		descriptorCache, err = reg.blobDescriptorCacheProvider.RepositoryScoped(canonicalName.Name())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &repository{
+		ctx:             ctx,
+		registry:        reg,
+		name:            canonicalName,
+		descriptorCache: descriptorCache,
+	}, nil
+}
+
+func (reg *registry) Blobs() distribution.BlobEnumerator {
+	return reg.blobStore
+}
+
+func (reg *registry) BlobStatter() distribution.BlobStatter {
+	return reg.statter
+}
+
+// repository provides name-scoped access to various services.
+type repository struct {
+	*registry
+	ctx             context.Context
+	name            reference.Named
+	descriptorCache distribution.BlobDescriptorService
+}
+
+// Name returns the name of the repository.
+func (repo *repository) Named() reference.Named {
+	return repo.name
+}
+
+func (repo *repository) Tags(ctx context.Context) distribution.TagService {
+	tags := &tagStore{
+		repository: repo,
+		blobStore:  repo.registry.blobStore,
+	}
+
+	return tags
+}
+
+// Manifests returns an instance of ManifestService. Instantiation is cheap and
+// may be context sensitive in the future. The instance should be used similar
+// to a request local.
+func (repo *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
+	manifestLinkPathFns := []linkPathFunc{
+		// NOTE(stevvooe): Need to search through multiple locations since
+		// 2.1.0 unintentionally linked into  _layers.
+		manifestRevisionLinkPath,
+		blobLinkPath,
+	}
+
+	manifestDirectoryPathSpec := manifestRevisionsPathSpec{name: repo.name.Name()}
+
+	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
+		blobStore:   repo.blobStore,
+		repository:  repo,
+		linkPathFns: manifestLinkPathFns,
+	}
+
+	if repo.registry.blobDescriptorServiceFactory != nil {
+		statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter)
+	}
+
+	blobStore := &linkedBlobStore{
+		ctx:                  ctx,
+		blobStore:            repo.blobStore,
+		repository:           repo,
+		deleteEnabled:        repo.registry.deleteEnabled,
+		blobAccessController: statter,
+
+		// TODO(stevvooe): linkPath limits this blob store to only
+		// manifests. This instance cannot be used for blob checks.
+		linkPathFns:           manifestLinkPathFns,
+		linkDirectoryPathSpec: manifestDirectoryPathSpec,
+	}
+
+	ms := &manifestStore{
+		ctx:        ctx,
+		repository: repo,
+		blobStore:  blobStore,
+		schema1Handler: &signedManifestHandler{
+			ctx:               ctx,
+			schema1SigningKey: repo.schema1SigningKey,
+			repository:        repo,
+			blobStore:         blobStore,
+		},
+		schema2Handler: &schema2ManifestHandler{
+			ctx:          ctx,
+			repository:   repo,
+			blobStore:    blobStore,
+			manifestURLs: repo.registry.manifestURLs,
+		},
+		manifestListHandler: &manifestListHandler{
+			ctx:        ctx,
+			repository: repo,
+			blobStore:  blobStore,
+		},
+	}
+
+	// Apply options
+	for _, option := range options {
+		err := option.Apply(ms)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ms, nil
+}
+
+// Blobs returns an instance of the BlobStore. Instantiation is cheap and
+// may be context sensitive in the future. The instance should be used similar
+// to a request local.
+func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
+	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
+		blobStore:   repo.blobStore,
+		repository:  repo,
+		linkPathFns: []linkPathFunc{blobLinkPath},
+	}
+
+	if repo.descriptorCache != nil {
+		statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter)
+	}
+
+	if repo.registry.blobDescriptorServiceFactory != nil {
+		statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter)
+	}
+
+	return &linkedBlobStore{
+		registry:             repo.registry,
+		blobStore:            repo.blobStore,
+		blobServer:           repo.blobServer,
+		blobAccessController: statter,
+		repository:           repo,
+		ctx:                  ctx,
+
+		// TODO(stevvooe): linkPath limits this blob store to only layers.
+		// This instance cannot be used for manifest checks.
+		linkPathFns:            []linkPathFunc{blobLinkPath},
+		deleteEnabled:          repo.registry.deleteEnabled,
+		resumableDigestEnabled: repo.resumableDigestEnabled,
+	}
+}

vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler.go

@@ -0,0 +1,136 @@
+package storage
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/opencontainers/go-digest"
+)
+
+var (
+	errUnexpectedURL = errors.New("unexpected URL on layer")
+	errMissingURL    = errors.New("missing URL on layer")
+	errInvalidURL    = errors.New("invalid URL on layer")
+)
+
+//schema2ManifestHandler is a ManifestHandler that covers schema2 manifests.
+type schema2ManifestHandler struct {
+	repository   distribution.Repository
+	blobStore    distribution.BlobStore
+	ctx          context.Context
+	manifestURLs manifestURLs
+}
+
+var _ ManifestHandler = &schema2ManifestHandler{}
+
+func (ms *schema2ManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Unmarshal")
+
+	var m schema2.DeserializedManifest
+	if err := json.Unmarshal(content, &m); err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (ms *schema2ManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Put")
+
+	m, ok := manifest.(*schema2.DeserializedManifest)
+	if !ok {
+		return "", fmt.Errorf("non-schema2 manifest put to schema2ManifestHandler: %T", manifest)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt, payload, err := m.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. As a policy, the registry only tries to store
+// valid content, leaving trust policies of that content up to consumers.
+func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst schema2.DeserializedManifest, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if skipDependencyVerification {
+		return nil
+	}
+
+	manifestService, err := ms.repository.Manifests(ctx)
+	if err != nil {
+		return err
+	}
+
+	blobsService := ms.repository.Blobs(ctx)
+
+	for _, descriptor := range mnfst.References() {
+		var err error
+
+		switch descriptor.MediaType {
+		case schema2.MediaTypeForeignLayer:
+			// Clients download this layer from an external URL, so do not check for
+			// its presense.
+			if len(descriptor.URLs) == 0 {
+				err = errMissingURL
+			}
+			allow := ms.manifestURLs.allow
+			deny := ms.manifestURLs.deny
+			for _, u := range descriptor.URLs {
+				var pu *url.URL
+				pu, err = url.Parse(u)
+				if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) {
+					err = errInvalidURL
+					break
+				}
+			}
+		case schema2.MediaTypeManifest, schema1.MediaTypeManifest:
+			var exists bool
+			exists, err = manifestService.Exists(ctx, descriptor.Digest)
+			if err != nil || !exists {
+				err = distribution.ErrBlobUnknown // just coerce to unknown.
+			}
+
+			fallthrough // double check the blob store.
+		default:
+			// forward all else to blob storage
+			if len(descriptor.URLs) == 0 {
+				_, err = blobsService.Stat(ctx, descriptor.Digest)
+			}
+		}
+
+		if err != nil {
+			if err != distribution.ErrBlobUnknown {
+				errs = append(errs, err)
+			}
+
+			// On error here, we always append unknown blob errors.
+			errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest})
+		}
+	}
+
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/signedmanifesthandler.go

@@ -0,0 +1,141 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/libtrust"
+	"github.com/opencontainers/go-digest"
+)
+
+// signedManifestHandler is a ManifestHandler that covers schema1 manifests. It
+// can unmarshal and put schema1 manifests that have been signed by libtrust.
+type signedManifestHandler struct {
+	repository        distribution.Repository
+	schema1SigningKey libtrust.PrivateKey
+	blobStore         distribution.BlobStore
+	ctx               context.Context
+}
+
+var _ ManifestHandler = &signedManifestHandler{}
+
+func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal")
+
+	var (
+		signatures [][]byte
+		err        error
+	)
+
+	jsig, err := libtrust.NewJSONSignature(content, signatures...)
+	if err != nil {
+		return nil, err
+	}
+
+	if ms.schema1SigningKey != nil {
+		if err := jsig.Sign(ms.schema1SigningKey); err != nil {
+			return nil, err
+		}
+	}
+
+	// Extract the pretty JWS
+	raw, err := jsig.PrettySignature("signatures")
+	if err != nil {
+		return nil, err
+	}
+
+	var sm schema1.SignedManifest
+	if err := json.Unmarshal(raw, &sm); err != nil {
+		return nil, err
+	}
+	return &sm, nil
+}
+
+func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Put")
+
+	sm, ok := manifest.(*schema1.SignedManifest)
+	if !ok {
+		return "", fmt.Errorf("non-schema1 manifest put to signedManifestHandler: %T", manifest)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *sm, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt := schema1.MediaTypeManifest
+	payload := sm.Canonical
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. It ensures that the signature is valid for the
+// enclosed payload. As a policy, the registry only tries to store valid
+// content, leaving trust policies of that content up to consumers.
+func (ms *signedManifestHandler) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if len(mnfst.Name) > reference.NameTotalLengthMax {
+		errs = append(errs,
+			distribution.ErrManifestNameInvalid{
+				Name:   mnfst.Name,
+				Reason: fmt.Errorf("manifest name must not be more than %v characters", reference.NameTotalLengthMax),
+			})
+	}
+
+	if !reference.NameRegexp.MatchString(mnfst.Name) {
+		errs = append(errs,
+			distribution.ErrManifestNameInvalid{
+				Name:   mnfst.Name,
+				Reason: fmt.Errorf("invalid manifest name format"),
+			})
+	}
+
+	if len(mnfst.History) != len(mnfst.FSLayers) {
+		errs = append(errs, fmt.Errorf("mismatched history and fslayer cardinality %d != %d",
+			len(mnfst.History), len(mnfst.FSLayers)))
+	}
+
+	if _, err := schema1.Verify(&mnfst); err != nil {
+		switch err {
+		case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
+			errs = append(errs, distribution.ErrManifestUnverified{})
+		default:
+			if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust
+				errs = append(errs, distribution.ErrManifestUnverified{})
+			} else {
+				errs = append(errs, err)
+			}
+		}
+	}
+
+	if !skipDependencyVerification {
+		for _, fsLayer := range mnfst.References() {
+			_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
+			if err != nil {
+				if err != distribution.ErrBlobUnknown {
+					errs = append(errs, err)
+				}
+
+				// On error here, we always append unknown blob errors.
+				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
+			}
+		}
+	}
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/tagstore.go

@@ -0,0 +1,191 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+var _ distribution.TagService = &tagStore{}
+
+// tagStore provides methods to manage manifest tags in a backend storage driver.
+// This implementation uses the same on-disk layout as the (now deleted) tag
+// store.  This provides backward compatibility with current registry deployments
+// which only makes use of the Digest field of the returned distribution.Descriptor
+// but does not enable full roundtripping of Descriptor objects
+type tagStore struct {
+	repository *repository
+	blobStore  *blobStore
+}
+
+// All returns all tags
+func (ts *tagStore) All(ctx context.Context) ([]string, error) {
+	var tags []string
+
+	pathSpec, err := pathFor(manifestTagPathSpec{
+		name: ts.repository.Named().Name(),
+	})
+	if err != nil {
+		return tags, err
+	}
+
+	entries, err := ts.blobStore.driver.List(ctx, pathSpec)
+	if err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Named().Name()}
+		default:
+			return tags, err
+		}
+	}
+
+	for _, entry := range entries {
+		_, filename := path.Split(entry)
+		tags = append(tags, filename)
+	}
+
+	return tags, nil
+}
+
+// exists returns true if the specified manifest tag exists in the repository.
+func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) {
+	tagPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return false, err
+	}
+
+	exists, err := exists(ctx, ts.blobStore.driver, tagPath)
+	if err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}
+
+// Tag tags the digest with the given tag, updating the the store to point at
+// the current tag. The digest must point to a manifest.
+func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
+	currentPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	lbs := ts.linkedBlobStore(ctx, tag)
+
+	// Link into the index
+	if err := lbs.linkBlob(ctx, desc); err != nil {
+		return err
+	}
+
+	// Overwrite the current link
+	return ts.blobStore.link(ctx, currentPath, desc.Digest)
+}
+
+// resolve the current revision for name and tag.
+func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
+	currentPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	revision, err := ts.blobStore.readlink(ctx, currentPath)
+	if err != nil {
+		switch err.(type) {
+		case storagedriver.PathNotFoundError:
+			return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag}
+		}
+
+		return distribution.Descriptor{}, err
+	}
+
+	return distribution.Descriptor{Digest: revision}, nil
+}
+
+// Untag removes the tag association
+func (ts *tagStore) Untag(ctx context.Context, tag string) error {
+	tagPath, err := pathFor(manifestTagPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	switch err.(type) {
+	case storagedriver.PathNotFoundError:
+		return distribution.ErrTagUnknown{Tag: tag}
+	case nil:
+		break
+	default:
+		return err
+	}
+
+	return ts.blobStore.driver.Delete(ctx, tagPath)
+}
+
+// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one
+// to index manifest blobs by tag name. While the tag store doesn't map
+// precisely to the linked blob store, using this ensures the links are
+// managed via the same code path.
+func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlobStore {
+	return &linkedBlobStore{
+		blobStore:  ts.blobStore,
+		repository: ts.repository,
+		ctx:        ctx,
+		linkPathFns: []linkPathFunc{func(name string, dgst digest.Digest) (string, error) {
+			return pathFor(manifestTagIndexEntryLinkPathSpec{
+				name:     name,
+				tag:      tag,
+				revision: dgst,
+			})
+
+		}},
+	}
+}
+
+// Lookup recovers a list of tags which refer to this digest.  When a manifest is deleted by
+// digest, tag entries which point to it need to be recovered to avoid dangling tags.
+func (ts *tagStore) Lookup(ctx context.Context, desc distribution.Descriptor) ([]string, error) {
+	allTags, err := ts.All(ctx)
+	switch err.(type) {
+	case distribution.ErrRepositoryUnknown:
+		// This tag store has been initialized but not yet populated
+		break
+	case nil:
+		break
+	default:
+		return nil, err
+	}
+
+	var tags []string
+	for _, tag := range allTags {
+		tagLinkPathSpec := manifestTagCurrentPathSpec{
+			name: ts.repository.Named().Name(),
+			tag:  tag,
+		}
+
+		tagLinkPath, err := pathFor(tagLinkPathSpec)
+		tagDigest, err := ts.blobStore.readlink(ctx, tagLinkPath)
+		if err != nil {
+			return nil, err
+		}
+
+		if tagDigest == desc.Digest {
+			tags = append(tags, tag)
+		}
+	}
+
+	return tags, nil
+}

vendor/github.com/docker/distribution/registry/storage/util.go

@@ -0,0 +1,21 @@
+package storage
+
+import (
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// Exists provides a utility method to test whether or not a path exists in
+// the given driver.
+func exists(ctx context.Context, drv driver.StorageDriver, path string) (bool, error) {
+	if _, err := drv.Stat(ctx, path); err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return false, nil
+		default:
+			return false, err
+		}
+	}
+
+	return true, nil
+}

vendor/github.com/docker/distribution/registry/storage/vacuum.go

@@ -0,0 +1,67 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/opencontainers/go-digest"
+)
+
+// vacuum contains functions for cleaning up repositories and blobs
+// These functions will only reliably work on strongly consistent
+// storage systems.
+// https://en.wikipedia.org/wiki/Consistency_model
+
+// NewVacuum creates a new Vacuum
+func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum {
+	return Vacuum{
+		ctx:    ctx,
+		driver: driver,
+	}
+}
+
+// Vacuum removes content from the filesystem
+type Vacuum struct {
+	driver driver.StorageDriver
+	ctx    context.Context
+}
+
+// RemoveBlob removes a blob from the filesystem
+func (v Vacuum) RemoveBlob(dgst string) error {
+	d, err := digest.Parse(dgst)
+	if err != nil {
+		return err
+	}
+
+	blobPath, err := pathFor(blobPathSpec{digest: d})
+	if err != nil {
+		return err
+	}
+
+	context.GetLogger(v.ctx).Infof("Deleting blob: %s", blobPath)
+
+	err = v.driver.Delete(v.ctx, blobPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RemoveRepository removes a repository directory from the
+// filesystem
+func (v Vacuum) RemoveRepository(repoName string) error {
+	rootForRepository, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return err
+	}
+	repoDir := path.Join(rootForRepository, repoName)
+	context.GetLogger(v.ctx).Infof("Deleting repo: %s", repoDir)
+	err = v.driver.Delete(v.ctx, repoDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/walk.go

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/docker/distribution/context"
+	storageDriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// ErrSkipDir is used as a return value from onFileFunc to indicate that
+// the directory named in the call is to be skipped. It is not returned
+// as an error by any function.
+var ErrSkipDir = errors.New("skip this directory")
+
+// WalkFn is called once per file by Walk
+// If the returned error is ErrSkipDir and fileInfo refers
+// to a directory, the directory will not be entered and Walk
+// will continue the traversal.  Otherwise Walk will return
+type WalkFn func(fileInfo storageDriver.FileInfo) error
+
+// Walk traverses a filesystem defined within driver, starting
+// from the given path, calling f on each file
+func Walk(ctx context.Context, driver storageDriver.StorageDriver, from string, f WalkFn) error {
+	children, err := driver.List(ctx, from)
+	if err != nil {
+		return err
+	}
+	sort.Stable(sort.StringSlice(children))
+	for _, child := range children {
+		// TODO(stevvooe): Calling driver.Stat for every entry is quite
+		// expensive when running against backends with a slow Stat
+		// implementation, such as s3. This is very likely a serious
+		// performance bottleneck.
+		fileInfo, err := driver.Stat(ctx, child)
+		if err != nil {
+			return err
+		}
+		err = f(fileInfo)
+		skipDir := (err == ErrSkipDir)
+		if err != nil && !skipDir {
+			return err
+		}
+
+		if fileInfo.IsDir() && !skipDir {
+			if err := Walk(ctx, driver, child, f); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// pushError formats an error type given a path and an error
+// and pushes it to a slice of errors
+func pushError(errors []error, path string, err error) []error {
+	return append(errors, fmt.Errorf("%s: %s", path, err))
+}