1
0
Fork 0

Vendor integration dependencies.

This commit is contained in:
Timo Reimann 2017-02-07 22:33:23 +01:00
parent dd5e3fba01
commit 55b57c736b
2451 changed files with 731611 additions and 0 deletions

175
integration/vendor/github.com/docker/docker/image/fs.go generated vendored Normal file
View file

@ -0,0 +1,175 @@
package image
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/pkg/ioutils"
)
// IDWalkFunc is function called by StoreBackend.Walk
type IDWalkFunc func(id ID) error
// StoreBackend provides interface for image.Store persistence
type StoreBackend interface {
Walk(f IDWalkFunc) error
Get(id ID) ([]byte, error)
Set(data []byte) (ID, error)
Delete(id ID) error
SetMetadata(id ID, key string, data []byte) error
GetMetadata(id ID, key string) ([]byte, error)
DeleteMetadata(id ID, key string) error
}
// fs implements StoreBackend using the filesystem.
type fs struct {
sync.RWMutex
root string
}
const (
contentDirName = "content"
metadataDirName = "metadata"
)
// NewFSStoreBackend returns new filesystem based backend for image.Store
func NewFSStoreBackend(root string) (StoreBackend, error) {
return newFSStore(root)
}
func newFSStore(root string) (*fs, error) {
s := &fs{
root: root,
}
if err := os.MkdirAll(filepath.Join(root, contentDirName, string(digest.Canonical)), 0700); err != nil {
return nil, err
}
if err := os.MkdirAll(filepath.Join(root, metadataDirName, string(digest.Canonical)), 0700); err != nil {
return nil, err
}
return s, nil
}
func (s *fs) contentFile(id ID) string {
dgst := digest.Digest(id)
return filepath.Join(s.root, contentDirName, string(dgst.Algorithm()), dgst.Hex())
}
func (s *fs) metadataDir(id ID) string {
dgst := digest.Digest(id)
return filepath.Join(s.root, metadataDirName, string(dgst.Algorithm()), dgst.Hex())
}
// Walk calls the supplied callback for each image ID in the storage backend.
func (s *fs) Walk(f IDWalkFunc) error {
// Only Canonical digest (sha256) is currently supported
s.RLock()
dir, err := ioutil.ReadDir(filepath.Join(s.root, contentDirName, string(digest.Canonical)))
s.RUnlock()
if err != nil {
return err
}
for _, v := range dir {
dgst := digest.NewDigestFromHex(string(digest.Canonical), v.Name())
if err := dgst.Validate(); err != nil {
logrus.Debugf("Skipping invalid digest %s: %s", dgst, err)
continue
}
if err := f(ID(dgst)); err != nil {
return err
}
}
return nil
}
// Get returns the content stored under a given ID.
func (s *fs) Get(id ID) ([]byte, error) {
s.RLock()
defer s.RUnlock()
return s.get(id)
}
func (s *fs) get(id ID) ([]byte, error) {
content, err := ioutil.ReadFile(s.contentFile(id))
if err != nil {
return nil, err
}
// todo: maybe optional
if ID(digest.FromBytes(content)) != id {
return nil, fmt.Errorf("failed to verify image: %v", id)
}
return content, nil
}
// Set stores content under a given ID.
func (s *fs) Set(data []byte) (ID, error) {
s.Lock()
defer s.Unlock()
if len(data) == 0 {
return "", fmt.Errorf("Invalid empty data")
}
id := ID(digest.FromBytes(data))
if err := ioutils.AtomicWriteFile(s.contentFile(id), data, 0600); err != nil {
return "", err
}
return id, nil
}
// Delete removes content and metadata files associated with the ID.
func (s *fs) Delete(id ID) error {
s.Lock()
defer s.Unlock()
if err := os.RemoveAll(s.metadataDir(id)); err != nil {
return err
}
if err := os.Remove(s.contentFile(id)); err != nil {
return err
}
return nil
}
// SetMetadata sets metadata for a given ID. It fails if there's no base file.
func (s *fs) SetMetadata(id ID, key string, data []byte) error {
s.Lock()
defer s.Unlock()
if _, err := s.get(id); err != nil {
return err
}
baseDir := filepath.Join(s.metadataDir(id))
if err := os.MkdirAll(baseDir, 0700); err != nil {
return err
}
return ioutils.AtomicWriteFile(filepath.Join(s.metadataDir(id), key), data, 0600)
}
// GetMetadata returns metadata for a given ID.
func (s *fs) GetMetadata(id ID, key string) ([]byte, error) {
s.RLock()
defer s.RUnlock()
if _, err := s.get(id); err != nil {
return nil, err
}
return ioutil.ReadFile(filepath.Join(s.metadataDir(id), key))
}
// DeleteMetadata removes the metadata associated with an ID.
func (s *fs) DeleteMetadata(id ID, key string) error {
s.Lock()
defer s.Unlock()
return os.RemoveAll(filepath.Join(s.metadataDir(id), key))
}

View file

@ -0,0 +1,140 @@
package image
import (
"encoding/json"
"errors"
"io"
"time"
"github.com/docker/distribution/digest"
"github.com/docker/engine-api/types/container"
)
// ID is the content-addressable ID of an image.
type ID digest.Digest
func (id ID) String() string {
return digest.Digest(id).String()
}
// V1Image stores the V1 image configuration.
type V1Image struct {
// ID a unique 64 character identifier of the image
ID string `json:"id,omitempty"`
// Parent id of the image
Parent string `json:"parent,omitempty"`
// Comment user added comment
Comment string `json:"comment,omitempty"`
// Created timestamp when image was created
Created time.Time `json:"created"`
// Container is the id of the container used to commit
Container string `json:"container,omitempty"`
// ContainerConfig is the configuration of the container that is committed into the image
ContainerConfig container.Config `json:"container_config,omitempty"`
// DockerVersion specifies version on which image is built
DockerVersion string `json:"docker_version,omitempty"`
// Author of the image
Author string `json:"author,omitempty"`
// Config is the configuration of the container received from the client
Config *container.Config `json:"config,omitempty"`
// Architecture is the hardware that the image is build and runs on
Architecture string `json:"architecture,omitempty"`
// OS is the operating system used to build and run the image
OS string `json:"os,omitempty"`
// Size is the total size of the image including all layers it is composed of
Size int64 `json:",omitempty"`
}
// Image stores the image configuration
type Image struct {
V1Image
Parent ID `json:"parent,omitempty"`
RootFS *RootFS `json:"rootfs,omitempty"`
History []History `json:"history,omitempty"`
OSVersion string `json:"os.version,omitempty"`
OSFeatures []string `json:"os.features,omitempty"`
// rawJSON caches the immutable JSON associated with this image.
rawJSON []byte
// computedID is the ID computed from the hash of the image config.
// Not to be confused with the legacy V1 ID in V1Image.
computedID ID
}
// RawJSON returns the immutable JSON associated with the image.
func (img *Image) RawJSON() []byte {
return img.rawJSON
}
// ID returns the image's content-addressable ID.
func (img *Image) ID() ID {
return img.computedID
}
// ImageID stringizes ID.
func (img *Image) ImageID() string {
return string(img.ID())
}
// RunConfig returns the image's container config.
func (img *Image) RunConfig() *container.Config {
return img.Config
}
// MarshalJSON serializes the image to JSON. It sorts the top-level keys so
// that JSON that's been manipulated by a push/pull cycle with a legacy
// registry won't end up with a different key order.
func (img *Image) MarshalJSON() ([]byte, error) {
type MarshalImage Image
pass1, err := json.Marshal(MarshalImage(*img))
if err != nil {
return nil, err
}
var c map[string]*json.RawMessage
if err := json.Unmarshal(pass1, &c); err != nil {
return nil, err
}
return json.Marshal(c)
}
// History stores build commands that were used to create an image
type History struct {
// Created timestamp for build point
Created time.Time `json:"created"`
// Author of the build point
Author string `json:"author,omitempty"`
// CreatedBy keeps the Dockerfile command used while building image.
CreatedBy string `json:"created_by,omitempty"`
// Comment is custom message set by the user when creating the image.
Comment string `json:"comment,omitempty"`
// EmptyLayer is set to true if this history item did not generate a
// layer. Otherwise, the history item is associated with the next
// layer in the RootFS section.
EmptyLayer bool `json:"empty_layer,omitempty"`
}
// Exporter provides interface for exporting and importing images
type Exporter interface {
Load(io.ReadCloser, io.Writer, bool) error
// TODO: Load(net.Context, io.ReadCloser, <- chan StatusMessage) error
Save([]string, io.Writer) error
}
// NewFromJSON creates an Image configuration from json.
func NewFromJSON(src []byte) (*Image, error) {
img := &Image{}
if err := json.Unmarshal(src, img); err != nil {
return nil, err
}
if img.RootFS == nil {
return nil, errors.New("Invalid image JSON, no RootFS key.")
}
img.rawJSON = src
return img, nil
}

View file

@ -0,0 +1,16 @@
package image
import "github.com/docker/docker/layer"
// TypeLayers is used for RootFS.Type for filesystems organized into layers.
const TypeLayers = "layers"
// NewRootFS returns empty RootFS struct
func NewRootFS() *RootFS {
return &RootFS{Type: TypeLayers}
}
// Append appends a new diffID to rootfs
func (r *RootFS) Append(id layer.DiffID) {
r.DiffIDs = append(r.DiffIDs, id)
}

View file

@ -0,0 +1,18 @@
// +build !windows
package image
import "github.com/docker/docker/layer"
// RootFS describes images root filesystem
// This is currently a placeholder that only supports layers. In the future
// this can be made into an interface that supports different implementations.
type RootFS struct {
Type string `json:"type"`
DiffIDs []layer.DiffID `json:"diff_ids,omitempty"`
}
// ChainID returns the ChainID for the top layer in RootFS.
func (r *RootFS) ChainID() layer.ChainID {
return layer.CreateChainID(r.DiffIDs)
}

View file

@ -0,0 +1,48 @@
// +build windows
package image
import (
"crypto/sha512"
"fmt"
"github.com/docker/distribution/digest"
"github.com/docker/docker/layer"
)
// TypeLayersWithBase is used for RootFS.Type for Windows filesystems that have layers and a centrally-stored base layer.
const TypeLayersWithBase = "layers+base"
// RootFS describes images root filesystem
// This is currently a placeholder that only supports layers. In the future
// this can be made into an interface that supports different implementations.
type RootFS struct {
Type string `json:"type"`
DiffIDs []layer.DiffID `json:"diff_ids,omitempty"`
BaseLayer string `json:"base_layer,omitempty"`
}
// BaseLayerID returns the 64 byte hex ID for the baselayer name.
func (r *RootFS) BaseLayerID() string {
if r.Type != TypeLayersWithBase {
panic("tried to get base layer ID without a base layer")
}
baseID := sha512.Sum384([]byte(r.BaseLayer))
return fmt.Sprintf("%x", baseID[:32])
}
// ChainID returns the ChainID for the top layer in RootFS.
func (r *RootFS) ChainID() layer.ChainID {
ids := r.DiffIDs
if r.Type == TypeLayersWithBase {
// Add an extra ID for the base.
baseDiffID := layer.DiffID(digest.FromBytes([]byte(r.BaseLayerID())))
ids = append([]layer.DiffID{baseDiffID}, ids...)
}
return layer.CreateChainID(ids)
}
// NewRootFSWithBaseLayer returns a RootFS struct with a base layer
func NewRootFSWithBaseLayer(baseLayer string) *RootFS {
return &RootFS{Type: TypeLayersWithBase, BaseLayer: baseLayer}
}

View file

@ -0,0 +1,295 @@
package image
import (
"encoding/json"
"errors"
"fmt"
"sync"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/layer"
)
// Store is an interface for creating and accessing images
type Store interface {
Create(config []byte) (ID, error)
Get(id ID) (*Image, error)
Delete(id ID) ([]layer.Metadata, error)
Search(partialID string) (ID, error)
SetParent(id ID, parent ID) error
GetParent(id ID) (ID, error)
Children(id ID) []ID
Map() map[ID]*Image
Heads() map[ID]*Image
}
// LayerGetReleaser is a minimal interface for getting and releasing images.
type LayerGetReleaser interface {
Get(layer.ChainID) (layer.Layer, error)
Release(layer.Layer) ([]layer.Metadata, error)
}
type imageMeta struct {
layer layer.Layer
children map[ID]struct{}
}
type store struct {
sync.Mutex
ls LayerGetReleaser
images map[ID]*imageMeta
fs StoreBackend
digestSet *digest.Set
}
// NewImageStore returns new store object for given layer store
func NewImageStore(fs StoreBackend, ls LayerGetReleaser) (Store, error) {
is := &store{
ls: ls,
images: make(map[ID]*imageMeta),
fs: fs,
digestSet: digest.NewSet(),
}
// load all current images and retain layers
if err := is.restore(); err != nil {
return nil, err
}
return is, nil
}
func (is *store) restore() error {
err := is.fs.Walk(func(id ID) error {
img, err := is.Get(id)
if err != nil {
logrus.Errorf("invalid image %v, %v", id, err)
return nil
}
var l layer.Layer
if chainID := img.RootFS.ChainID(); chainID != "" {
l, err = is.ls.Get(chainID)
if err != nil {
return err
}
}
if err := is.digestSet.Add(digest.Digest(id)); err != nil {
return err
}
imageMeta := &imageMeta{
layer: l,
children: make(map[ID]struct{}),
}
is.images[ID(id)] = imageMeta
return nil
})
if err != nil {
return err
}
// Second pass to fill in children maps
for id := range is.images {
if parent, err := is.GetParent(id); err == nil {
if parentMeta := is.images[parent]; parentMeta != nil {
parentMeta.children[id] = struct{}{}
}
}
}
return nil
}
func (is *store) Create(config []byte) (ID, error) {
var img Image
err := json.Unmarshal(config, &img)
if err != nil {
return "", err
}
// Must reject any config that references diffIDs from the history
// which aren't among the rootfs layers.
rootFSLayers := make(map[layer.DiffID]struct{})
for _, diffID := range img.RootFS.DiffIDs {
rootFSLayers[diffID] = struct{}{}
}
layerCounter := 0
for _, h := range img.History {
if !h.EmptyLayer {
layerCounter++
}
}
if layerCounter > len(img.RootFS.DiffIDs) {
return "", errors.New("too many non-empty layers in History section")
}
dgst, err := is.fs.Set(config)
if err != nil {
return "", err
}
imageID := ID(dgst)
is.Lock()
defer is.Unlock()
if _, exists := is.images[imageID]; exists {
return imageID, nil
}
layerID := img.RootFS.ChainID()
var l layer.Layer
if layerID != "" {
l, err = is.ls.Get(layerID)
if err != nil {
return "", err
}
}
imageMeta := &imageMeta{
layer: l,
children: make(map[ID]struct{}),
}
is.images[imageID] = imageMeta
if err := is.digestSet.Add(digest.Digest(imageID)); err != nil {
delete(is.images, imageID)
return "", err
}
return imageID, nil
}
func (is *store) Search(term string) (ID, error) {
is.Lock()
defer is.Unlock()
dgst, err := is.digestSet.Lookup(term)
if err != nil {
if err == digest.ErrDigestNotFound {
err = fmt.Errorf("No such image: %s", term)
}
return "", err
}
return ID(dgst), nil
}
func (is *store) Get(id ID) (*Image, error) {
// todo: Check if image is in images
// todo: Detect manual insertions and start using them
config, err := is.fs.Get(id)
if err != nil {
return nil, err
}
img, err := NewFromJSON(config)
if err != nil {
return nil, err
}
img.computedID = id
img.Parent, err = is.GetParent(id)
if err != nil {
img.Parent = ""
}
return img, nil
}
func (is *store) Delete(id ID) ([]layer.Metadata, error) {
is.Lock()
defer is.Unlock()
imageMeta := is.images[id]
if imageMeta == nil {
return nil, fmt.Errorf("unrecognized image ID %s", id.String())
}
for id := range imageMeta.children {
is.fs.DeleteMetadata(id, "parent")
}
if parent, err := is.GetParent(id); err == nil && is.images[parent] != nil {
delete(is.images[parent].children, id)
}
if err := is.digestSet.Remove(digest.Digest(id)); err != nil {
logrus.Errorf("error removing %s from digest set: %q", id, err)
}
delete(is.images, id)
is.fs.Delete(id)
if imageMeta.layer != nil {
return is.ls.Release(imageMeta.layer)
}
return nil, nil
}
func (is *store) SetParent(id, parent ID) error {
is.Lock()
defer is.Unlock()
parentMeta := is.images[parent]
if parentMeta == nil {
return fmt.Errorf("unknown parent image ID %s", parent.String())
}
if parent, err := is.GetParent(id); err == nil && is.images[parent] != nil {
delete(is.images[parent].children, id)
}
parentMeta.children[id] = struct{}{}
return is.fs.SetMetadata(id, "parent", []byte(parent))
}
func (is *store) GetParent(id ID) (ID, error) {
d, err := is.fs.GetMetadata(id, "parent")
if err != nil {
return "", err
}
return ID(d), nil // todo: validate?
}
func (is *store) Children(id ID) []ID {
is.Lock()
defer is.Unlock()
return is.children(id)
}
func (is *store) children(id ID) []ID {
var ids []ID
if is.images[id] != nil {
for id := range is.images[id].children {
ids = append(ids, id)
}
}
return ids
}
func (is *store) Heads() map[ID]*Image {
return is.imagesMap(false)
}
func (is *store) Map() map[ID]*Image {
return is.imagesMap(true)
}
func (is *store) imagesMap(all bool) map[ID]*Image {
is.Lock()
defer is.Unlock()
images := make(map[ID]*Image)
for id := range is.images {
if !all && len(is.children(id)) > 0 {
continue
}
img, err := is.Get(id)
if err != nil {
logrus.Errorf("invalid image access: %q, error: %q", id, err)
continue
}
images[id] = img
}
return images
}

View file

@ -0,0 +1,156 @@
package v1
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"strings"
"github.com/Sirupsen/logrus"
"github.com/docker/distribution/digest"
"github.com/docker/docker/image"
"github.com/docker/docker/layer"
"github.com/docker/engine-api/types/versions"
)
var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
// noFallbackMinVersion is the minimum version for which v1compatibility
// information will not be marshaled through the Image struct to remove
// blank fields.
var noFallbackMinVersion = "1.8.3"
// HistoryFromConfig creates a History struct from v1 configuration JSON
func HistoryFromConfig(imageJSON []byte, emptyLayer bool) (image.History, error) {
h := image.History{}
var v1Image image.V1Image
if err := json.Unmarshal(imageJSON, &v1Image); err != nil {
return h, err
}
return image.History{
Author: v1Image.Author,
Created: v1Image.Created,
CreatedBy: strings.Join(v1Image.ContainerConfig.Cmd, " "),
Comment: v1Image.Comment,
EmptyLayer: emptyLayer,
}, nil
}
// CreateID creates an ID from v1 image, layerID and parent ID.
// Used for backwards compatibility with old clients.
func CreateID(v1Image image.V1Image, layerID layer.ChainID, parent digest.Digest) (digest.Digest, error) {
v1Image.ID = ""
v1JSON, err := json.Marshal(v1Image)
if err != nil {
return "", err
}
var config map[string]*json.RawMessage
if err := json.Unmarshal(v1JSON, &config); err != nil {
return "", err
}
// FIXME: note that this is slightly incompatible with RootFS logic
config["layer_id"] = rawJSON(layerID)
if parent != "" {
config["parent"] = rawJSON(parent)
}
configJSON, err := json.Marshal(config)
if err != nil {
return "", err
}
logrus.Debugf("CreateV1ID %s", configJSON)
return digest.FromBytes(configJSON), nil
}
// MakeConfigFromV1Config creates an image config from the legacy V1 config format.
func MakeConfigFromV1Config(imageJSON []byte, rootfs *image.RootFS, history []image.History) ([]byte, error) {
var dver struct {
DockerVersion string `json:"docker_version"`
}
if err := json.Unmarshal(imageJSON, &dver); err != nil {
return nil, err
}
useFallback := versions.LessThan(dver.DockerVersion, noFallbackMinVersion)
if useFallback {
var v1Image image.V1Image
err := json.Unmarshal(imageJSON, &v1Image)
if err != nil {
return nil, err
}
imageJSON, err = json.Marshal(v1Image)
if err != nil {
return nil, err
}
}
var c map[string]*json.RawMessage
if err := json.Unmarshal(imageJSON, &c); err != nil {
return nil, err
}
delete(c, "id")
delete(c, "parent")
delete(c, "Size") // Size is calculated from data on disk and is inconsistent
delete(c, "parent_id")
delete(c, "layer_id")
delete(c, "throwaway")
c["rootfs"] = rawJSON(rootfs)
c["history"] = rawJSON(history)
return json.Marshal(c)
}
// MakeV1ConfigFromConfig creates an legacy V1 image config from an Image struct
func MakeV1ConfigFromConfig(img *image.Image, v1ID, parentV1ID string, throwaway bool) ([]byte, error) {
// Top-level v1compatibility string should be a modified version of the
// image config.
var configAsMap map[string]*json.RawMessage
if err := json.Unmarshal(img.RawJSON(), &configAsMap); err != nil {
return nil, err
}
// Delete fields that didn't exist in old manifest
imageType := reflect.TypeOf(img).Elem()
for i := 0; i < imageType.NumField(); i++ {
f := imageType.Field(i)
jsonName := strings.Split(f.Tag.Get("json"), ",")[0]
// Parent is handled specially below.
if jsonName != "" && jsonName != "parent" {
delete(configAsMap, jsonName)
}
}
configAsMap["id"] = rawJSON(v1ID)
if parentV1ID != "" {
configAsMap["parent"] = rawJSON(parentV1ID)
}
if throwaway {
configAsMap["throwaway"] = rawJSON(true)
}
return json.Marshal(configAsMap)
}
func rawJSON(value interface{}) *json.RawMessage {
jsonval, err := json.Marshal(value)
if err != nil {
return nil
}
return (*json.RawMessage)(&jsonval)
}
// ValidateID checks whether an ID string is a valid image ID.
func ValidateID(id string) error {
if ok := validHex.MatchString(id); !ok {
return fmt.Errorf("image ID %q is invalid", id)
}
return nil
}