1
0
Fork 0

Custom resource definition

Co-authored-by: Mathieu Lonjaret <mathieu.lonjaret@gmail.com>
This commit is contained in:
Ludovic Fernandez 2019-03-14 15:56:06 +01:00 committed by Traefiker Bot
parent cfaf47c8a2
commit 4c060a78cc
1348 changed files with 92364 additions and 55766 deletions

View file

@ -0,0 +1 @@
go-to-protobuf

View file

@ -0,0 +1,4 @@
approvers:
- smarterclayton
reviewers:
- smarterclayton

View file

@ -0,0 +1,39 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package main
import (
goflag "flag"
flag "github.com/spf13/pflag"
"k8s.io/code-generator/cmd/go-to-protobuf/protobuf"
)
var g = protobuf.New()
func init() {
g.BindFlags(flag.CommandLine)
goflag.Set("logtostderr", "true")
flag.CommandLine.AddGoFlagSet(goflag.CommandLine)
}
func main() {
flag.Parse()
protobuf.Run(g)
}

View file

@ -0,0 +1,349 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// go-to-protobuf generates a Protobuf IDL from a Go struct, respecting any
// existing IDL tags on the Go struct.
package protobuf
import (
"bytes"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"k8s.io/code-generator/pkg/util"
"k8s.io/gengo/args"
"k8s.io/gengo/generator"
"k8s.io/gengo/namer"
"k8s.io/gengo/parser"
"k8s.io/gengo/types"
flag "github.com/spf13/pflag"
)
type Generator struct {
Common args.GeneratorArgs
APIMachineryPackages string
Packages string
OutputBase string
VendorOutputBase string
ProtoImport []string
Conditional string
Clean bool
OnlyIDL bool
KeepGogoproto bool
SkipGeneratedRewrite bool
DropEmbeddedFields string
}
func New() *Generator {
sourceTree := args.DefaultSourceTree()
common := args.GeneratorArgs{
OutputBase: sourceTree,
GoHeaderFilePath: filepath.Join(sourceTree, util.BoilerplatePath()),
}
defaultProtoImport := filepath.Join(sourceTree, "k8s.io", "kubernetes", "vendor", "github.com", "gogo", "protobuf", "protobuf")
cwd, err := os.Getwd()
if err != nil {
log.Fatalf("Cannot get current directory.")
}
return &Generator{
Common: common,
OutputBase: sourceTree,
VendorOutputBase: filepath.Join(cwd, "vendor"),
ProtoImport: []string{defaultProtoImport},
APIMachineryPackages: strings.Join([]string{
`+k8s.io/apimachinery/pkg/util/intstr`,
`+k8s.io/apimachinery/pkg/api/resource`,
`+k8s.io/apimachinery/pkg/runtime/schema`,
`+k8s.io/apimachinery/pkg/runtime`,
`k8s.io/apimachinery/pkg/apis/meta/v1`,
`k8s.io/apimachinery/pkg/apis/meta/v1beta1`,
`k8s.io/apimachinery/pkg/apis/testapigroup/v1`,
}, ","),
Packages: "",
DropEmbeddedFields: "k8s.io/apimachinery/pkg/apis/meta/v1.TypeMeta",
}
}
func (g *Generator) BindFlags(flag *flag.FlagSet) {
flag.StringVarP(&g.Common.GoHeaderFilePath, "go-header-file", "h", g.Common.GoHeaderFilePath, "File containing boilerplate header text. The string YEAR will be replaced with the current 4-digit year.")
flag.BoolVar(&g.Common.VerifyOnly, "verify-only", g.Common.VerifyOnly, "If true, only verify existing output, do not write anything.")
flag.StringVarP(&g.Packages, "packages", "p", g.Packages, "comma-separated list of directories to get input types from. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
flag.StringVar(&g.APIMachineryPackages, "apimachinery-packages", g.APIMachineryPackages, "comma-separated list of directories to get apimachinery input types from which are needed by any API. Directories prefixed with '-' are not generated, directories prefixed with '+' only create types with explicit IDL instructions.")
flag.StringVarP(&g.OutputBase, "output-base", "o", g.OutputBase, "Output base; defaults to $GOPATH/src/")
flag.StringVar(&g.VendorOutputBase, "vendor-output-base", g.VendorOutputBase, "The vendor/ directory to look for packages in; defaults to $PWD/vendor/.")
flag.StringSliceVar(&g.ProtoImport, "proto-import", g.ProtoImport, "The search path for the core protobuf .protos, required; defaults $GOPATH/src/k8s.io/kubernetes/vendor/github.com/gogo/protobuf/protobuf.")
flag.StringVar(&g.Conditional, "conditional", g.Conditional, "An optional Golang build tag condition to add to the generated Go code")
flag.BoolVar(&g.Clean, "clean", g.Clean, "If true, remove all generated files for the specified Packages.")
flag.BoolVar(&g.OnlyIDL, "only-idl", g.OnlyIDL, "If true, only generate the IDL for each package.")
flag.BoolVar(&g.KeepGogoproto, "keep-gogoproto", g.KeepGogoproto, "If true, the generated IDL will contain gogoprotobuf extensions which are normally removed")
flag.BoolVar(&g.SkipGeneratedRewrite, "skip-generated-rewrite", g.SkipGeneratedRewrite, "If true, skip fixing up the generated.pb.go file (debugging only).")
flag.StringVar(&g.DropEmbeddedFields, "drop-embedded-fields", g.DropEmbeddedFields, "Comma-delimited list of embedded Go types to omit from generated protobufs")
}
func Run(g *Generator) {
if g.Common.VerifyOnly {
g.OnlyIDL = true
g.Clean = false
}
b := parser.New()
b.AddBuildTags("proto")
omitTypes := map[types.Name]struct{}{}
for _, t := range strings.Split(g.DropEmbeddedFields, ",") {
name := types.Name{}
if i := strings.LastIndex(t, "."); i != -1 {
name.Package, name.Name = t[:i], t[i+1:]
} else {
name.Name = t
}
if len(name.Name) == 0 {
log.Fatalf("--drop-embedded-types requires names in the form of [GOPACKAGE.]TYPENAME: %v", t)
}
omitTypes[name] = struct{}{}
}
boilerplate, err := g.Common.LoadGoBoilerplate()
if err != nil {
log.Fatalf("Failed loading boilerplate (consider using the go-header-file flag): %v", err)
}
protobufNames := NewProtobufNamer()
outputPackages := generator.Packages{}
nonOutputPackages := map[string]struct{}{}
var packages []string
if len(g.APIMachineryPackages) != 0 {
packages = append(packages, strings.Split(g.APIMachineryPackages, ",")...)
}
if len(g.Packages) != 0 {
packages = append(packages, strings.Split(g.Packages, ",")...)
}
if len(packages) == 0 {
log.Fatalf("Both apimachinery-packages and packages are empty. At least one package must be specified.")
}
for _, d := range packages {
generateAllTypes, outputPackage := true, true
switch {
case strings.HasPrefix(d, "+"):
d = d[1:]
generateAllTypes = false
case strings.HasPrefix(d, "-"):
d = d[1:]
outputPackage = false
}
name := protoSafePackage(d)
parts := strings.SplitN(d, "=", 2)
if len(parts) > 1 {
d = parts[0]
name = parts[1]
}
p := newProtobufPackage(d, name, generateAllTypes, omitTypes)
header := append([]byte{}, boilerplate...)
header = append(header, p.HeaderText...)
p.HeaderText = header
protobufNames.Add(p)
if outputPackage {
outputPackages = append(outputPackages, p)
} else {
nonOutputPackages[name] = struct{}{}
}
}
if !g.Common.VerifyOnly {
for _, p := range outputPackages {
if err := p.(*protobufPackage).Clean(g.OutputBase); err != nil {
log.Fatalf("Unable to clean package %s: %v", p.Name(), err)
}
}
}
if g.Clean {
return
}
for _, p := range protobufNames.List() {
if err := b.AddDir(p.Path()); err != nil {
log.Fatalf("Unable to add directory %q: %v", p.Path(), err)
}
}
c, err := generator.NewContext(
b,
namer.NameSystems{
"public": namer.NewPublicNamer(3),
"proto": protobufNames,
},
"public",
)
if err != nil {
log.Fatalf("Failed making a context: %v", err)
}
c.Verify = g.Common.VerifyOnly
c.FileTypes["protoidl"] = NewProtoFile()
var vendoredOutputPackages, localOutputPackages generator.Packages
for _, p := range protobufNames.packages {
if _, ok := nonOutputPackages[p.Name()]; ok {
// if we're not outputting the package, don't include it in either package list
continue
}
p.Vendored = strings.Contains(c.Universe[p.PackagePath].SourcePath, "/vendor/")
if p.Vendored {
vendoredOutputPackages = append(vendoredOutputPackages, p)
} else {
localOutputPackages = append(localOutputPackages, p)
}
}
if err := protobufNames.AssignTypesToPackages(c); err != nil {
log.Fatalf("Failed to identify Common types: %v", err)
}
if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
log.Fatalf("Failed executing vendor generator: %v", err)
}
if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
log.Fatalf("Failed executing local generator: %v", err)
}
if g.OnlyIDL {
return
}
if _, err := exec.LookPath("protoc"); err != nil {
log.Fatalf("Unable to find 'protoc': %v", err)
}
searchArgs := []string{"-I", ".", "-I", g.OutputBase}
if len(g.ProtoImport) != 0 {
for _, s := range g.ProtoImport {
searchArgs = append(searchArgs, "-I", s)
}
}
args := append(searchArgs, fmt.Sprintf("--gogo_out=%s", g.OutputBase))
buf := &bytes.Buffer{}
if len(g.Conditional) > 0 {
fmt.Fprintf(buf, "// +build %s\n\n", g.Conditional)
}
buf.Write(boilerplate)
for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)
path := filepath.Join(g.OutputBase, p.ImportPath())
outputPath := filepath.Join(g.OutputBase, p.OutputPath())
if p.Vendored {
path = filepath.Join(g.VendorOutputBase, p.ImportPath())
outputPath = filepath.Join(g.VendorOutputBase, p.OutputPath())
}
// generate the gogoprotobuf protoc
cmd := exec.Command("protoc", append(args, path)...)
out, err := cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to generate protoc on %s: %v", p.PackageName, err)
}
if g.SkipGeneratedRewrite {
continue
}
// alter the generated protobuf file to remove the generated types (but leave the serializers) and rewrite the
// package statement to match the desired package name
if err := RewriteGeneratedGogoProtobufFile(outputPath, p.ExtractGeneratedType, p.OptionalTypeName, buf.Bytes()); err != nil {
log.Fatalf("Unable to rewrite generated %s: %v", outputPath, err)
}
// sort imports
cmd = exec.Command("goimports", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to rewrite imports for %s: %v", p.PackageName, err)
}
// format and simplify the generated file
cmd = exec.Command("gofmt", "-s", "-w", outputPath)
out, err = cmd.CombinedOutput()
if len(out) > 0 {
log.Printf(string(out))
}
if err != nil {
log.Println(strings.Join(cmd.Args, " "))
log.Fatalf("Unable to apply gofmt for %s: %v", p.PackageName, err)
}
}
if g.SkipGeneratedRewrite {
return
}
if !g.KeepGogoproto {
// generate, but do so without gogoprotobuf extensions
for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)
p.OmitGogo = true
}
if err := c.ExecutePackages(g.VendorOutputBase, vendoredOutputPackages); err != nil {
log.Fatalf("Failed executing vendor generator: %v", err)
}
if err := c.ExecutePackages(g.OutputBase, localOutputPackages); err != nil {
log.Fatalf("Failed executing local generator: %v", err)
}
}
for _, outputPackage := range outputPackages {
p := outputPackage.(*protobufPackage)
if len(p.StructTags) == 0 {
continue
}
pattern := filepath.Join(g.OutputBase, p.PackagePath, "*.go")
if p.Vendored {
pattern = filepath.Join(g.VendorOutputBase, p.PackagePath, "*.go")
}
files, err := filepath.Glob(pattern)
if err != nil {
log.Fatalf("Can't glob pattern %q: %v", pattern, err)
}
for _, s := range files {
if strings.HasSuffix(s, "_test.go") {
continue
}
if err := RewriteTypesWithProtobufStructTags(s, p.StructTags); err != nil {
log.Fatalf("Unable to rewrite with struct tags %s: %v", s, err)
}
}
}
}

View file

@ -0,0 +1,769 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"fmt"
"io"
"log"
"reflect"
"sort"
"strconv"
"strings"
"github.com/golang/glog"
"k8s.io/gengo/generator"
"k8s.io/gengo/namer"
"k8s.io/gengo/types"
)
// genProtoIDL produces a .proto IDL.
type genProtoIDL struct {
generator.DefaultGen
localPackage types.Name
localGoPackage types.Name
imports namer.ImportTracker
generateAll bool
omitGogo bool
omitFieldTypes map[types.Name]struct{}
}
func (g *genProtoIDL) PackageVars(c *generator.Context) []string {
if g.omitGogo {
return []string{
fmt.Sprintf("option go_package = %q;", g.localGoPackage.Name),
}
}
return []string{
"option (gogoproto.marshaler_all) = true;",
"option (gogoproto.stable_marshaler_all) = true;",
"option (gogoproto.sizer_all) = true;",
"option (gogoproto.goproto_stringer_all) = false;",
"option (gogoproto.stringer_all) = true;",
"option (gogoproto.unmarshaler_all) = true;",
"option (gogoproto.goproto_unrecognized_all) = false;",
"option (gogoproto.goproto_enum_prefix_all) = false;",
"option (gogoproto.goproto_getters_all) = false;",
fmt.Sprintf("option go_package = %q;", g.localGoPackage.Name),
}
}
func (g *genProtoIDL) Filename() string { return g.OptionalName + ".proto" }
func (g *genProtoIDL) FileType() string { return "protoidl" }
func (g *genProtoIDL) Namers(c *generator.Context) namer.NameSystems {
return namer.NameSystems{
// The local namer returns the correct protobuf name for a proto type
// in the context of a package
"local": localNamer{g.localPackage},
}
}
// Filter ignores types that are identified as not exportable.
func (g *genProtoIDL) Filter(c *generator.Context, t *types.Type) bool {
tagVals := types.ExtractCommentTags("+", t.CommentLines)["protobuf"]
if tagVals != nil {
if tagVals[0] == "false" {
// Type specified "false".
return false
}
if tagVals[0] == "true" {
// Type specified "true".
return true
}
glog.Fatalf(`Comment tag "protobuf" must be true or false, found: %q`, tagVals[0])
}
if !g.generateAll {
// We're not generating everything.
return false
}
seen := map[*types.Type]bool{}
ok := isProtoable(seen, t)
return ok
}
func isProtoable(seen map[*types.Type]bool, t *types.Type) bool {
if seen[t] {
// be optimistic in the case of type cycles.
return true
}
seen[t] = true
switch t.Kind {
case types.Builtin:
return true
case types.Alias:
return isProtoable(seen, t.Underlying)
case types.Slice, types.Pointer:
return isProtoable(seen, t.Elem)
case types.Map:
return isProtoable(seen, t.Key) && isProtoable(seen, t.Elem)
case types.Struct:
if len(t.Members) == 0 {
return true
}
for _, m := range t.Members {
if isProtoable(seen, m.Type) {
return true
}
}
return false
case types.Func, types.Chan:
return false
case types.DeclarationOf, types.Unknown, types.Unsupported:
return false
case types.Interface:
return false
default:
log.Printf("WARNING: type %q is not portable: %s", t.Kind, t.Name)
return false
}
}
// isOptionalAlias should return true if the specified type has an underlying type
// (is an alias) of a map or slice and has the comment tag protobuf.nullable=true,
// indicating that the type should be nullable in protobuf.
func isOptionalAlias(t *types.Type) bool {
if t.Underlying == nil || (t.Underlying.Kind != types.Map && t.Underlying.Kind != types.Slice) {
return false
}
if extractBoolTagOrDie("protobuf.nullable", t.CommentLines) == false {
return false
}
return true
}
func (g *genProtoIDL) Imports(c *generator.Context) (imports []string) {
lines := []string{}
// TODO: this could be expressed more cleanly
for _, line := range g.imports.ImportLines() {
if g.omitGogo && line == "github.com/gogo/protobuf/gogoproto/gogo.proto" {
continue
}
lines = append(lines, line)
}
return lines
}
// GenerateType makes the body of a file implementing a set for type t.
func (g *genProtoIDL) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
sw := generator.NewSnippetWriter(w, c, "$", "$")
b := bodyGen{
locator: &protobufLocator{
namer: c.Namers["proto"].(ProtobufFromGoNamer),
tracker: g.imports,
universe: c.Universe,
localGoPackage: g.localGoPackage.Package,
},
localPackage: g.localPackage,
omitGogo: g.omitGogo,
omitFieldTypes: g.omitFieldTypes,
t: t,
}
switch t.Kind {
case types.Alias:
return b.doAlias(sw)
case types.Struct:
return b.doStruct(sw)
default:
return b.unknown(sw)
}
}
// ProtobufFromGoNamer finds the protobuf name of a type (and its package, and
// the package path) from its Go name.
type ProtobufFromGoNamer interface {
GoNameToProtoName(name types.Name) types.Name
}
type ProtobufLocator interface {
ProtoTypeFor(t *types.Type) (*types.Type, error)
GoTypeForName(name types.Name) *types.Type
CastTypeName(name types.Name) string
}
type protobufLocator struct {
namer ProtobufFromGoNamer
tracker namer.ImportTracker
universe types.Universe
localGoPackage string
}
// CastTypeName returns the cast type name of a Go type
// TODO: delegate to a new localgo namer?
func (p protobufLocator) CastTypeName(name types.Name) string {
if name.Package == p.localGoPackage {
return name.Name
}
return name.String()
}
func (p protobufLocator) GoTypeForName(name types.Name) *types.Type {
if len(name.Package) == 0 {
name.Package = p.localGoPackage
}
return p.universe.Type(name)
}
// ProtoTypeFor locates a Protobuf type for the provided Go type (if possible).
func (p protobufLocator) ProtoTypeFor(t *types.Type) (*types.Type, error) {
switch {
// we've already converted the type, or it's a map
case t.Kind == types.Protobuf || t.Kind == types.Map:
p.tracker.AddType(t)
return t, nil
}
// it's a fundamental type
if t, ok := isFundamentalProtoType(t); ok {
p.tracker.AddType(t)
return t, nil
}
// it's a message
if t.Kind == types.Struct || isOptionalAlias(t) {
t := &types.Type{
Name: p.namer.GoNameToProtoName(t.Name),
Kind: types.Protobuf,
CommentLines: t.CommentLines,
}
p.tracker.AddType(t)
return t, nil
}
return nil, errUnrecognizedType
}
type bodyGen struct {
locator ProtobufLocator
localPackage types.Name
omitGogo bool
omitFieldTypes map[types.Name]struct{}
t *types.Type
}
func (b bodyGen) unknown(sw *generator.SnippetWriter) error {
return fmt.Errorf("not sure how to generate: %#v", b.t)
}
func (b bodyGen) doAlias(sw *generator.SnippetWriter) error {
if !isOptionalAlias(b.t) {
return nil
}
var kind string
switch b.t.Underlying.Kind {
case types.Map:
kind = "map"
default:
kind = "slice"
}
optional := &types.Type{
Name: b.t.Name,
Kind: types.Struct,
CommentLines: b.t.CommentLines,
SecondClosestCommentLines: b.t.SecondClosestCommentLines,
Members: []types.Member{
{
Name: "Items",
CommentLines: []string{fmt.Sprintf("items, if empty, will result in an empty %s\n", kind)},
Type: b.t.Underlying,
},
},
}
nested := b
nested.t = optional
return nested.doStruct(sw)
}
func (b bodyGen) doStruct(sw *generator.SnippetWriter) error {
if len(b.t.Name.Name) == 0 {
return nil
}
if namer.IsPrivateGoName(b.t.Name.Name) {
return nil
}
var alias *types.Type
var fields []protoField
options := []string{}
allOptions := types.ExtractCommentTags("+", b.t.CommentLines)
for k, v := range allOptions {
switch {
case strings.HasPrefix(k, "protobuf.options."):
key := strings.TrimPrefix(k, "protobuf.options.")
switch key {
case "marshal":
if v[0] == "false" {
if !b.omitGogo {
options = append(options,
"(gogoproto.marshaler) = false",
"(gogoproto.unmarshaler) = false",
"(gogoproto.sizer) = false",
)
}
}
default:
if !b.omitGogo || !strings.HasPrefix(key, "(gogoproto.") {
if key == "(gogoproto.goproto_stringer)" && v[0] == "false" {
options = append(options, "(gogoproto.stringer) = false")
}
options = append(options, fmt.Sprintf("%s = %s", key, v[0]))
}
}
// protobuf.as allows a type to have the same message contents as another Go type
case k == "protobuf.as":
fields = nil
if alias = b.locator.GoTypeForName(types.Name{Name: v[0]}); alias == nil {
return fmt.Errorf("type %v references alias %q which does not exist", b.t, v[0])
}
// protobuf.embed instructs the generator to use the named type in this package
// as an embedded message.
case k == "protobuf.embed":
fields = []protoField{
{
Tag: 1,
Name: v[0],
Type: &types.Type{
Name: types.Name{
Name: v[0],
Package: b.localPackage.Package,
Path: b.localPackage.Path,
},
},
},
}
}
}
if alias == nil {
alias = b.t
}
// If we don't explicitly embed anything, generate fields by traversing fields.
if fields == nil {
memberFields, err := membersToFields(b.locator, alias, b.localPackage, b.omitFieldTypes)
if err != nil {
return fmt.Errorf("type %v cannot be converted to protobuf: %v", b.t, err)
}
fields = memberFields
}
out := sw.Out()
genComment(out, b.t.CommentLines, "")
sw.Do(`message $.Name.Name$ {
`, b.t)
if len(options) > 0 {
sort.Sort(sort.StringSlice(options))
for _, s := range options {
fmt.Fprintf(out, " option %s;\n", s)
}
fmt.Fprintln(out)
}
for i, field := range fields {
genComment(out, field.CommentLines, " ")
fmt.Fprintf(out, " ")
switch {
case field.Map:
case field.Repeated:
fmt.Fprintf(out, "repeated ")
case field.Required:
fmt.Fprintf(out, "required ")
default:
fmt.Fprintf(out, "optional ")
}
sw.Do(`$.Type|local$ $.Name$ = $.Tag$`, field)
if len(field.Extras) > 0 {
extras := []string{}
for k, v := range field.Extras {
if b.omitGogo && strings.HasPrefix(k, "(gogoproto.") {
continue
}
extras = append(extras, fmt.Sprintf("%s = %s", k, v))
}
sort.Sort(sort.StringSlice(extras))
if len(extras) > 0 {
fmt.Fprintf(out, " [")
fmt.Fprint(out, strings.Join(extras, ", "))
fmt.Fprintf(out, "]")
}
}
fmt.Fprintf(out, ";\n")
if i != len(fields)-1 {
fmt.Fprintf(out, "\n")
}
}
fmt.Fprintf(out, "}\n\n")
return nil
}
type protoField struct {
LocalPackage types.Name
Tag int
Name string
Type *types.Type
Map bool
Repeated bool
Optional bool
Required bool
Nullable bool
Extras map[string]string
CommentLines []string
}
var (
errUnrecognizedType = fmt.Errorf("did not recognize the provided type")
)
func isFundamentalProtoType(t *types.Type) (*types.Type, bool) {
// TODO: when we enable proto3, also include other fundamental types in the google.protobuf package
// switch {
// case t.Kind == types.Struct && t.Name == types.Name{Package: "time", Name: "Time"}:
// return &types.Type{
// Kind: types.Protobuf,
// Name: types.Name{Path: "google/protobuf/timestamp.proto", Package: "google.protobuf", Name: "Timestamp"},
// }, true
// }
switch t.Kind {
case types.Slice:
if t.Elem.Name.Name == "byte" && len(t.Elem.Name.Package) == 0 {
return &types.Type{Name: types.Name{Name: "bytes"}, Kind: types.Protobuf}, true
}
case types.Builtin:
switch t.Name.Name {
case "string", "uint32", "int32", "uint64", "int64", "bool":
return &types.Type{Name: types.Name{Name: t.Name.Name}, Kind: types.Protobuf}, true
case "int":
return &types.Type{Name: types.Name{Name: "int64"}, Kind: types.Protobuf}, true
case "uint":
return &types.Type{Name: types.Name{Name: "uint64"}, Kind: types.Protobuf}, true
case "float64", "float":
return &types.Type{Name: types.Name{Name: "double"}, Kind: types.Protobuf}, true
case "float32":
return &types.Type{Name: types.Name{Name: "float"}, Kind: types.Protobuf}, true
case "uintptr":
return &types.Type{Name: types.Name{Name: "uint64"}, Kind: types.Protobuf}, true
}
// TODO: complex?
}
return t, false
}
func memberTypeToProtobufField(locator ProtobufLocator, field *protoField, t *types.Type) error {
var err error
switch t.Kind {
case types.Protobuf:
field.Type, err = locator.ProtoTypeFor(t)
case types.Builtin:
field.Type, err = locator.ProtoTypeFor(t)
case types.Map:
valueField := &protoField{}
if err := memberTypeToProtobufField(locator, valueField, t.Elem); err != nil {
return err
}
keyField := &protoField{}
if err := memberTypeToProtobufField(locator, keyField, t.Key); err != nil {
return err
}
// All other protobuf types have kind types.Protobuf, so setting types.Map
// here would be very misleading.
field.Type = &types.Type{
Kind: types.Protobuf,
Key: keyField.Type,
Elem: valueField.Type,
}
if !strings.HasPrefix(t.Name.Name, "map[") {
field.Extras["(gogoproto.casttype)"] = strconv.Quote(locator.CastTypeName(t.Name))
}
if k, ok := keyField.Extras["(gogoproto.casttype)"]; ok {
field.Extras["(gogoproto.castkey)"] = k
}
if v, ok := valueField.Extras["(gogoproto.casttype)"]; ok {
field.Extras["(gogoproto.castvalue)"] = v
}
field.Map = true
case types.Pointer:
if err := memberTypeToProtobufField(locator, field, t.Elem); err != nil {
return err
}
field.Nullable = true
case types.Alias:
if isOptionalAlias(t) {
field.Type, err = locator.ProtoTypeFor(t)
field.Nullable = true
} else {
if err := memberTypeToProtobufField(locator, field, t.Underlying); err != nil {
log.Printf("failed to alias: %s %s: err %v", t.Name, t.Underlying.Name, err)
return err
}
// If this is not an alias to a slice, cast to the alias
if !field.Repeated {
if field.Extras == nil {
field.Extras = make(map[string]string)
}
field.Extras["(gogoproto.casttype)"] = strconv.Quote(locator.CastTypeName(t.Name))
}
}
case types.Slice:
if t.Elem.Name.Name == "byte" && len(t.Elem.Name.Package) == 0 {
field.Type = &types.Type{Name: types.Name{Name: "bytes"}, Kind: types.Protobuf}
return nil
}
if err := memberTypeToProtobufField(locator, field, t.Elem); err != nil {
return err
}
field.Repeated = true
case types.Struct:
if len(t.Name.Name) == 0 {
return errUnrecognizedType
}
field.Type, err = locator.ProtoTypeFor(t)
field.Nullable = false
default:
return errUnrecognizedType
}
return err
}
// protobufTagToField extracts information from an existing protobuf tag
func protobufTagToField(tag string, field *protoField, m types.Member, t *types.Type, localPackage types.Name) error {
if len(tag) == 0 || tag == "-" {
return nil
}
// protobuf:"bytes,3,opt,name=Id,customtype=github.com/gogo/protobuf/test.Uuid"
parts := strings.Split(tag, ",")
if len(parts) < 3 {
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, not enough segments\n", m.Name, t.Name)
}
protoTag, err := strconv.Atoi(parts[1])
if err != nil {
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, field ID is %q which is not an integer: %v\n", m.Name, t.Name, parts[1], err)
}
field.Tag = protoTag
// In general there is doesn't make sense to parse the protobuf tags to get the type,
// as all auto-generated once will have wire type "bytes", "varint" or "fixed64".
// However, sometimes we explicitly set them to have a custom serialization, e.g.:
// type Time struct {
// time.Time `protobuf:"Timestamp,1,req,name=time"`
// }
// to force the generator to use a given type (that we manually wrote serialization &
// deserialization methods for).
switch parts[0] {
case "varint", "fixed32", "fixed64", "bytes", "group":
default:
name := types.Name{}
if last := strings.LastIndex(parts[0], "."); last != -1 {
prefix := parts[0][:last]
name = types.Name{
Name: parts[0][last+1:],
Package: prefix,
Path: strings.Replace(prefix, ".", "/", -1),
}
} else {
name = types.Name{
Name: parts[0],
Package: localPackage.Package,
Path: localPackage.Path,
}
}
field.Type = &types.Type{
Name: name,
Kind: types.Protobuf,
}
}
protoExtra := make(map[string]string)
for i, extra := range parts[3:] {
parts := strings.SplitN(extra, "=", 2)
if len(parts) != 2 {
return fmt.Errorf("member %q of %q malformed 'protobuf' tag, tag %d should be key=value, got %q\n", m.Name, t.Name, i+4, extra)
}
switch parts[0] {
case "name":
protoExtra[parts[0]] = parts[1]
case "casttype", "castkey", "castvalue":
parts[0] = fmt.Sprintf("(gogoproto.%s)", parts[0])
protoExtra[parts[0]] = strconv.Quote(parts[1])
}
}
field.Extras = protoExtra
if name, ok := protoExtra["name"]; ok {
field.Name = name
delete(protoExtra, "name")
}
return nil
}
func membersToFields(locator ProtobufLocator, t *types.Type, localPackage types.Name, omitFieldTypes map[types.Name]struct{}) ([]protoField, error) {
fields := []protoField{}
for _, m := range t.Members {
if namer.IsPrivateGoName(m.Name) {
// skip private fields
continue
}
if _, ok := omitFieldTypes[types.Name{Name: m.Type.Name.Name, Package: m.Type.Name.Package}]; ok {
continue
}
tags := reflect.StructTag(m.Tags)
field := protoField{
LocalPackage: localPackage,
Tag: -1,
Extras: make(map[string]string),
}
protobufTag := tags.Get("protobuf")
if protobufTag == "-" {
continue
}
if err := protobufTagToField(protobufTag, &field, m, t, localPackage); err != nil {
return nil, err
}
// extract information from JSON field tag
if tag := tags.Get("json"); len(tag) > 0 {
parts := strings.Split(tag, ",")
if len(field.Name) == 0 && len(parts[0]) != 0 {
field.Name = parts[0]
}
if field.Tag == -1 && field.Name == "-" {
continue
}
}
if field.Type == nil {
if err := memberTypeToProtobufField(locator, &field, m.Type); err != nil {
return nil, fmt.Errorf("unable to embed type %q as field %q in %q: %v", m.Type, field.Name, t.Name, err)
}
}
if len(field.Name) == 0 {
field.Name = namer.IL(m.Name)
}
if field.Map && field.Repeated {
// maps cannot be repeated
field.Repeated = false
field.Nullable = true
}
if !field.Nullable {
field.Extras["(gogoproto.nullable)"] = "false"
}
if (field.Type.Name.Name == "bytes" && field.Type.Name.Package == "") || (field.Repeated && field.Type.Name.Package == "" && namer.IsPrivateGoName(field.Type.Name.Name)) {
delete(field.Extras, "(gogoproto.nullable)")
}
if field.Name != m.Name {
field.Extras["(gogoproto.customname)"] = strconv.Quote(m.Name)
}
field.CommentLines = m.CommentLines
fields = append(fields, field)
}
// assign tags
highest := 0
byTag := make(map[int]*protoField)
// fields are in Go struct order, which we preserve
for i := range fields {
field := &fields[i]
tag := field.Tag
if tag != -1 {
if existing, ok := byTag[tag]; ok {
return nil, fmt.Errorf("field %q and %q both have tag %d", field.Name, existing.Name, tag)
}
byTag[tag] = field
}
if tag > highest {
highest = tag
}
}
// starting from the highest observed tag, assign new field tags
for i := range fields {
field := &fields[i]
if field.Tag != -1 {
continue
}
highest++
field.Tag = highest
byTag[field.Tag] = field
}
return fields, nil
}
func genComment(out io.Writer, lines []string, indent string) {
for {
l := len(lines)
if l == 0 || len(lines[l-1]) != 0 {
break
}
lines = lines[:l-1]
}
for _, c := range lines {
fmt.Fprintf(out, "%s// %s\n", indent, c)
}
}
func formatProtoFile(source []byte) ([]byte, error) {
// TODO; Is there any protobuf formatter?
return source, nil
}
func assembleProtoFile(w io.Writer, f *generator.File) {
w.Write(f.Header)
fmt.Fprint(w, "syntax = 'proto2';\n\n")
if len(f.PackageName) > 0 {
fmt.Fprintf(w, "package %s;\n\n", f.PackageName)
}
if len(f.Imports) > 0 {
imports := []string{}
for i := range f.Imports {
imports = append(imports, i)
}
sort.Strings(imports)
for _, s := range imports {
fmt.Fprintf(w, "import %q;\n", s)
}
fmt.Fprint(w, "\n")
}
if f.Vars.Len() > 0 {
fmt.Fprintf(w, "%s\n", f.Vars.String())
}
w.Write(f.Body.Bytes())
}
func NewProtoFile() *generator.DefaultFileType {
return &generator.DefaultFileType{
Format: formatProtoFile,
Assemble: assembleProtoFile,
}
}

View file

@ -0,0 +1,50 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"k8s.io/gengo/namer"
"k8s.io/gengo/types"
)
type ImportTracker struct {
namer.DefaultImportTracker
}
func NewImportTracker(local types.Name, typesToAdd ...*types.Type) *ImportTracker {
tracker := namer.NewDefaultImportTracker(local)
tracker.IsInvalidType = func(t *types.Type) bool { return t.Kind != types.Protobuf }
tracker.LocalName = func(name types.Name) string { return name.Package }
tracker.PrintImport = func(path, name string) string { return path }
tracker.AddTypes(typesToAdd...)
return &ImportTracker{
DefaultImportTracker: tracker,
}
}
// AddNullable ensures that support for the nullable Gogo-protobuf extension is added.
func (tracker *ImportTracker) AddNullable() {
tracker.AddType(&types.Type{
Kind: types.Protobuf,
Name: types.Name{
Name: "nullable",
Package: "gogoproto",
Path: "github.com/gogo/protobuf/gogoproto/gogo.proto",
},
})
}

View file

@ -0,0 +1,208 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"fmt"
"reflect"
"strings"
"k8s.io/gengo/generator"
"k8s.io/gengo/namer"
"k8s.io/gengo/types"
)
type localNamer struct {
localPackage types.Name
}
func (n localNamer) Name(t *types.Type) string {
if t.Key != nil && t.Elem != nil {
return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem))
}
if len(n.localPackage.Package) != 0 && n.localPackage.Package == t.Name.Package {
return t.Name.Name
}
return t.Name.String()
}
type protobufNamer struct {
packages []*protobufPackage
packagesByPath map[string]*protobufPackage
}
func NewProtobufNamer() *protobufNamer {
return &protobufNamer{
packagesByPath: make(map[string]*protobufPackage),
}
}
func (n *protobufNamer) Name(t *types.Type) string {
if t.Kind == types.Map {
return fmt.Sprintf("map<%s, %s>", n.Name(t.Key), n.Name(t.Elem))
}
return t.Name.String()
}
func (n *protobufNamer) List() []generator.Package {
packages := make([]generator.Package, 0, len(n.packages))
for i := range n.packages {
packages = append(packages, n.packages[i])
}
return packages
}
func (n *protobufNamer) Add(p *protobufPackage) {
if _, ok := n.packagesByPath[p.PackagePath]; !ok {
n.packagesByPath[p.PackagePath] = p
n.packages = append(n.packages, p)
}
}
func (n *protobufNamer) GoNameToProtoName(name types.Name) types.Name {
if p, ok := n.packagesByPath[name.Package]; ok {
return types.Name{
Name: name.Name,
Package: p.PackageName,
Path: p.ImportPath(),
}
}
for _, p := range n.packages {
if _, ok := p.FilterTypes[name]; ok {
return types.Name{
Name: name.Name,
Package: p.PackageName,
Path: p.ImportPath(),
}
}
}
return types.Name{Name: name.Name}
}
func protoSafePackage(name string) string {
pkg := strings.Replace(name, "/", ".", -1)
return strings.Replace(pkg, "-", "_", -1)
}
type typeNameSet map[types.Name]*protobufPackage
// assignGoTypeToProtoPackage looks for Go and Protobuf types that are referenced by a type in
// a package. It will not recurse into protobuf types.
func assignGoTypeToProtoPackage(p *protobufPackage, t *types.Type, local, global typeNameSet, optional map[types.Name]struct{}) {
newT, isProto := isFundamentalProtoType(t)
if isProto {
t = newT
}
if otherP, ok := global[t.Name]; ok {
if _, ok := local[t.Name]; !ok {
p.Imports.AddType(&types.Type{
Kind: types.Protobuf,
Name: otherP.ProtoTypeName(),
})
}
return
}
if t.Name.Package == p.PackagePath {
// Associate types only to their own package
global[t.Name] = p
}
if _, ok := local[t.Name]; ok {
return
}
// don't recurse into existing proto types
if isProto {
p.Imports.AddType(t)
return
}
local[t.Name] = p
for _, m := range t.Members {
if namer.IsPrivateGoName(m.Name) {
continue
}
field := &protoField{}
tag := reflect.StructTag(m.Tags).Get("protobuf")
if tag == "-" {
continue
}
if err := protobufTagToField(tag, field, m, t, p.ProtoTypeName()); err == nil && field.Type != nil {
assignGoTypeToProtoPackage(p, field.Type, local, global, optional)
continue
}
assignGoTypeToProtoPackage(p, m.Type, local, global, optional)
}
// TODO: should methods be walked?
if t.Elem != nil {
assignGoTypeToProtoPackage(p, t.Elem, local, global, optional)
}
if t.Key != nil {
assignGoTypeToProtoPackage(p, t.Key, local, global, optional)
}
if t.Underlying != nil {
if t.Kind == types.Alias && isOptionalAlias(t) {
optional[t.Name] = struct{}{}
}
assignGoTypeToProtoPackage(p, t.Underlying, local, global, optional)
}
}
// isTypeApplicableToProtobuf checks to see if a type is relevant for protobuf processing.
// Currently, it filters out functions and private types.
func isTypeApplicableToProtobuf(t *types.Type) bool {
// skip functions -- we don't care about them for protobuf
if t.Kind == types.Func || (t.Kind == types.DeclarationOf && t.Underlying.Kind == types.Func) {
return false
}
// skip private types
if namer.IsPrivateGoName(t.Name.Name) {
return false
}
return true
}
func (n *protobufNamer) AssignTypesToPackages(c *generator.Context) error {
global := make(typeNameSet)
for _, p := range n.packages {
local := make(typeNameSet)
optional := make(map[types.Name]struct{})
p.Imports = NewImportTracker(p.ProtoTypeName())
for _, t := range c.Order {
if t.Name.Package != p.PackagePath {
continue
}
if !isTypeApplicableToProtobuf(t) {
// skip types that we don't care about, like functions
continue
}
assignGoTypeToProtoPackage(p, t, local, global, optional)
}
p.FilterTypes = make(map[types.Name]struct{})
p.LocalNames = make(map[string]struct{})
p.OptionalTypeNames = make(map[string]struct{})
for k, v := range local {
if v == p {
p.FilterTypes[k] = struct{}{}
p.LocalNames[k.Name] = struct{}{}
if _, ok := optional[k]; ok {
p.OptionalTypeNames[k.Name] = struct{}{}
}
}
}
}
return nil
}

View file

@ -0,0 +1,215 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"fmt"
"go/ast"
"log"
"os"
"path/filepath"
"reflect"
"strings"
"k8s.io/gengo/generator"
"k8s.io/gengo/types"
)
func newProtobufPackage(packagePath, packageName string, generateAll bool, omitFieldTypes map[types.Name]struct{}) *protobufPackage {
pkg := &protobufPackage{
DefaultPackage: generator.DefaultPackage{
// The protobuf package name (foo.bar.baz)
PackageName: packageName,
// A path segment relative to the GOPATH root (foo/bar/baz)
PackagePath: packagePath,
HeaderText: []byte(
`
// This file was autogenerated by go-to-protobuf. Do not edit it manually!
`),
PackageDocumentation: []byte(fmt.Sprintf(
`// Package %s is an autogenerated protobuf IDL.
`, packageName)),
},
GenerateAll: generateAll,
OmitFieldTypes: omitFieldTypes,
}
pkg.FilterFunc = pkg.filterFunc
pkg.GeneratorFunc = pkg.generatorFunc
return pkg
}
// protobufPackage contains the protobuf implementation of Package.
type protobufPackage struct {
generator.DefaultPackage
// If true, this package has been vendored into our source tree and thus can
// only be generated by changing the vendor tree.
Vendored bool
// If true, generate protobuf serializations for all public types.
// If false, only generate protobuf serializations for structs that
// request serialization.
GenerateAll bool
// A list of types to filter to; if not specified all types will be included.
FilterTypes map[types.Name]struct{}
// If true, omit any gogoprotobuf extensions not defined as types.
OmitGogo bool
// A list of field types that will be excluded from the output struct
OmitFieldTypes map[types.Name]struct{}
// A list of names that this package exports
LocalNames map[string]struct{}
// A list of type names in this package that will need marshaller rewriting
// to remove synthetic protobuf fields.
OptionalTypeNames map[string]struct{}
// A list of struct tags to generate onto named struct fields
StructTags map[string]map[string]string
// An import tracker for this package
Imports *ImportTracker
}
func (p *protobufPackage) Clean(outputBase string) error {
for _, s := range []string{p.ImportPath(), p.OutputPath()} {
if err := os.Remove(filepath.Join(outputBase, s)); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
func (p *protobufPackage) ProtoTypeName() types.Name {
return types.Name{
Name: p.Path(), // the go path "foo/bar/baz"
Package: p.Name(), // the protobuf package "foo.bar.baz"
Path: p.ImportPath(), // the path of the import to get the proto
}
}
func (p *protobufPackage) filterFunc(c *generator.Context, t *types.Type) bool {
switch t.Kind {
case types.Func, types.Chan:
return false
case types.Struct:
if t.Name.Name == "struct{}" {
return false
}
case types.Builtin:
return false
case types.Alias:
if !isOptionalAlias(t) {
return false
}
case types.Slice, types.Array, types.Map:
return false
case types.Pointer:
return false
}
if _, ok := isFundamentalProtoType(t); ok {
return false
}
_, ok := p.FilterTypes[t.Name]
return ok
}
func (p *protobufPackage) HasGoType(name string) bool {
_, ok := p.LocalNames[name]
return ok
}
func (p *protobufPackage) OptionalTypeName(name string) bool {
_, ok := p.OptionalTypeNames[name]
return ok
}
func (p *protobufPackage) ExtractGeneratedType(t *ast.TypeSpec) bool {
if !p.HasGoType(t.Name.Name) {
return false
}
switch s := t.Type.(type) {
case *ast.StructType:
for i, f := range s.Fields.List {
if len(f.Tag.Value) == 0 {
continue
}
tag := strings.Trim(f.Tag.Value, "`")
protobufTag := reflect.StructTag(tag).Get("protobuf")
if len(protobufTag) == 0 {
continue
}
if len(f.Names) > 1 {
log.Printf("WARNING: struct %s field %d %s: defined multiple names but single protobuf tag", t.Name.Name, i, f.Names[0].Name)
// TODO hard error?
}
if p.StructTags == nil {
p.StructTags = make(map[string]map[string]string)
}
m := p.StructTags[t.Name.Name]
if m == nil {
m = make(map[string]string)
p.StructTags[t.Name.Name] = m
}
m[f.Names[0].Name] = tag
}
default:
log.Printf("WARNING: unexpected Go AST type definition: %#v", t)
}
return true
}
func (p *protobufPackage) generatorFunc(c *generator.Context) []generator.Generator {
generators := []generator.Generator{}
p.Imports.AddNullable()
generators = append(generators, &genProtoIDL{
DefaultGen: generator.DefaultGen{
OptionalName: "generated",
},
localPackage: types.Name{Package: p.PackageName, Path: p.PackagePath},
localGoPackage: types.Name{Package: p.PackagePath, Name: p.GoPackageName()},
imports: p.Imports,
generateAll: p.GenerateAll,
omitGogo: p.OmitGogo,
omitFieldTypes: p.OmitFieldTypes,
})
return generators
}
func (p *protobufPackage) GoPackageName() string {
return filepath.Base(p.PackagePath)
}
func (p *protobufPackage) ImportPath() string {
return filepath.Join(p.PackagePath, "generated.proto")
}
func (p *protobufPackage) OutputPath() string {
return filepath.Join(p.PackagePath, "generated.pb.go")
}
var (
_ = generator.Package(&protobufPackage{})
)

View file

@ -0,0 +1,452 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"bytes"
"errors"
"fmt"
"go/ast"
"go/format"
"go/parser"
"go/printer"
"go/token"
"io/ioutil"
"os"
"reflect"
"strings"
customreflect "k8s.io/code-generator/third_party/forked/golang/reflect"
)
func rewriteFile(name string, header []byte, rewriteFn func(*token.FileSet, *ast.File) error) error {
fset := token.NewFileSet()
src, err := ioutil.ReadFile(name)
if err != nil {
return err
}
file, err := parser.ParseFile(fset, name, src, parser.DeclarationErrors|parser.ParseComments)
if err != nil {
return err
}
if err := rewriteFn(fset, file); err != nil {
return err
}
b := &bytes.Buffer{}
b.Write(header)
if err := printer.Fprint(b, fset, file); err != nil {
return err
}
body, err := format.Source(b.Bytes())
if err != nil {
return err
}
f, err := os.OpenFile(name, os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer f.Close()
if _, err := f.Write(body); err != nil {
return err
}
return f.Close()
}
// ExtractFunc extracts information from the provided TypeSpec and returns true if the type should be
// removed from the destination file.
type ExtractFunc func(*ast.TypeSpec) bool
// OptionalFunc returns true if the provided local name is a type that has protobuf.nullable=true
// and should have its marshal functions adjusted to remove the 'Items' accessor.
type OptionalFunc func(name string) bool
func RewriteGeneratedGogoProtobufFile(name string, extractFn ExtractFunc, optionalFn OptionalFunc, header []byte) error {
return rewriteFile(name, header, func(fset *token.FileSet, file *ast.File) error {
cmap := ast.NewCommentMap(fset, file, file.Comments)
// transform methods that point to optional maps or slices
for _, d := range file.Decls {
rewriteOptionalMethods(d, optionalFn)
}
// remove types that are already declared
decls := []ast.Decl{}
for _, d := range file.Decls {
if dropExistingTypeDeclarations(d, extractFn) {
continue
}
if dropEmptyImportDeclarations(d) {
continue
}
decls = append(decls, d)
}
file.Decls = decls
// remove unmapped comments
file.Comments = cmap.Filter(file).Comments()
return nil
})
}
// rewriteOptionalMethods makes specific mutations to marshaller methods that belong to types identified
// as being "optional" (they may be nil on the wire). This allows protobuf to serialize a map or slice and
// properly discriminate between empty and nil (which is not possible in protobuf).
// TODO: move into upstream gogo-protobuf once https://github.com/gogo/protobuf/issues/181
// has agreement
func rewriteOptionalMethods(decl ast.Decl, isOptional OptionalFunc) {
switch t := decl.(type) {
case *ast.FuncDecl:
ident, ptr, ok := receiver(t)
if !ok {
return
}
// correct initialization of the form `m.Field = &OptionalType{}` to
// `m.Field = OptionalType{}`
if t.Name.Name == "Unmarshal" {
ast.Walk(optionalAssignmentVisitor{fn: isOptional}, t.Body)
}
if !isOptional(ident.Name) {
return
}
switch t.Name.Name {
case "Unmarshal":
ast.Walk(&optionalItemsVisitor{}, t.Body)
case "MarshalTo", "Size", "String":
ast.Walk(&optionalItemsVisitor{}, t.Body)
fallthrough
case "Marshal":
// if the method has a pointer receiver, set it back to a normal receiver
if ptr {
t.Recv.List[0].Type = ident
}
}
}
}
type optionalAssignmentVisitor struct {
fn OptionalFunc
}
// Visit walks the provided node, transforming field initializations of the form
// m.Field = &OptionalType{} -> m.Field = OptionalType{}
func (v optionalAssignmentVisitor) Visit(n ast.Node) ast.Visitor {
switch t := n.(type) {
case *ast.AssignStmt:
if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
if !isFieldSelector(t.Lhs[0], "m", "") {
return nil
}
unary, ok := t.Rhs[0].(*ast.UnaryExpr)
if !ok || unary.Op != token.AND {
return nil
}
composite, ok := unary.X.(*ast.CompositeLit)
if !ok || composite.Type == nil || len(composite.Elts) != 0 {
return nil
}
if ident, ok := composite.Type.(*ast.Ident); ok && v.fn(ident.Name) {
t.Rhs[0] = composite
}
}
return nil
}
return v
}
type optionalItemsVisitor struct{}
// Visit walks the provided node, looking for specific patterns to transform that match
// the effective outcome of turning struct{ map[x]y || []x } into map[x]y or []x.
func (v *optionalItemsVisitor) Visit(n ast.Node) ast.Visitor {
switch t := n.(type) {
case *ast.RangeStmt:
if isFieldSelector(t.X, "m", "Items") {
t.X = &ast.Ident{Name: "m"}
}
case *ast.AssignStmt:
if len(t.Lhs) == 1 && len(t.Rhs) == 1 {
switch lhs := t.Lhs[0].(type) {
case *ast.IndexExpr:
if isFieldSelector(lhs.X, "m", "Items") {
lhs.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
default:
if isFieldSelector(t.Lhs[0], "m", "Items") {
t.Lhs[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
}
switch rhs := t.Rhs[0].(type) {
case *ast.CallExpr:
if ident, ok := rhs.Fun.(*ast.Ident); ok && ident.Name == "append" {
ast.Walk(v, rhs)
if len(rhs.Args) > 0 {
switch arg := rhs.Args[0].(type) {
case *ast.Ident:
if arg.Name == "m" {
rhs.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
}
}
return nil
}
}
}
case *ast.IfStmt:
switch cond := t.Cond.(type) {
case *ast.BinaryExpr:
if cond.Op == token.EQL {
if isFieldSelector(cond.X, "m", "Items") && isIdent(cond.Y, "nil") {
cond.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
}
}
if t.Init != nil {
// Find form:
// if err := m[len(m.Items)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
// return err
// }
switch s := t.Init.(type) {
case *ast.AssignStmt:
if call, ok := s.Rhs[0].(*ast.CallExpr); ok {
if sel, ok := call.Fun.(*ast.SelectorExpr); ok {
if x, ok := sel.X.(*ast.IndexExpr); ok {
// m[] -> (*m)[]
if sel2, ok := x.X.(*ast.SelectorExpr); ok {
if ident, ok := sel2.X.(*ast.Ident); ok && ident.Name == "m" {
x.X = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
}
// len(m.Items) -> len(*m)
if bin, ok := x.Index.(*ast.BinaryExpr); ok {
if call2, ok := bin.X.(*ast.CallExpr); ok && len(call2.Args) == 1 {
if isFieldSelector(call2.Args[0], "m", "Items") {
call2.Args[0] = &ast.StarExpr{X: &ast.Ident{Name: "m"}}
}
}
}
}
}
}
}
}
case *ast.IndexExpr:
if isFieldSelector(t.X, "m", "Items") {
t.X = &ast.Ident{Name: "m"}
return nil
}
case *ast.CallExpr:
changed := false
for i := range t.Args {
if isFieldSelector(t.Args[i], "m", "Items") {
t.Args[i] = &ast.Ident{Name: "m"}
changed = true
}
}
if changed {
return nil
}
}
return v
}
func isFieldSelector(n ast.Expr, name, field string) bool {
s, ok := n.(*ast.SelectorExpr)
if !ok || s.Sel == nil || (field != "" && s.Sel.Name != field) {
return false
}
return isIdent(s.X, name)
}
func isIdent(n ast.Expr, value string) bool {
ident, ok := n.(*ast.Ident)
return ok && ident.Name == value
}
func receiver(f *ast.FuncDecl) (ident *ast.Ident, pointer bool, ok bool) {
if f.Recv == nil || len(f.Recv.List) != 1 {
return nil, false, false
}
switch t := f.Recv.List[0].Type.(type) {
case *ast.StarExpr:
identity, ok := t.X.(*ast.Ident)
if !ok {
return nil, false, false
}
return identity, true, true
case *ast.Ident:
return t, false, true
}
return nil, false, false
}
// dropExistingTypeDeclarations removes any type declaration for which extractFn returns true. The function
// returns true if the entire declaration should be dropped.
func dropExistingTypeDeclarations(decl ast.Decl, extractFn ExtractFunc) bool {
switch t := decl.(type) {
case *ast.GenDecl:
if t.Tok != token.TYPE {
return false
}
specs := []ast.Spec{}
for _, s := range t.Specs {
switch spec := s.(type) {
case *ast.TypeSpec:
if extractFn(spec) {
continue
}
specs = append(specs, spec)
}
}
if len(specs) == 0 {
return true
}
t.Specs = specs
}
return false
}
// dropEmptyImportDeclarations strips any generated but no-op imports from the generated code
// to prevent generation from being able to define side-effects. The function returns true
// if the entire declaration should be dropped.
func dropEmptyImportDeclarations(decl ast.Decl) bool {
switch t := decl.(type) {
case *ast.GenDecl:
if t.Tok != token.IMPORT {
return false
}
specs := []ast.Spec{}
for _, s := range t.Specs {
switch spec := s.(type) {
case *ast.ImportSpec:
if spec.Name != nil && spec.Name.Name == "_" {
continue
}
specs = append(specs, spec)
}
}
if len(specs) == 0 {
return true
}
t.Specs = specs
}
return false
}
func RewriteTypesWithProtobufStructTags(name string, structTags map[string]map[string]string) error {
return rewriteFile(name, []byte{}, func(fset *token.FileSet, file *ast.File) error {
allErrs := []error{}
// set any new struct tags
for _, d := range file.Decls {
if errs := updateStructTags(d, structTags, []string{"protobuf"}); len(errs) > 0 {
allErrs = append(allErrs, errs...)
}
}
if len(allErrs) > 0 {
var s string
for _, err := range allErrs {
s += err.Error() + "\n"
}
return errors.New(s)
}
return nil
})
}
func updateStructTags(decl ast.Decl, structTags map[string]map[string]string, toCopy []string) []error {
var errs []error
t, ok := decl.(*ast.GenDecl)
if !ok {
return nil
}
if t.Tok != token.TYPE {
return nil
}
for _, s := range t.Specs {
spec, ok := s.(*ast.TypeSpec)
if !ok {
continue
}
typeName := spec.Name.Name
fieldTags, ok := structTags[typeName]
if !ok {
continue
}
st, ok := spec.Type.(*ast.StructType)
if !ok {
continue
}
for i := range st.Fields.List {
f := st.Fields.List[i]
var name string
if len(f.Names) == 0 {
switch t := f.Type.(type) {
case *ast.Ident:
name = t.Name
case *ast.SelectorExpr:
name = t.Sel.Name
default:
errs = append(errs, fmt.Errorf("unable to get name for tag from struct %q, field %#v", spec.Name.Name, t))
continue
}
} else {
name = f.Names[0].Name
}
value, ok := fieldTags[name]
if !ok {
continue
}
var tags customreflect.StructTags
if f.Tag != nil {
oldTags, err := customreflect.ParseStructTags(strings.Trim(f.Tag.Value, "`"))
if err != nil {
errs = append(errs, fmt.Errorf("unable to read struct tag from struct %q, field %q: %v", spec.Name.Name, name, err))
continue
}
tags = oldTags
}
for _, name := range toCopy {
// don't overwrite existing tags
if tags.Has(name) {
continue
}
// append new tags
if v := reflect.StructTag(value).Get(name); len(v) > 0 {
tags = append(tags, customreflect.StructTag{Name: name, Value: v})
}
}
if len(tags) == 0 {
continue
}
if f.Tag == nil {
f.Tag = &ast.BasicLit{}
}
f.Tag.Value = tags.String()
}
}
return errs
}

View file

@ -0,0 +1,33 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"github.com/golang/glog"
"k8s.io/gengo/types"
)
// extractBoolTagOrDie gets the comment-tags for the key and asserts that, if
// it exists, the value is boolean. If the tag did not exist, it returns
// false.
func extractBoolTagOrDie(key string, lines []string) bool {
val, err := types.ExtractSingleBoolCommentTag("+", key, false, lines)
if err != nil {
glog.Fatal(err)
}
return val
}

View file

@ -0,0 +1,32 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package main defines the protoc-gen-gogo binary we use to generate our proto go files,
// as well as takes dependencies on the correct gogo/protobuf packages for godeps.
package main
import (
"github.com/gogo/protobuf/vanity/command"
// dependencies that are required for our packages
_ "github.com/gogo/protobuf/gogoproto"
_ "github.com/gogo/protobuf/proto"
_ "github.com/gogo/protobuf/sortkeys"
)
func main() {
command.Write(command.Generate(command.Read()))
}