Vendor dependencies for GCS

This commit is contained in:
Alexander Neumann 2017-08-05 20:17:15 +02:00
parent ba75a3884c
commit 8ca6a9a240
1228 changed files with 1769186 additions and 1 deletions

View file

@ -0,0 +1,427 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package disco represents Google API discovery documents.
package disco
import (
"encoding/json"
"fmt"
"reflect"
"sort"
"strings"
)
// A Document is an API discovery document.
type Document struct {
ID string `json:"id"`
Name string `json:"name"`
Version string `json:"version"`
Title string `json:"title"`
RootURL string `json:"rootUrl"`
ServicePath string `json:"servicePath"`
BasePath string `json:"basePath"`
DocumentationLink string `json:"documentationLink"`
Auth Auth `json:"auth"`
Features []string `json:"features"`
Methods MethodList `json:"methods"`
Schemas map[string]*Schema `json:"schemas"`
Resources ResourceList `json:"resources"`
}
// init performs additional initialization and checks that
// were not done during unmarshaling.
func (d *Document) init() error {
schemasByID := map[string]*Schema{}
for _, s := range d.Schemas {
schemasByID[s.ID] = s
}
for name, s := range d.Schemas {
if s.Ref != "" {
return fmt.Errorf("top level schema %q is a reference", name)
}
s.Name = name
if err := s.init(schemasByID); err != nil {
return err
}
}
for _, m := range d.Methods {
if err := m.init(schemasByID); err != nil {
return err
}
}
for _, r := range d.Resources {
if err := r.init("", schemasByID); err != nil {
return err
}
}
return nil
}
// NewDocument unmarshals the bytes into a Document.
// It also validates the document to make sure it is error-free.
func NewDocument(bytes []byte) (*Document, error) {
var doc Document
if err := json.Unmarshal(bytes, &doc); err != nil {
return nil, err
}
if err := doc.init(); err != nil {
return nil, err
}
return &doc, nil
}
// Auth represents the auth section of a discovery document.
// Only OAuth2 information is retained.
type Auth struct {
OAuth2Scopes []Scope
}
// A Scope is an OAuth2 scope.
type Scope struct {
URL string
Description string
}
// UnmarshalJSON implements the json.Unmarshaler interface.
func (a *Auth) UnmarshalJSON(data []byte) error {
// Pull out the oauth2 scopes and turn them into nice structs.
// Ignore other auth information.
var m struct {
OAuth2 struct {
Scopes map[string]struct {
Description string
}
}
}
if err := json.Unmarshal(data, &m); err != nil {
return err
}
// Sort keys to provide a deterministic ordering, mainly for testing.
for _, k := range sortedKeys(m.OAuth2.Scopes) {
a.OAuth2Scopes = append(a.OAuth2Scopes, Scope{
URL: k,
Description: m.OAuth2.Scopes[k].Description,
})
}
return nil
}
// A Schema holds a JSON Schema as defined by
// https://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.1.
// We only support the subset of JSON Schema needed for Google API generation.
type Schema struct {
ID string // union types not supported
Type string // union types not supported
Format string
Description string
Properties PropertyList
ItemSchema *Schema `json:"items"` // array of schemas not supported
AdditionalProperties *Schema // boolean not supported
Ref string `json:"$ref"`
Default string
Pattern string
Enums []string `json:"enum"`
// Google extensions to JSON Schema
EnumDescriptions []string
Variant *Variant
RefSchema *Schema `json:"-"` // Schema referred to by $ref
Name string `json:"-"` // Schema name, if top level
Kind Kind `json:"-"`
}
type Variant struct {
Discriminant string
Map []*VariantMapItem
}
type VariantMapItem struct {
TypeValue string `json:"type_value"`
Ref string `json:"$ref"`
}
func (s *Schema) init(topLevelSchemas map[string]*Schema) error {
if s == nil {
return nil
}
var err error
if s.Ref != "" {
if s.RefSchema, err = resolveRef(s.Ref, topLevelSchemas); err != nil {
return err
}
}
s.Kind, err = s.initKind()
if err != nil {
return err
}
if s.Kind == ArrayKind && s.ItemSchema == nil {
return fmt.Errorf("schema %+v: array does not have items", s)
}
if s.Kind != ArrayKind && s.ItemSchema != nil {
return fmt.Errorf("schema %+v: non-array has items", s)
}
if err := s.AdditionalProperties.init(topLevelSchemas); err != nil {
return err
}
if err := s.ItemSchema.init(topLevelSchemas); err != nil {
return err
}
for _, p := range s.Properties {
if err := p.Schema.init(topLevelSchemas); err != nil {
return err
}
}
return nil
}
func resolveRef(ref string, topLevelSchemas map[string]*Schema) (*Schema, error) {
rs, ok := topLevelSchemas[ref]
if !ok {
return nil, fmt.Errorf("could not resolve schema reference %q", ref)
}
return rs, nil
}
func (s *Schema) initKind() (Kind, error) {
if s.Ref != "" {
return ReferenceKind, nil
}
switch s.Type {
case "string", "number", "integer", "boolean", "any":
return SimpleKind, nil
case "object":
if s.AdditionalProperties != nil {
if s.AdditionalProperties.Type == "any" {
return AnyStructKind, nil
}
return MapKind, nil
}
return StructKind, nil
case "array":
return ArrayKind, nil
default:
return 0, fmt.Errorf("unknown type %q for schema %q", s.Type, s.ID)
}
}
// ElementSchema returns the schema for the element type of s. For maps,
// this is the schema of the map values. For arrays, it is the schema
// of the array item type.
//
// ElementSchema panics if called on a schema that is not of kind map or array.
func (s *Schema) ElementSchema() *Schema {
switch s.Kind {
case MapKind:
return s.AdditionalProperties
case ArrayKind:
return s.ItemSchema
default:
panic("ElementSchema called on schema of type " + s.Type)
}
}
// IsIntAsString reports whether the schema represents an integer value
// formatted as a string.
func (s *Schema) IsIntAsString() bool {
return s.Type == "string" && strings.Contains(s.Format, "int")
}
// Kind classifies a Schema.
type Kind int
const (
// SimpleKind is the category for any JSON Schema that maps to a
// primitive Go type: strings, numbers, booleans, and "any" (since it
// maps to interface{}).
SimpleKind Kind = iota
// StructKind is the category for a JSON Schema that declares a JSON
// object without any additional (arbitrary) properties.
StructKind
// MapKind is the category for a JSON Schema that declares a JSON
// object with additional (arbitrary) properties that have a non-"any"
// schema type.
MapKind
// AnyStructKind is the category for a JSON Schema that declares a
// JSON object with additional (arbitrary) properties that can be any
// type.
AnyStructKind
// ArrayKind is the category for a JSON Schema that declares an
// "array" type.
ArrayKind
// ReferenceKind is the category for a JSON Schema that is a reference
// to another JSON Schema. During code generation, these references
// are resolved using the API.schemas map.
// See https://tools.ietf.org/html/draft-zyp-json-schema-03#section-5.28
// for more details on the format.
ReferenceKind
)
type Property struct {
Name string
Schema *Schema
}
type PropertyList []*Property
func (pl *PropertyList) UnmarshalJSON(data []byte) error {
// In the discovery doc, properties are a map. Convert to a list.
var m map[string]*Schema
if err := json.Unmarshal(data, &m); err != nil {
return err
}
for _, k := range sortedKeys(m) {
*pl = append(*pl, &Property{
Name: k,
Schema: m[k],
})
}
return nil
}
type ResourceList []*Resource
func (rl *ResourceList) UnmarshalJSON(data []byte) error {
// In the discovery doc, resources are a map. Convert to a list.
var m map[string]*Resource
if err := json.Unmarshal(data, &m); err != nil {
return err
}
for _, k := range sortedKeys(m) {
r := m[k]
r.Name = k
*rl = append(*rl, r)
}
return nil
}
// A Resource holds information about a Google API Resource.
type Resource struct {
Name string
FullName string // {parent.FullName}.{Name}
Methods MethodList
Resources ResourceList
}
func (r *Resource) init(parentFullName string, topLevelSchemas map[string]*Schema) error {
r.FullName = fmt.Sprintf("%s.%s", parentFullName, r.Name)
for _, m := range r.Methods {
if err := m.init(topLevelSchemas); err != nil {
return err
}
}
for _, r2 := range r.Resources {
if err := r2.init(r.FullName, topLevelSchemas); err != nil {
return err
}
}
return nil
}
type MethodList []*Method
func (ml *MethodList) UnmarshalJSON(data []byte) error {
// In the discovery doc, resources are a map. Convert to a list.
var m map[string]*Method
if err := json.Unmarshal(data, &m); err != nil {
return err
}
for _, k := range sortedKeys(m) {
meth := m[k]
meth.Name = k
*ml = append(*ml, meth)
}
return nil
}
// A Method holds information about a resource method.
type Method struct {
Name string
ID string
Path string
HTTPMethod string
Description string
Parameters ParameterList
ParameterOrder []string
Request *Schema
Response *Schema
Scopes []string
MediaUpload *MediaUpload
SupportsMediaDownload bool
JSONMap map[string]interface{} `json:"-"`
}
type MediaUpload struct {
Accept []string
MaxSize string
Protocols map[string]Protocol
}
type Protocol struct {
Multipart bool
Path string
}
func (m *Method) init(topLevelSchemas map[string]*Schema) error {
if err := m.Request.init(topLevelSchemas); err != nil {
return err
}
if err := m.Response.init(topLevelSchemas); err != nil {
return err
}
return nil
}
func (m *Method) UnmarshalJSON(data []byte) error {
type T Method // avoid a recursive call to UnmarshalJSON
if err := json.Unmarshal(data, (*T)(m)); err != nil {
return err
}
// Keep the unmarshalled map around, because the generator
// outputs it as a comment after the method body.
// TODO(jba): make this unnecessary.
return json.Unmarshal(data, &m.JSONMap)
}
type ParameterList []*Parameter
func (pl *ParameterList) UnmarshalJSON(data []byte) error {
// In the discovery doc, resources are a map. Convert to a list.
var m map[string]*Parameter
if err := json.Unmarshal(data, &m); err != nil {
return err
}
for _, k := range sortedKeys(m) {
p := m[k]
p.Name = k
*pl = append(*pl, p)
}
return nil
}
// A Parameter holds information about a method parameter.
type Parameter struct {
Name string
Schema
Required bool
Repeated bool
Location string
}
// sortedKeys returns the keys of m, which must be a map[string]T, in sorted order.
func sortedKeys(m interface{}) []string {
vkeys := reflect.ValueOf(m).MapKeys()
var keys []string
for _, vk := range vkeys {
keys = append(keys, vk.Interface().(string))
}
sort.Strings(keys)
return keys
}

View file

@ -0,0 +1,264 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package disco
import (
"io/ioutil"
"reflect"
"testing"
)
var stringSchema = &Schema{
Type: "string",
Kind: SimpleKind,
}
func TestDocument(t *testing.T) {
bytes, err := ioutil.ReadFile("testdata/test-api.json")
if err != nil {
t.Fatal(err)
}
got, err := NewDocument(bytes)
if err != nil {
t.Fatal(err)
}
want := &Document{
ID: "storage:v1",
Name: "storage",
Version: "v1",
Title: "Cloud Storage JSON API",
RootURL: "https://www.googleapis.com/",
ServicePath: "storage/v1/",
BasePath: "/storage/v1/",
DocumentationLink: "https://developers.google.com/storage/docs/json_api/",
Auth: Auth{
OAuth2Scopes: []Scope{
{"https://www.googleapis.com/auth/cloud-platform",
"View and manage your data across Google Cloud Platform services"},
{"https://www.googleapis.com/auth/cloud-platform.read-only",
"View your data across Google Cloud Platform services"},
{"https://www.googleapis.com/auth/devstorage.full_control",
"Manage your data and permissions in Google Cloud Storage"},
{"https://www.googleapis.com/auth/devstorage.read_only",
"View your data in Google Cloud Storage"},
{"https://www.googleapis.com/auth/devstorage.read_write",
"Manage your data in Google Cloud Storage"},
},
},
Features: []string{"dataWrapper"},
Schemas: map[string]*Schema{
"Bucket": &Schema{
Name: "Bucket",
ID: "Bucket",
Type: "object",
Description: "A bucket.",
Kind: StructKind,
Properties: []*Property{
{"cors", &Schema{
Type: "array",
Kind: ArrayKind,
ItemSchema: &Schema{
Type: "object",
Kind: StructKind,
Properties: []*Property{
{"maxAgeSeconds", &Schema{
Type: "integer",
Format: "int32",
Kind: SimpleKind,
}},
{"method", &Schema{
Type: "array",
Kind: ArrayKind,
ItemSchema: stringSchema,
}},
},
},
}},
{"id", stringSchema},
{"kind", &Schema{
Type: "string",
Kind: SimpleKind,
Default: "storage#bucket",
}},
},
},
"Buckets": &Schema{
ID: "Buckets",
Name: "Buckets",
Type: "object",
Kind: StructKind,
Properties: []*Property{
{"items", &Schema{
Type: "array",
Kind: ArrayKind,
ItemSchema: &Schema{
Kind: ReferenceKind,
Ref: "Bucket",
RefSchema: nil,
},
}},
},
},
"VariantExample": &Schema{
ID: "VariantExample",
Name: "VariantExample",
Type: "object",
Kind: StructKind,
Variant: &Variant{
Discriminant: "type",
Map: []*VariantMapItem{
{TypeValue: "Bucket", Ref: "Bucket"},
{TypeValue: "Buckets", Ref: "Buckets"},
},
},
},
},
Methods: MethodList{
&Method{
Name: "getCertForOpenIdConnect",
ID: "oauth2.getCertForOpenIdConnect",
Path: "oauth2/v1/certs",
HTTPMethod: "GET",
Response: &Schema{Ref: "Bucket", Kind: ReferenceKind},
},
},
Resources: ResourceList{
&Resource{
Name: "buckets",
FullName: ".buckets",
Methods: MethodList{
&Method{
Name: "get",
ID: "storage.buckets.get",
Path: "b/{bucket}",
HTTPMethod: "GET",
Description: "d",
Parameters: ParameterList{
&Parameter{
Name: "bucket",
Schema: Schema{
Type: "string",
},
Required: true,
Location: "path",
},
&Parameter{
Name: "ifMetagenerationMatch",
Schema: Schema{
Type: "string",
Format: "int64",
},
Location: "query",
},
&Parameter{
Name: "projection",
Schema: Schema{
Type: "string",
Enums: []string{"full", "noAcl"},
EnumDescriptions: []string{
"Include all properties.",
"Omit owner, acl and defaultObjectAcl properties.",
},
},
Location: "query",
},
},
ParameterOrder: []string{"bucket"},
Response: &Schema{Ref: "Bucket", Kind: ReferenceKind},
Scopes: []string{
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/cloud-platform.read-only",
"https://www.googleapis.com/auth/devstorage.full_control",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/devstorage.read_write",
},
SupportsMediaDownload: true,
MediaUpload: &MediaUpload{
Accept: []string{"application/octet-stream"},
MaxSize: "1GB",
Protocols: map[string]Protocol{
"simple": Protocol{
Multipart: true,
Path: "/upload/customDataSources/{customDataSourceId}/uploads",
},
"resumable": Protocol{
Multipart: true,
Path: "/resumable/upload/customDataSources/{customDataSourceId}/uploads",
},
},
},
},
},
},
},
}
// Resolve schema references.
bucket := want.Schemas["Bucket"]
want.Schemas["Buckets"].Properties[0].Schema.ItemSchema.RefSchema = bucket
want.Methods[0].Response.RefSchema = bucket
want.Resources[0].Methods[0].Response.RefSchema = bucket
for k, gs := range got.Schemas {
ws := want.Schemas[k]
if !reflect.DeepEqual(gs, ws) {
t.Fatalf("schema %s: got\n%+v\nwant\n%+v", k, gs, ws)
}
}
if len(got.Schemas) != len(want.Schemas) {
t.Errorf("want %d schemas, got %d", len(got.Schemas), len(want.Schemas))
}
compareMethodLists(t, got.Methods, want.Methods)
for i, gr := range got.Resources {
wr := want.Resources[i]
compareMethodLists(t, gr.Methods, wr.Methods)
if !reflect.DeepEqual(gr, wr) {
t.Fatalf("resource %d: got\n%+v\nwant\n%+v", i, gr, wr)
}
}
if len(got.Resources) != len(want.Resources) {
t.Errorf("want %d resources, got %d", len(got.Resources), len(want.Resources))
}
if !reflect.DeepEqual(got, want) {
t.Errorf("got\n%+v\nwant\n%+v", got, want)
}
}
func compareMethodLists(t *testing.T, got, want MethodList) {
if len(got) != len(want) {
t.Fatalf("got %d methods, want %d", len(got), len(want))
}
for i, gm := range got {
gm.JSONMap = nil // don't compare the raw JSON
wm := want[i]
if !reflect.DeepEqual(gm, wm) {
t.Errorf("#%d: got\n%+v\nwant\n%+v", i, gm, wm)
}
}
}
func TestDocumentErrors(t *testing.T) {
for _, in := range []string{
`{"name": "X"`, // malformed JSON
`{"id": 3}`, // ID is an int instead of a string
`{"auth": "oauth2": { "scopes": "string" }}`, // wrong auth structure
} {
_, err := NewDocument([]byte(in))
if err == nil {
t.Errorf("%s: got nil, want error", in)
}
}
}
func TestSchemaErrors(t *testing.T) {
for _, s := range []*Schema{
{Type: "array"}, // missing item schema
{Type: "string", ItemSchema: &Schema{}}, // items w/o array
{Type: "moose"}, // bad kind
{Ref: "Thing"}, // unresolved reference
} {
if err := s.init(nil); err == nil {
t.Errorf("%+v: got nil, want error", s)
}
}
}

View file

@ -0,0 +1,235 @@
{
"kind": "discovery#restDescription",
"etag": "\"tbys6C40o18GZwyMen5GMkdK-3s/sMgjc4eoIFjgub4daTU-MGW0WMA\"",
"discoveryVersion": "v1",
"id": "storage:v1",
"name": "storage",
"version": "v1",
"revision": "20161109",
"title": "Cloud Storage JSON API",
"description": "Stores and retrieves potentially large, immutable data objects.",
"ownerDomain": "google.com",
"ownerName": "Google",
"icons": {
"x16": "https://www.google.com/images/icons/product/cloud_storage-16.png",
"x32": "https://www.google.com/images/icons/product/cloud_storage-32.png"
},
"documentationLink": "https://developers.google.com/storage/docs/json_api/",
"labels": [
"labs"
],
"protocol": "rest",
"baseUrl": "https://www.googleapis.com/storage/v1/",
"basePath": "/storage/v1/",
"rootUrl": "https://www.googleapis.com/",
"servicePath": "storage/v1/",
"batchPath": "batch",
"parameters": {
"alt": {
"type": "string",
"description": "Data format for the response.",
"default": "json",
"enum": [
"json"
],
"enumDescriptions": [
"Responses with Content-Type of application/json"
],
"location": "query"
},
"fields": {
"type": "string",
"description": "Selector specifying which fields to include in a partial response.",
"location": "query"
},
"key": {
"type": "string",
"description": "API key. Your API key identifies your project and provides you with API access, quota, and reports. Required unless you provide an OAuth 2.0 token.",
"location": "query"
},
"oauth_token": {
"type": "string",
"description": "OAuth 2.0 token for the current user.",
"location": "query"
},
"prettyPrint": {
"type": "boolean",
"description": "Returns response with indentations and line breaks.",
"default": "true",
"location": "query"
},
"quotaUser": {
"type": "string",
"description": "Available to use for quota purposes for server-side applications. Can be any arbitrary string assigned to a user, but should not exceed 40 characters. Overrides userIp if both are provided.",
"location": "query"
},
"userIp": {
"type": "string",
"description": "IP address of the site where the request originates. Use this if you want to enforce per-user limits.",
"location": "query"
}
},
"auth": {
"oauth2": {
"scopes": {
"https://www.googleapis.com/auth/cloud-platform": {
"description": "View and manage your data across Google Cloud Platform services"
},
"https://www.googleapis.com/auth/cloud-platform.read-only": {
"description": "View your data across Google Cloud Platform services"
},
"https://www.googleapis.com/auth/devstorage.full_control": {
"description": "Manage your data and permissions in Google Cloud Storage"
},
"https://www.googleapis.com/auth/devstorage.read_only": {
"description": "View your data in Google Cloud Storage"
},
"https://www.googleapis.com/auth/devstorage.read_write": {
"description": "Manage your data in Google Cloud Storage"
}
}
}
},
"features": [
"dataWrapper"
],
"schemas": {
"Bucket": {
"id": "Bucket",
"type": "object",
"description": "A bucket.",
"properties": {
"cors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"maxAgeSeconds": {
"type": "integer",
"format": "int32"
},
"method": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
},
"id": {
"type": "string"
},
"kind": {
"type": "string",
"default": "storage#bucket"
}
}
},
"Buckets": {
"id": "Buckets",
"type": "object",
"properties": {
"items": {
"type": "array",
"items": {
"$ref": "Bucket"
}
}
}
},
"VariantExample": {
"id": "VariantExample",
"type": "object",
"variant": {
"discriminant": "type",
"map": [
{
"type_value": "Bucket",
"$ref": "Bucket"
},
{
"type_value": "Buckets",
"$ref": "Buckets"
}
]
}
}
},
"methods": {
"getCertForOpenIdConnect": {
"id": "oauth2.getCertForOpenIdConnect",
"path": "oauth2/v1/certs",
"httpMethod": "GET",
"response": {
"$ref": "Bucket"
}
}
},
"resources": {
"buckets": {
"methods": {
"get": {
"id": "storage.buckets.get",
"path": "b/{bucket}",
"httpMethod": "GET",
"description": "d",
"parameters": {
"bucket": {
"type": "string",
"required": true,
"location": "path"
},
"ifMetagenerationMatch": {
"type": "string",
"format": "int64",
"location": "query"
},
"projection": {
"type": "string",
"enum": [
"full",
"noAcl"
],
"enumDescriptions": [
"Include all properties.",
"Omit owner, acl and defaultObjectAcl properties."
],
"location": "query"
}
},
"parameterOrder": [
"bucket"
],
"response": {
"$ref": "Bucket"
},
"scopes": [
"https://www.googleapis.com/auth/cloud-platform",
"https://www.googleapis.com/auth/cloud-platform.read-only",
"https://www.googleapis.com/auth/devstorage.full_control",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/devstorage.read_write"
],
"supportsMediaDownload": true,
"mediaUpload": {
"accept": [
"application/octet-stream"
],
"maxSize": "1GB",
"protocols": {
"simple": {
"multipart": true,
"path": "/upload/customDataSources/{customDataSourceId}/uploads"
},
"resumable": {
"multipart": true,
"path": "/resumable/upload/customDataSources/{customDataSourceId}/uploads"
}
}
}
}
}
}
}
}