Vendor dependencies for GCS

vendor/google.golang.org/appengine/remote_api/client.go

@@ -0,0 +1,174 @@
+// Copyright 2013 Google Inc. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package remote_api
+
+// This file provides the client for connecting remotely to a user's production
+// application.
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+	"golang.org/x/net/context"
+
+	"google.golang.org/appengine/internal"
+	pb "google.golang.org/appengine/internal/remote_api"
+)
+
+// NewRemoteContext returns a context that gives access to the production
+// APIs for the application at the given host. All communication will be
+// performed over SSL unless the host is localhost.
+func NewRemoteContext(host string, client *http.Client) (context.Context, error) {
+	// Add an appcfg header to outgoing requests.
+	t := client.Transport
+	if t == nil {
+		t = http.DefaultTransport
+	}
+	client.Transport = &headerAddingRoundTripper{t}
+
+	url := url.URL{
+		Scheme: "https",
+		Host:   host,
+		Path:   "/_ah/remote_api",
+	}
+	if host == "localhost" || strings.HasPrefix(host, "localhost:") {
+		url.Scheme = "http"
+	}
+	u := url.String()
+	appID, err := getAppID(client, u)
+	if err != nil {
+		return nil, fmt.Errorf("unable to contact server: %v", err)
+	}
+	rc := &remoteContext{
+		client: client,
+		url:    u,
+	}
+	ctx := internal.WithCallOverride(context.Background(), rc.call)
+	ctx = internal.WithLogOverride(ctx, rc.logf)
+	ctx = internal.WithAppIDOverride(ctx, appID)
+	return ctx, nil
+}
+
+type remoteContext struct {
+	client *http.Client
+	url    string
+}
+
+var logLevels = map[int64]string{
+	0: "DEBUG",
+	1: "INFO",
+	2: "WARNING",
+	3: "ERROR",
+	4: "CRITICAL",
+}
+
+func (c *remoteContext) logf(level int64, format string, args ...interface{}) {
+	log.Printf(logLevels[level]+": "+format, args...)
+}
+
+func (c *remoteContext) call(ctx context.Context, service, method string, in, out proto.Message) error {
+	req, err := proto.Marshal(in)
+	if err != nil {
+		return fmt.Errorf("error marshalling request: %v", err)
+	}
+
+	remReq := &pb.Request{
+		ServiceName: proto.String(service),
+		Method:      proto.String(method),
+		Request:     req,
+		// NOTE(djd): RequestId is unused in the server.
+	}
+
+	req, err = proto.Marshal(remReq)
+	if err != nil {
+		return fmt.Errorf("proto.Marshal: %v", err)
+	}
+
+	// TODO(djd): Respect ctx.Deadline()?
+	resp, err := c.client.Post(c.url, "application/octet-stream", bytes.NewReader(req))
+	if err != nil {
+		return fmt.Errorf("error sending request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body)
+	}
+	if err != nil {
+		return fmt.Errorf("failed reading response: %v", err)
+	}
+	remResp := &pb.Response{}
+	if err := proto.Unmarshal(body, remResp); err != nil {
+		return fmt.Errorf("error unmarshalling response: %v", err)
+	}
+
+	if ae := remResp.GetApplicationError(); ae != nil {
+		return &internal.APIError{
+			Code:    ae.GetCode(),
+			Detail:  ae.GetDetail(),
+			Service: service,
+		}
+	}
+
+	if remResp.Response == nil {
+		return fmt.Errorf("unexpected response: %s", proto.MarshalTextString(remResp))
+	}
+
+	return proto.Unmarshal(remResp.Response, out)
+}
+
+// This is a forgiving regexp designed to parse the app ID from YAML.
+var appIDRE = regexp.MustCompile(`app_id["']?\s*:\s*['"]?([-a-z0-9.:~]+)`)
+
+func getAppID(client *http.Client, url string) (string, error) {
+	// Generate a pseudo-random token for handshaking.
+	token := strconv.Itoa(rand.New(rand.NewSource(time.Now().UnixNano())).Int())
+
+	resp, err := client.Get(fmt.Sprintf("%s?rtok=%s", url, token))
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("bad response %d; body: %q", resp.StatusCode, body)
+	}
+	if err != nil {
+		return "", fmt.Errorf("failed reading response: %v", err)
+	}
+
+	// Check the token is present in response.
+	if !bytes.Contains(body, []byte(token)) {
+		return "", fmt.Errorf("token not found: want %q; body %q", token, body)
+	}
+
+	match := appIDRE.FindSubmatch(body)
+	if match == nil {
+		return "", fmt.Errorf("app ID not found: body %q", body)
+	}
+
+	return string(match[1]), nil
+}
+
+type headerAddingRoundTripper struct {
+	Wrapped http.RoundTripper
+}
+
+func (t *headerAddingRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
+	r.Header.Set("X-Appcfg-Api-Version", "1")
+	return t.Wrapped.RoundTrip(r)
+}

vendor/google.golang.org/appengine/remote_api/client_test.go

@@ -0,0 +1,24 @@
+// Copyright 2014 Google Inc. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package remote_api
+
+import (
+	"testing"
+)
+
+func TestAppIDRE(t *testing.T) {
+	appID := "s~my-appid-539"
+	tests := []string{
+		"{rtok: 8306111115908860449, app_id: s~my-appid-539}\n",
+		"{rtok: 8306111115908860449, app_id: 's~my-appid-539'}\n",
+		`{rtok: 8306111115908860449, app_id: "s~my-appid-539"}`,
+		`{rtok: 8306111115908860449, "app_id":"s~my-appid-539"}`,
+	}
+	for _, v := range tests {
+		if g := appIDRE.FindStringSubmatch(v); g == nil || g[1] != appID {
+			t.Errorf("appIDRE.FindStringSubmatch(%s) got %q, want %q", v, g, appID)
+		}
+	}
+}

vendor/google.golang.org/appengine/remote_api/remote_api.go

@@ -0,0 +1,152 @@
+// Copyright 2012 Google Inc. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+/*
+Package remote_api implements the /_ah/remote_api endpoint.
+This endpoint is used by offline tools such as the bulk loader.
+*/
+package remote_api // import "google.golang.org/appengine/remote_api"
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+
+	"github.com/golang/protobuf/proto"
+
+	"google.golang.org/appengine"
+	"google.golang.org/appengine/internal"
+	pb "google.golang.org/appengine/internal/remote_api"
+	"google.golang.org/appengine/log"
+	"google.golang.org/appengine/user"
+)
+
+func init() {
+	http.HandleFunc("/_ah/remote_api", handle)
+}
+
+func handle(w http.ResponseWriter, req *http.Request) {
+	c := appengine.NewContext(req)
+
+	u := user.Current(c)
+	if u == nil {
+		u, _ = user.CurrentOAuth(c,
+			"https://www.googleapis.com/auth/cloud-platform",
+			"https://www.googleapis.com/auth/appengine.apis",
+		)
+	}
+
+	if u == nil || !u.Admin {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.WriteHeader(http.StatusUnauthorized)
+		io.WriteString(w, "You must be logged in as an administrator to access this.\n")
+		return
+	}
+	if req.Header.Get("X-Appcfg-Api-Version") == "" {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.WriteHeader(http.StatusForbidden)
+		io.WriteString(w, "This request did not contain a necessary header.\n")
+		return
+	}
+
+	if req.Method != "POST" {
+		// Response must be YAML.
+		rtok := req.FormValue("rtok")
+		if rtok == "" {
+			rtok = "0"
+		}
+		w.Header().Set("Content-Type", "text/yaml; charset=utf-8")
+		fmt.Fprintf(w, `{app_id: %q, rtok: %q}`, internal.FullyQualifiedAppID(c), rtok)
+		return
+	}
+
+	defer req.Body.Close()
+	body, err := ioutil.ReadAll(req.Body)
+	if err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		log.Errorf(c, "Failed reading body: %v", err)
+		return
+	}
+	remReq := &pb.Request{}
+	if err := proto.Unmarshal(body, remReq); err != nil {
+		w.WriteHeader(http.StatusBadRequest)
+		log.Errorf(c, "Bad body: %v", err)
+		return
+	}
+
+	service, method := *remReq.ServiceName, *remReq.Method
+	if !requestSupported(service, method) {
+		w.WriteHeader(http.StatusBadRequest)
+		log.Errorf(c, "Unsupported RPC /%s.%s", service, method)
+		return
+	}
+
+	rawReq := &rawMessage{remReq.Request}
+	rawRes := &rawMessage{}
+	err = internal.Call(c, service, method, rawReq, rawRes)
+
+	remRes := &pb.Response{}
+	if err == nil {
+		remRes.Response = rawRes.buf
+	} else if ae, ok := err.(*internal.APIError); ok {
+		remRes.ApplicationError = &pb.ApplicationError{
+			Code:   &ae.Code,
+			Detail: &ae.Detail,
+		}
+	} else {
+		// This shouldn't normally happen.
+		log.Errorf(c, "appengine/remote_api: Unexpected error of type %T: %v", err, err)
+		remRes.ApplicationError = &pb.ApplicationError{
+			Code:   proto.Int32(0),
+			Detail: proto.String(err.Error()),
+		}
+	}
+	out, err := proto.Marshal(remRes)
+	if err != nil {
+		// This should not be possible.
+		w.WriteHeader(500)
+		log.Errorf(c, "proto.Marshal: %v", err)
+		return
+	}
+
+	log.Infof(c, "Spooling %d bytes of response to /%s.%s", len(out), service, method)
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.Header().Set("Content-Length", strconv.Itoa(len(out)))
+	w.Write(out)
+}
+
+// rawMessage is a protocol buffer type that is already serialised.
+// This allows the remote_api code here to handle messages
+// without having to know the real type.
+type rawMessage struct {
+	buf []byte
+}
+
+func (rm *rawMessage) Marshal() ([]byte, error) {
+	return rm.buf, nil
+}
+
+func (rm *rawMessage) Unmarshal(buf []byte) error {
+	rm.buf = make([]byte, len(buf))
+	copy(rm.buf, buf)
+	return nil
+}
+
+func requestSupported(service, method string) bool {
+	// This list of supported services is taken from SERVICE_PB_MAP in remote_api_services.py
+	switch service {
+	case "app_identity_service", "blobstore", "capability_service", "channel", "datastore_v3",
+		"datastore_v4", "file", "images", "logservice", "mail", "matcher", "memcache", "remote_datastore",
+		"remote_socket", "search", "modules", "system", "taskqueue", "urlfetch", "user", "xmpp":
+		return true
+	}
+	return false
+}
+
+// Methods to satisfy proto.Message.
+func (rm *rawMessage) Reset()         { rm.buf = nil }
+func (rm *rawMessage) String() string { return strconv.Quote(string(rm.buf)) }
+func (*rawMessage) ProtoMessage()     {}