Add request and response context to traces (#2162)

Automatically submitted.
This commit is contained in:
Nic Cope 2018-10-05 13:13:16 -07:00 committed by corbot[bot]
parent 1018a8267a
commit 8a9c6174fc
7 changed files with 719 additions and 13 deletions

3
Gopkg.lock generated
View file

@ -348,6 +348,7 @@
".",
"ext",
"log",
"mocktracer",
]
pruneopts = ""
revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
@ -812,9 +813,11 @@
"github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc",
"github.com/matttproud/golang_protobuf_extensions/pbutil",
"github.com/opentracing/opentracing-go",
"github.com/opentracing/opentracing-go/mocktracer",
"github.com/openzipkin/zipkin-go-opentracing",
"github.com/prometheus/client_model/go",
"github.com/prometheus/common/expfmt",
"golang.org/x/sys/unix",
"google.golang.org/grpc",
"google.golang.org/grpc/credentials",
"google.golang.org/grpc/grpclog",

View file

@ -10,8 +10,11 @@ import (
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/metrics"
"github.com/coredns/coredns/plugin/pkg/dnstest"
"github.com/coredns/coredns/plugin/pkg/rcode"
// Plugin the trace package.
_ "github.com/coredns/coredns/plugin/pkg/trace"
"github.com/coredns/coredns/request"
ddtrace "github.com/DataDog/dd-trace-go/opentracing"
"github.com/miekg/dns"
@ -19,6 +22,12 @@ import (
zipkin "github.com/openzipkin/zipkin-go-opentracing"
)
const (
tagName = "coredns.io/name"
tagType = "coredns.io/type"
tagRcode = "coredns.io/rcode"
)
type trace struct {
Next plugin.Handler
Endpoint string
@ -94,10 +103,26 @@ func (t *trace) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
trace = true
}
}
if span := ot.SpanFromContext(ctx); span == nil && trace {
span := t.Tracer().StartSpan("servedns:" + metrics.WithServer(ctx))
defer span.Finish()
ctx = ot.ContextWithSpan(ctx, span)
span := ot.SpanFromContext(ctx)
if !trace || span != nil {
return plugin.NextOrFailure(t.Name(), t.Next, ctx, w, r)
}
return plugin.NextOrFailure(t.Name(), t.Next, ctx, w, r)
req := request.Request{W: w, Req: r}
span = t.Tracer().StartSpan(spanName(ctx, req))
defer span.Finish()
rw := dnstest.NewRecorder(w)
ctx = ot.ContextWithSpan(ctx, span)
status, err := plugin.NextOrFailure(t.Name(), t.Next, ctx, rw, r)
span.SetTag(tagName, req.Name())
span.SetTag(tagType, req.Type())
span.SetTag(tagRcode, rcode.ToString(rw.Rcode))
return status, err
}
func spanName(ctx context.Context, req request.Request) string {
return "servedns:" + metrics.WithServer(ctx) + " " + req.Name()
}

View file

@ -1,20 +1,24 @@
package trace
import (
"context"
"testing"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/pkg/dnstest"
"github.com/coredns/coredns/plugin/pkg/rcode"
"github.com/coredns/coredns/plugin/test"
"github.com/coredns/coredns/request"
"github.com/mholt/caddy"
"github.com/miekg/dns"
"github.com/opentracing/opentracing-go/mocktracer"
)
// createTestTrace creates a trace plugin to be used in tests
func createTestTrace(config string) (*caddy.Controller, *trace, error) {
c := caddy.NewTestController("dns", config)
m, err := traceParse(c)
return c, m, err
}
const server = "coolServer"
func TestTrace(t *testing.T) {
_, m, err := createTestTrace(`trace`)
func TestStartup(t *testing.T) {
m, err := traceParse(caddy.NewTestController("dns", `trace`))
if err != nil {
t.Errorf("Error parsing test input: %s", err)
return
@ -31,3 +35,65 @@ func TestTrace(t *testing.T) {
t.Errorf("Error, no tracer created")
}
}
func TestTrace(t *testing.T) {
cases := []struct {
name string
rcode int
question *dns.Msg
server string
}{
{
name: "NXDOMAIN",
rcode: dns.RcodeNameError,
question: new(dns.Msg).SetQuestion("example.org.", dns.TypeA),
},
{
name: "NOERROR",
rcode: dns.RcodeSuccess,
question: new(dns.Msg).SetQuestion("example.net.", dns.TypeCNAME),
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
w := dnstest.NewRecorder(&test.ResponseWriter{})
m := mocktracer.New()
tr := &trace{
Next: test.HandlerFunc(func(_ context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
m := new(dns.Msg)
m.SetRcode(r, tc.rcode)
w.WriteMsg(m)
return tc.rcode, nil
}),
every: 1,
tracer: m,
}
ctx := context.WithValue(context.TODO(), plugin.ServerCtx{}, server)
if _, err := tr.ServeDNS(ctx, w, tc.question); err != nil {
t.Fatalf("Error during tr.ServeDNS(ctx, w, %v): %v", tc.question, err)
}
fs := m.FinishedSpans()
// Each trace consists of two spans; the root and the Next function.
if len(fs) != 2 {
t.Fatalf("Unexpected span count: len(fs): want 2, got %v", len(fs))
}
rootSpan := fs[1]
req := request.Request{W: w, Req: tc.question}
if rootSpan.OperationName != spanName(ctx, req) {
t.Errorf("Unexpected span name: rootSpan.Name: want %v, got %v", spanName(ctx, req), rootSpan.OperationName)
}
if rootSpan.Tag(tagName) != req.Name() {
t.Errorf("Unexpected span tag: rootSpan.Tag(%v): want %v, got %v", tagName, req.Name(), rootSpan.Tag(tagName))
}
if rootSpan.Tag(tagType) != req.Type() {
t.Errorf("Unexpected span tag: rootSpan.Tag(%v): want %v, got %v", tagType, req.Type(), rootSpan.Tag(tagType))
}
if rootSpan.Tag(tagRcode) != rcode.ToString(tc.rcode) {
t.Errorf("Unexpected span tag: rootSpan.Tag(%v): want %v, got %v", tagRcode, rcode.ToString(tc.rcode), rootSpan.Tag(tagRcode))
}
})
}
}

View file

@ -0,0 +1,105 @@
package mocktracer
import (
"fmt"
"reflect"
"time"
"github.com/opentracing/opentracing-go/log"
)
// MockLogRecord represents data logged to a Span via Span.LogFields or
// Span.LogKV.
type MockLogRecord struct {
Timestamp time.Time
Fields []MockKeyValue
}
// MockKeyValue represents a single key:value pair.
type MockKeyValue struct {
Key string
// All MockLogRecord values are coerced to strings via fmt.Sprint(), though
// we retain their type separately.
ValueKind reflect.Kind
ValueString string
}
// EmitString belongs to the log.Encoder interface
func (m *MockKeyValue) EmitString(key, value string) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitBool belongs to the log.Encoder interface
func (m *MockKeyValue) EmitBool(key string, value bool) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitInt belongs to the log.Encoder interface
func (m *MockKeyValue) EmitInt(key string, value int) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitInt32 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitInt32(key string, value int32) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitInt64 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitInt64(key string, value int64) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitUint32 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitUint32(key string, value uint32) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitUint64 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitUint64(key string, value uint64) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitFloat32 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitFloat32(key string, value float32) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitFloat64 belongs to the log.Encoder interface
func (m *MockKeyValue) EmitFloat64(key string, value float64) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitObject belongs to the log.Encoder interface
func (m *MockKeyValue) EmitObject(key string, value interface{}) {
m.Key = key
m.ValueKind = reflect.TypeOf(value).Kind()
m.ValueString = fmt.Sprint(value)
}
// EmitLazyLogger belongs to the log.Encoder interface
func (m *MockKeyValue) EmitLazyLogger(value log.LazyLogger) {
var meta MockKeyValue
value(&meta)
m.Key = meta.Key
m.ValueKind = meta.ValueKind
m.ValueString = meta.ValueString
}

View file

@ -0,0 +1,282 @@
package mocktracer
import (
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/opentracing/opentracing-go"
"github.com/opentracing/opentracing-go/ext"
"github.com/opentracing/opentracing-go/log"
)
// MockSpanContext is an opentracing.SpanContext implementation.
//
// It is entirely unsuitable for production use, but appropriate for tests
// that want to verify tracing behavior in other frameworks/applications.
//
// By default all spans have Sampled=true flag, unless {"sampling.priority": 0}
// tag is set.
type MockSpanContext struct {
TraceID int
SpanID int
Sampled bool
Baggage map[string]string
}
var mockIDSource = uint32(42)
func nextMockID() int {
return int(atomic.AddUint32(&mockIDSource, 1))
}
// ForeachBaggageItem belongs to the SpanContext interface
func (c MockSpanContext) ForeachBaggageItem(handler func(k, v string) bool) {
for k, v := range c.Baggage {
if !handler(k, v) {
break
}
}
}
// WithBaggageItem creates a new context with an extra baggage item.
func (c MockSpanContext) WithBaggageItem(key, value string) MockSpanContext {
var newBaggage map[string]string
if c.Baggage == nil {
newBaggage = map[string]string{key: value}
} else {
newBaggage = make(map[string]string, len(c.Baggage)+1)
for k, v := range c.Baggage {
newBaggage[k] = v
}
newBaggage[key] = value
}
// Use positional parameters so the compiler will help catch new fields.
return MockSpanContext{c.TraceID, c.SpanID, c.Sampled, newBaggage}
}
// MockSpan is an opentracing.Span implementation that exports its internal
// state for testing purposes.
type MockSpan struct {
sync.RWMutex
ParentID int
OperationName string
StartTime time.Time
FinishTime time.Time
// All of the below are protected by the embedded RWMutex.
SpanContext MockSpanContext
tags map[string]interface{}
logs []MockLogRecord
tracer *MockTracer
}
func newMockSpan(t *MockTracer, name string, opts opentracing.StartSpanOptions) *MockSpan {
tags := opts.Tags
if tags == nil {
tags = map[string]interface{}{}
}
traceID := nextMockID()
parentID := int(0)
var baggage map[string]string
sampled := true
if len(opts.References) > 0 {
traceID = opts.References[0].ReferencedContext.(MockSpanContext).TraceID
parentID = opts.References[0].ReferencedContext.(MockSpanContext).SpanID
sampled = opts.References[0].ReferencedContext.(MockSpanContext).Sampled
baggage = opts.References[0].ReferencedContext.(MockSpanContext).Baggage
}
spanContext := MockSpanContext{traceID, nextMockID(), sampled, baggage}
startTime := opts.StartTime
if startTime.IsZero() {
startTime = time.Now()
}
return &MockSpan{
ParentID: parentID,
OperationName: name,
StartTime: startTime,
tags: tags,
logs: []MockLogRecord{},
SpanContext: spanContext,
tracer: t,
}
}
// Tags returns a copy of tags accumulated by the span so far
func (s *MockSpan) Tags() map[string]interface{} {
s.RLock()
defer s.RUnlock()
tags := make(map[string]interface{})
for k, v := range s.tags {
tags[k] = v
}
return tags
}
// Tag returns a single tag
func (s *MockSpan) Tag(k string) interface{} {
s.RLock()
defer s.RUnlock()
return s.tags[k]
}
// Logs returns a copy of logs accumulated in the span so far
func (s *MockSpan) Logs() []MockLogRecord {
s.RLock()
defer s.RUnlock()
logs := make([]MockLogRecord, len(s.logs))
copy(logs, s.logs)
return logs
}
// Context belongs to the Span interface
func (s *MockSpan) Context() opentracing.SpanContext {
return s.SpanContext
}
// SetTag belongs to the Span interface
func (s *MockSpan) SetTag(key string, value interface{}) opentracing.Span {
s.Lock()
defer s.Unlock()
if key == string(ext.SamplingPriority) {
if v, ok := value.(uint16); ok {
s.SpanContext.Sampled = v > 0
return s
}
if v, ok := value.(int); ok {
s.SpanContext.Sampled = v > 0
return s
}
}
s.tags[key] = value
return s
}
// SetBaggageItem belongs to the Span interface
func (s *MockSpan) SetBaggageItem(key, val string) opentracing.Span {
s.Lock()
defer s.Unlock()
s.SpanContext = s.SpanContext.WithBaggageItem(key, val)
return s
}
// BaggageItem belongs to the Span interface
func (s *MockSpan) BaggageItem(key string) string {
s.RLock()
defer s.RUnlock()
return s.SpanContext.Baggage[key]
}
// Finish belongs to the Span interface
func (s *MockSpan) Finish() {
s.Lock()
s.FinishTime = time.Now()
s.Unlock()
s.tracer.recordSpan(s)
}
// FinishWithOptions belongs to the Span interface
func (s *MockSpan) FinishWithOptions(opts opentracing.FinishOptions) {
s.Lock()
s.FinishTime = opts.FinishTime
s.Unlock()
// Handle any late-bound LogRecords.
for _, lr := range opts.LogRecords {
s.logFieldsWithTimestamp(lr.Timestamp, lr.Fields...)
}
// Handle (deprecated) BulkLogData.
for _, ld := range opts.BulkLogData {
if ld.Payload != nil {
s.logFieldsWithTimestamp(
ld.Timestamp,
log.String("event", ld.Event),
log.Object("payload", ld.Payload))
} else {
s.logFieldsWithTimestamp(
ld.Timestamp,
log.String("event", ld.Event))
}
}
s.tracer.recordSpan(s)
}
// String allows printing span for debugging
func (s *MockSpan) String() string {
return fmt.Sprintf(
"traceId=%d, spanId=%d, parentId=%d, sampled=%t, name=%s",
s.SpanContext.TraceID, s.SpanContext.SpanID, s.ParentID,
s.SpanContext.Sampled, s.OperationName)
}
// LogFields belongs to the Span interface
func (s *MockSpan) LogFields(fields ...log.Field) {
s.logFieldsWithTimestamp(time.Now(), fields...)
}
// The caller MUST NOT hold s.Lock
func (s *MockSpan) logFieldsWithTimestamp(ts time.Time, fields ...log.Field) {
lr := MockLogRecord{
Timestamp: ts,
Fields: make([]MockKeyValue, len(fields)),
}
for i, f := range fields {
outField := &(lr.Fields[i])
f.Marshal(outField)
}
s.Lock()
defer s.Unlock()
s.logs = append(s.logs, lr)
}
// LogKV belongs to the Span interface.
//
// This implementations coerces all "values" to strings, though that is not
// something all implementations need to do. Indeed, a motivated person can and
// probably should have this do a typed switch on the values.
func (s *MockSpan) LogKV(keyValues ...interface{}) {
if len(keyValues)%2 != 0 {
s.LogFields(log.Error(fmt.Errorf("Non-even keyValues len: %v", len(keyValues))))
return
}
fields, err := log.InterleavedKVToFields(keyValues...)
if err != nil {
s.LogFields(log.Error(err), log.String("function", "LogKV"))
return
}
s.LogFields(fields...)
}
// LogEvent belongs to the Span interface
func (s *MockSpan) LogEvent(event string) {
s.LogFields(log.String("event", event))
}
// LogEventWithPayload belongs to the Span interface
func (s *MockSpan) LogEventWithPayload(event string, payload interface{}) {
s.LogFields(log.String("event", event), log.Object("payload", payload))
}
// Log belongs to the Span interface
func (s *MockSpan) Log(data opentracing.LogData) {
panic("MockSpan.Log() no longer supported")
}
// SetOperationName belongs to the Span interface
func (s *MockSpan) SetOperationName(operationName string) opentracing.Span {
s.Lock()
defer s.Unlock()
s.OperationName = operationName
return s
}
// Tracer belongs to the Span interface
func (s *MockSpan) Tracer() opentracing.Tracer {
return s.tracer
}

View file

@ -0,0 +1,105 @@
package mocktracer
import (
"sync"
"github.com/opentracing/opentracing-go"
)
// New returns a MockTracer opentracing.Tracer implementation that's intended
// to facilitate tests of OpenTracing instrumentation.
func New() *MockTracer {
t := &MockTracer{
finishedSpans: []*MockSpan{},
injectors: make(map[interface{}]Injector),
extractors: make(map[interface{}]Extractor),
}
// register default injectors/extractors
textPropagator := new(TextMapPropagator)
t.RegisterInjector(opentracing.TextMap, textPropagator)
t.RegisterExtractor(opentracing.TextMap, textPropagator)
httpPropagator := &TextMapPropagator{HTTPHeaders: true}
t.RegisterInjector(opentracing.HTTPHeaders, httpPropagator)
t.RegisterExtractor(opentracing.HTTPHeaders, httpPropagator)
return t
}
// MockTracer is only intended for testing OpenTracing instrumentation.
//
// It is entirely unsuitable for production use, but appropriate for tests
// that want to verify tracing behavior in other frameworks/applications.
type MockTracer struct {
sync.RWMutex
finishedSpans []*MockSpan
injectors map[interface{}]Injector
extractors map[interface{}]Extractor
}
// FinishedSpans returns all spans that have been Finish()'ed since the
// MockTracer was constructed or since the last call to its Reset() method.
func (t *MockTracer) FinishedSpans() []*MockSpan {
t.RLock()
defer t.RUnlock()
spans := make([]*MockSpan, len(t.finishedSpans))
copy(spans, t.finishedSpans)
return spans
}
// Reset clears the internally accumulated finished spans. Note that any
// extant MockSpans will still append to finishedSpans when they Finish(),
// even after a call to Reset().
func (t *MockTracer) Reset() {
t.Lock()
defer t.Unlock()
t.finishedSpans = []*MockSpan{}
}
// StartSpan belongs to the Tracer interface.
func (t *MockTracer) StartSpan(operationName string, opts ...opentracing.StartSpanOption) opentracing.Span {
sso := opentracing.StartSpanOptions{}
for _, o := range opts {
o.Apply(&sso)
}
return newMockSpan(t, operationName, sso)
}
// RegisterInjector registers injector for given format
func (t *MockTracer) RegisterInjector(format interface{}, injector Injector) {
t.injectors[format] = injector
}
// RegisterExtractor registers extractor for given format
func (t *MockTracer) RegisterExtractor(format interface{}, extractor Extractor) {
t.extractors[format] = extractor
}
// Inject belongs to the Tracer interface.
func (t *MockTracer) Inject(sm opentracing.SpanContext, format interface{}, carrier interface{}) error {
spanContext, ok := sm.(MockSpanContext)
if !ok {
return opentracing.ErrInvalidCarrier
}
injector, ok := t.injectors[format]
if !ok {
return opentracing.ErrUnsupportedFormat
}
return injector.Inject(spanContext, carrier)
}
// Extract belongs to the Tracer interface.
func (t *MockTracer) Extract(format interface{}, carrier interface{}) (opentracing.SpanContext, error) {
extractor, ok := t.extractors[format]
if !ok {
return nil, opentracing.ErrUnsupportedFormat
}
return extractor.Extract(carrier)
}
func (t *MockTracer) recordSpan(span *MockSpan) {
t.Lock()
defer t.Unlock()
t.finishedSpans = append(t.finishedSpans, span)
}

View file

@ -0,0 +1,120 @@
package mocktracer
import (
"fmt"
"net/url"
"strconv"
"strings"
"github.com/opentracing/opentracing-go"
)
const mockTextMapIdsPrefix = "mockpfx-ids-"
const mockTextMapBaggagePrefix = "mockpfx-baggage-"
var emptyContext = MockSpanContext{}
// Injector is responsible for injecting SpanContext instances in a manner suitable
// for propagation via a format-specific "carrier" object. Typically the
// injection will take place across an RPC boundary, but message queues and
// other IPC mechanisms are also reasonable places to use an Injector.
type Injector interface {
// Inject takes `SpanContext` and injects it into `carrier`. The actual type
// of `carrier` depends on the `format` passed to `Tracer.Inject()`.
//
// Implementations may return opentracing.ErrInvalidCarrier or any other
// implementation-specific error if injection fails.
Inject(ctx MockSpanContext, carrier interface{}) error
}
// Extractor is responsible for extracting SpanContext instances from a
// format-specific "carrier" object. Typically the extraction will take place
// on the server side of an RPC boundary, but message queues and other IPC
// mechanisms are also reasonable places to use an Extractor.
type Extractor interface {
// Extract decodes a SpanContext instance from the given `carrier`,
// or (nil, opentracing.ErrSpanContextNotFound) if no context could
// be found in the `carrier`.
Extract(carrier interface{}) (MockSpanContext, error)
}
// TextMapPropagator implements Injector/Extractor for TextMap and HTTPHeaders formats.
type TextMapPropagator struct {
HTTPHeaders bool
}
// Inject implements the Injector interface
func (t *TextMapPropagator) Inject(spanContext MockSpanContext, carrier interface{}) error {
writer, ok := carrier.(opentracing.TextMapWriter)
if !ok {
return opentracing.ErrInvalidCarrier
}
// Ids:
writer.Set(mockTextMapIdsPrefix+"traceid", strconv.Itoa(spanContext.TraceID))
writer.Set(mockTextMapIdsPrefix+"spanid", strconv.Itoa(spanContext.SpanID))
writer.Set(mockTextMapIdsPrefix+"sampled", fmt.Sprint(spanContext.Sampled))
// Baggage:
for baggageKey, baggageVal := range spanContext.Baggage {
safeVal := baggageVal
if t.HTTPHeaders {
safeVal = url.QueryEscape(baggageVal)
}
writer.Set(mockTextMapBaggagePrefix+baggageKey, safeVal)
}
return nil
}
// Extract implements the Extractor interface
func (t *TextMapPropagator) Extract(carrier interface{}) (MockSpanContext, error) {
reader, ok := carrier.(opentracing.TextMapReader)
if !ok {
return emptyContext, opentracing.ErrInvalidCarrier
}
rval := MockSpanContext{0, 0, true, nil}
err := reader.ForeachKey(func(key, val string) error {
lowerKey := strings.ToLower(key)
switch {
case lowerKey == mockTextMapIdsPrefix+"traceid":
// Ids:
i, err := strconv.Atoi(val)
if err != nil {
return err
}
rval.TraceID = i
case lowerKey == mockTextMapIdsPrefix+"spanid":
// Ids:
i, err := strconv.Atoi(val)
if err != nil {
return err
}
rval.SpanID = i
case lowerKey == mockTextMapIdsPrefix+"sampled":
b, err := strconv.ParseBool(val)
if err != nil {
return err
}
rval.Sampled = b
case strings.HasPrefix(lowerKey, mockTextMapBaggagePrefix):
// Baggage:
if rval.Baggage == nil {
rval.Baggage = make(map[string]string)
}
safeVal := val
if t.HTTPHeaders {
// unescape errors are ignored, nothing can be done
if rawVal, err := url.QueryUnescape(val); err == nil {
safeVal = rawVal
}
}
rval.Baggage[lowerKey[len(mockTextMapBaggagePrefix):]] = safeVal
}
return nil
})
if rval.TraceID == 0 || rval.SpanID == 0 {
return emptyContext, opentracing.ErrSpanContextNotFound
}
if err != nil {
return emptyContext, err
}
return rval, nil
}