diff --git a/plugin/dnstap/README.md b/plugin/dnstap/README.md index 8b0716044..b90c45fc7 100644 --- a/plugin/dnstap/README.md +++ b/plugin/dnstap/README.md @@ -18,6 +18,7 @@ Every message is sent to the socket as soon as it comes in, the *dnstap* plugin dnstap SOCKET [full] { [identity IDENTITY] [version VERSION] + [extra EXTRA] [skipverify] } ~~~ @@ -26,6 +27,7 @@ dnstap SOCKET [full] { * `full` to include the wire-format DNS message. * **IDENTITY** to override the identity of the server. Defaults to the hostname. * **VERSION** to override the version field. Defaults to the CoreDNS version. +* **EXTRA** to define "extra" field in dnstap payload, [metadata](../metadata/) replacement available here. * `skipverify` to skip tls verification during connection. Default to be secure ## Examples @@ -63,6 +65,16 @@ dnstap /tmp/dnstap.sock { } ~~~ +Log to a socket, customize the "extra" field in dnstap payload. You may use metadata provided by other plugins in the extra field. + +~~~ txt +forward . 8.8.8.8 +metadata +dnstap /tmp/dnstap.sock { + extra "upstream: {/forward/upstream}" +} +~~~ + Log to a remote TLS endpoint. ~~~ txt @@ -124,7 +136,9 @@ And then in your plugin: ~~~ go import ( - github.com/coredns/coredns/plugin/dnstap/msg + "github.com/coredns/coredns/plugin/dnstap/msg" + "github.com/coredns/coredns/request" + tap "github.com/dnstap/golang-dnstap" ) @@ -138,7 +152,12 @@ func (x ExamplePlugin) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dn q.QueryMessage = buf } msg.SetType(q, tap.Message_CLIENT_QUERY) + + // if no metadata interpretation is needed, just send the message tapPlugin.TapMessage(q) + + // OR: to interpret the metadata in "extra" field, give more context info + tapPlugin.TapMessageWithMetadata(ctx, q, request.Request{W: w, Req: query}) } // ... } diff --git a/plugin/dnstap/handler.go b/plugin/dnstap/handler.go index a4b1acff0..59dbabab2 100644 --- a/plugin/dnstap/handler.go +++ b/plugin/dnstap/handler.go @@ -6,6 +6,8 @@ import ( "github.com/coredns/coredns/plugin" "github.com/coredns/coredns/plugin/dnstap/msg" + "github.com/coredns/coredns/plugin/pkg/replacer" + "github.com/coredns/coredns/request" tap "github.com/dnstap/golang-dnstap" "github.com/miekg/dns" @@ -15,20 +17,40 @@ import ( type Dnstap struct { Next plugin.Handler io tapper + repl replacer.Replacer // IncludeRawMessage will include the raw DNS message into the dnstap messages if true. IncludeRawMessage bool Identity []byte Version []byte + ExtraFormat string } -// TapMessage sends the message m to the dnstap interface. +// TapMessage sends the message m to the dnstap interface, without populating "Extra" field. func (h Dnstap) TapMessage(m *tap.Message) { - t := tap.Dnstap_MESSAGE - h.io.Dnstap(&tap.Dnstap{Type: &t, Message: m, Identity: h.Identity, Version: h.Version}) + if h.ExtraFormat == "" { + h.tapWithExtra(m, nil) + } else { + h.tapWithExtra(m, []byte(h.ExtraFormat)) + } } -func (h Dnstap) tapQuery(w dns.ResponseWriter, query *dns.Msg, queryTime time.Time) { +// TapMessageWithMetadata sends the message m to the dnstap interface, with "Extra" field being populated. +func (h Dnstap) TapMessageWithMetadata(ctx context.Context, m *tap.Message, state request.Request) { + if h.ExtraFormat == "" { + h.tapWithExtra(m, nil) + return + } + extraStr := h.repl.Replace(ctx, state, nil, h.ExtraFormat) + h.tapWithExtra(m, []byte(extraStr)) +} + +func (h Dnstap) tapWithExtra(m *tap.Message, extra []byte) { + t := tap.Dnstap_MESSAGE + h.io.Dnstap(&tap.Dnstap{Type: &t, Message: m, Identity: h.Identity, Version: h.Version, Extra: extra}) +} + +func (h Dnstap) tapQuery(ctx context.Context, w dns.ResponseWriter, query *dns.Msg, queryTime time.Time) { q := new(tap.Message) msg.SetQueryTime(q, queryTime) msg.SetQueryAddress(q, w.RemoteAddr()) @@ -38,7 +60,8 @@ func (h Dnstap) tapQuery(w dns.ResponseWriter, query *dns.Msg, queryTime time.Ti q.QueryMessage = buf } msg.SetType(q, tap.Message_CLIENT_QUERY) - h.TapMessage(q) + state := request.Request{W: w, Req: query} + h.TapMessageWithMetadata(ctx, q, state) } // ServeDNS logs the client query and response to dnstap and passes the dnstap Context. @@ -47,12 +70,13 @@ func (h Dnstap) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) ResponseWriter: w, Dnstap: h, query: r, + ctx: ctx, queryTime: time.Now(), } // The query tap message should be sent before sending the query to the // forwarder. Otherwise, the tap messages will come out out of order. - h.tapQuery(w, r, rw.queryTime) + h.tapQuery(ctx, w, r, rw.queryTime) return plugin.NextOrFailure(h.Name(), h.Next, ctx, rw, r) } diff --git a/plugin/dnstap/handler_test.go b/plugin/dnstap/handler_test.go index 2c54f70e6..cb492ac10 100644 --- a/plugin/dnstap/handler_test.go +++ b/plugin/dnstap/handler_test.go @@ -6,13 +6,14 @@ import ( "testing" "github.com/coredns/coredns/plugin/dnstap/msg" + "github.com/coredns/coredns/plugin/metadata" test "github.com/coredns/coredns/plugin/test" tap "github.com/dnstap/golang-dnstap" "github.com/miekg/dns" ) -func testCase(t *testing.T, tapq, tapr *tap.Message, q, r *dns.Msg) { +func testCase(t *testing.T, tapq, tapr *tap.Dnstap, q, r *dns.Msg, extraFormat string) { w := writer{t: t} w.queue = append(w.queue, tapq, tapr) h := Dnstap{ @@ -20,9 +21,17 @@ func testCase(t *testing.T, tapq, tapr *tap.Message, q, r *dns.Msg) { w dns.ResponseWriter, _ *dns.Msg) (int, error) { return 0, w.WriteMsg(r) }), - io: &w, + io: &w, + ExtraFormat: extraFormat, } - _, err := h.ServeDNS(context.TODO(), &test.ResponseWriter{}, q) + ctx := metadata.ContextWithMetadata(context.TODO()) + ok := metadata.SetValueFunc(ctx, "metadata/test", func() string { + return "MetadataValue" + }) + if !ok { + t.Fatal("Failed to set metadata") + } + _, err := h.ServeDNS(ctx, &test.ResponseWriter{}, q) if err != nil { t.Fatal(err) } @@ -30,7 +39,7 @@ func testCase(t *testing.T, tapq, tapr *tap.Message, q, r *dns.Msg) { type writer struct { t *testing.T - queue []*tap.Message + queue []*tap.Dnstap } func (w *writer) Dnstap(e *tap.Dnstap) { @@ -38,7 +47,7 @@ func (w *writer) Dnstap(e *tap.Dnstap) { w.t.Error("Message not expected") } - ex := w.queue[0] + ex := w.queue[0].Message got := e.Message if string(ex.QueryAddress) != string(got.QueryAddress) { @@ -53,6 +62,9 @@ func (w *writer) Dnstap(e *tap.Dnstap) { if *ex.SocketFamily != *got.SocketFamily { w.t.Errorf("Expected socket family %d, got %d", *ex.SocketFamily, *got.SocketFamily) } + if string(w.queue[0].Extra) != string(e.Extra) { + w.t.Errorf("Expected extra %s, got %s", w.queue[0].Extra, e.Extra) + } w.queue = w.queue[1:] } @@ -64,11 +76,29 @@ func TestDnstap(t *testing.T) { test.A("example.org. 3600 IN A 10.0.0.1"), }, }.Msg() - tapq := testMessage() // leave type unset for deepEqual - msg.SetType(tapq, tap.Message_CLIENT_QUERY) - tapr := testMessage() - msg.SetType(tapr, tap.Message_CLIENT_RESPONSE) - testCase(t, tapq, tapr, q, r) + + tapq := &tap.Dnstap{ + Message: testMessage(), + } + msg.SetType(tapq.Message, tap.Message_CLIENT_QUERY) + tapr := &tap.Dnstap{ + Message: testMessage(), + } + msg.SetType(tapr.Message, tap.Message_CLIENT_RESPONSE) + testCase(t, tapq, tapr, q, r, "") + + tapq_with_extra := &tap.Dnstap{ + Message: testMessage(), // leave type unset for deepEqual + Extra: []byte("extra_field_MetadataValue_A_example.org._IN_udp_29_10.240.0.1_40212_127.0.0.1"), + } + msg.SetType(tapq_with_extra.Message, tap.Message_CLIENT_QUERY) + tapr_with_extra := &tap.Dnstap{ + Message: testMessage(), + Extra: []byte("extra_field_MetadataValue_A_example.org._IN_udp_29_10.240.0.1_40212_127.0.0.1"), + } + msg.SetType(tapr_with_extra.Message, tap.Message_CLIENT_RESPONSE) + extraFormat := "extra_field_{/metadata/test}_{type}_{name}_{class}_{proto}_{size}_{remote}_{port}_{local}" + testCase(t, tapq_with_extra, tapr_with_extra, q, r, extraFormat) } func testMessage() *tap.Message { @@ -82,3 +112,25 @@ func testMessage() *tap.Message { QueryPort: &port, } } + +func TestTapMessage(t *testing.T) { + extraFormat := "extra_field_no_replacement_{/metadata/test}_{type}_{name}_{class}_{proto}_{size}_{remote}_{port}_{local}" + tapq := &tap.Dnstap{ + Message: testMessage(), + // extra field would not be replaced, since TapMessage won't pass context + Extra: []byte(extraFormat), + } + msg.SetType(tapq.Message, tap.Message_CLIENT_QUERY) + + w := writer{t: t} + w.queue = append(w.queue, tapq) + h := Dnstap{ + Next: test.HandlerFunc(func(_ context.Context, + w dns.ResponseWriter, r *dns.Msg) (int, error) { + return 0, w.WriteMsg(r) + }), + io: &w, + ExtraFormat: extraFormat, + } + h.TapMessage(tapq.Message) +} diff --git a/plugin/dnstap/setup.go b/plugin/dnstap/setup.go index c61c453fd..0186f4d51 100644 --- a/plugin/dnstap/setup.go +++ b/plugin/dnstap/setup.go @@ -9,6 +9,7 @@ import ( "github.com/coredns/coredns/core/dnsserver" "github.com/coredns/coredns/plugin" clog "github.com/coredns/coredns/plugin/pkg/log" + "github.com/coredns/coredns/plugin/pkg/replacer" ) var log = clog.NewWithPlugin("dnstap") @@ -21,6 +22,7 @@ func parseConfig(c *caddy.Controller) ([]*Dnstap, error) { for c.Next() { // directive name d := Dnstap{} endpoint := "" + d.repl = replacer.New() args := c.RemainingArgs() @@ -79,6 +81,13 @@ func parseConfig(c *caddy.Controller) ([]*Dnstap, error) { } d.Version = []byte(c.Val()) } + case "extra": + { + if !c.NextArg() { + return nil, c.ArgErr() + } + d.ExtraFormat = c.Val() + } } } dnstaps = append(dnstaps, &d) diff --git a/plugin/dnstap/setup_test.go b/plugin/dnstap/setup_test.go index 029a4a73b..83659638d 100644 --- a/plugin/dnstap/setup_test.go +++ b/plugin/dnstap/setup_test.go @@ -10,11 +10,12 @@ import ( ) type results struct { - endpoint string - full bool - proto string - identity []byte - version []byte + endpoint string + full bool + proto string + identity []byte + version []byte + extraFormat string } func TestConfig(t *testing.T) { @@ -24,27 +25,33 @@ func TestConfig(t *testing.T) { fail bool expect []results }{ - {"dnstap dnstap.sock full", false, []results{{"dnstap.sock", true, "unix", []byte(hostname), []byte("-")}}}, - {"dnstap unix://dnstap.sock", false, []results{{"dnstap.sock", false, "unix", []byte(hostname), []byte("-")}}}, - {"dnstap tcp://127.0.0.1:6000", false, []results{{"127.0.0.1:6000", false, "tcp", []byte(hostname), []byte("-")}}}, - {"dnstap tcp://[::1]:6000", false, []results{{"[::1]:6000", false, "tcp", []byte(hostname), []byte("-")}}}, - {"dnstap tcp://example.com:6000", false, []results{{"example.com:6000", false, "tcp", []byte(hostname), []byte("-")}}}, - {"dnstap", true, []results{{"fail", false, "tcp", []byte(hostname), []byte("-")}}}, - {"dnstap dnstap.sock full {\nidentity NAME\nversion VER\n}\n", false, []results{{"dnstap.sock", true, "unix", []byte("NAME"), []byte("VER")}}}, - {"dnstap dnstap.sock {\nidentity NAME\nversion VER\n}\n", false, []results{{"dnstap.sock", false, "unix", []byte("NAME"), []byte("VER")}}}, - {"dnstap {\nidentity NAME\nversion VER\n}\n", true, []results{{"fail", false, "tcp", []byte("NAME"), []byte("VER")}}}, + {"dnstap dnstap.sock full", false, []results{{"dnstap.sock", true, "unix", []byte(hostname), []byte("-"), ""}}}, + {"dnstap unix://dnstap.sock", false, []results{{"dnstap.sock", false, "unix", []byte(hostname), []byte("-"), ""}}}, + {"dnstap tcp://127.0.0.1:6000", false, []results{{"127.0.0.1:6000", false, "tcp", []byte(hostname), []byte("-"), ""}}}, + {"dnstap tcp://[::1]:6000", false, []results{{"[::1]:6000", false, "tcp", []byte(hostname), []byte("-"), ""}}}, + {"dnstap tcp://example.com:6000", false, []results{{"example.com:6000", false, "tcp", []byte(hostname), []byte("-"), ""}}}, + {"dnstap", true, []results{{"fail", false, "tcp", []byte(hostname), []byte("-"), ""}}}, + {"dnstap dnstap.sock full {\nidentity NAME\nversion VER\n}\n", false, []results{{"dnstap.sock", true, "unix", []byte("NAME"), []byte("VER"), ""}}}, + {"dnstap dnstap.sock full {\nidentity NAME\nversion VER\nextra EXTRA\n}\n", false, []results{{"dnstap.sock", true, "unix", []byte("NAME"), []byte("VER"), "EXTRA"}}}, + {"dnstap dnstap.sock {\nidentity NAME\nversion VER\nextra EXTRA\n}\n", false, []results{{"dnstap.sock", false, "unix", []byte("NAME"), []byte("VER"), "EXTRA"}}}, + {"dnstap {\nidentity NAME\nversion VER\nextra EXTRA\n}\n", true, []results{{"fail", false, "tcp", []byte("NAME"), []byte("VER"), "EXTRA"}}}, {`dnstap dnstap.sock full { identity NAME version VER + extra EXTRA } dnstap tcp://127.0.0.1:6000 { identity NAME2 version VER2 + extra EXTRA2 }`, false, []results{ - {"dnstap.sock", true, "unix", []byte("NAME"), []byte("VER")}, - {"127.0.0.1:6000", false, "tcp", []byte("NAME2"), []byte("VER2")}, + {"dnstap.sock", true, "unix", []byte("NAME"), []byte("VER"), "EXTRA"}, + {"127.0.0.1:6000", false, "tcp", []byte("NAME2"), []byte("VER2"), "EXTRA2"}, }}, - {"dnstap tls://127.0.0.1:6000", false, []results{{"127.0.0.1:6000", false, "tls", []byte(hostname), []byte("-")}}}, + {"dnstap tls://127.0.0.1:6000", false, []results{{"127.0.0.1:6000", false, "tls", []byte(hostname), []byte("-"), ""}}}, + {"dnstap dnstap.sock {\nidentity\n}\n", true, []results{{"dnstap.sock", false, "unix", []byte(hostname), []byte("-"), ""}}}, + {"dnstap dnstap.sock {\nversion\n}\n", true, []results{{"dnstap.sock", false, "unix", []byte(hostname), []byte("-"), ""}}}, + {"dnstap dnstap.sock {\nextra\n}\n", true, []results{{"dnstap.sock", false, "unix", []byte(hostname), []byte("-"), ""}}}, } for i, tc := range tests { c := caddy.NewTestController("dns", tc.in) @@ -75,6 +82,9 @@ func TestConfig(t *testing.T) { if x := string(tap.Version); x != string(tc.expect[i].version) { t.Errorf("Test %d: expected version %s, got %s", i, tc.expect[i].version, x) } + if x := tap.ExtraFormat; x != tc.expect[i].extraFormat { + t.Errorf("Test %d: expected extra format %s, got %s", i, tc.expect[i].extraFormat, x) + } } } } diff --git a/plugin/dnstap/writer.go b/plugin/dnstap/writer.go index 177263496..afd19ea5c 100644 --- a/plugin/dnstap/writer.go +++ b/plugin/dnstap/writer.go @@ -1,9 +1,11 @@ package dnstap import ( + "context" "time" "github.com/coredns/coredns/plugin/dnstap/msg" + "github.com/coredns/coredns/request" tap "github.com/dnstap/golang-dnstap" "github.com/miekg/dns" @@ -13,6 +15,7 @@ import ( type ResponseWriter struct { queryTime time.Time query *dns.Msg + ctx context.Context dns.ResponseWriter Dnstap } @@ -35,6 +38,7 @@ func (w *ResponseWriter) WriteMsg(resp *dns.Msg) error { } msg.SetType(r, tap.Message_CLIENT_RESPONSE) - w.TapMessage(r) + state := request.Request{W: w.ResponseWriter, Req: w.query} + w.TapMessageWithMetadata(w.ctx, r, state) return nil } diff --git a/plugin/forward/dnstap.go b/plugin/forward/dnstap.go index e9962d268..8195bb49d 100644 --- a/plugin/forward/dnstap.go +++ b/plugin/forward/dnstap.go @@ -1,6 +1,7 @@ package forward import ( + "context" "net" "strconv" "time" @@ -14,7 +15,7 @@ import ( ) // toDnstap will send the forward and received message to the dnstap plugin. -func toDnstap(f *Forward, host string, state request.Request, opts proxy.Options, reply *dns.Msg, start time.Time) { +func toDnstap(ctx context.Context, f *Forward, host string, state request.Request, opts proxy.Options, reply *dns.Msg, start time.Time) { h, p, _ := net.SplitHostPort(host) // this is preparsed and can't err here port, _ := strconv.ParseUint(p, 10, 32) // same here ip := net.ParseIP(h) @@ -45,7 +46,7 @@ func toDnstap(f *Forward, host string, state request.Request, opts proxy.Options q.QueryMessage = buf } msg.SetType(q, tap.Message_FORWARDER_QUERY) - t.TapMessage(q) + t.TapMessageWithMetadata(ctx, q, state) // Response if reply != nil { @@ -59,7 +60,7 @@ func toDnstap(f *Forward, host string, state request.Request, opts proxy.Options msg.SetResponseAddress(r, ta) msg.SetResponseTime(r, time.Now()) msg.SetType(r, tap.Message_FORWARDER_RESPONSE) - t.TapMessage(r) + t.TapMessageWithMetadata(ctx, r, state) } } } diff --git a/plugin/forward/forward.go b/plugin/forward/forward.go index b3df8330f..e53d74ae2 100644 --- a/plugin/forward/forward.go +++ b/plugin/forward/forward.go @@ -167,7 +167,7 @@ func (f *Forward) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg } if len(f.tapPlugins) != 0 { - toDnstap(f, proxy.Addr(), state, opts, ret, start) + toDnstap(ctx, f, proxy.Addr(), state, opts, ret, start) } upstreamErr = err diff --git a/plugin/pkg/replacer/replacer.go b/plugin/pkg/replacer/replacer.go index f927305c2..457244328 100644 --- a/plugin/pkg/replacer/replacer.go +++ b/plugin/pkg/replacer/replacer.go @@ -59,31 +59,6 @@ var labels = map[string]struct{}{ // appendValue appends the current value of label. func appendValue(b []byte, state request.Request, rr *dnstest.Recorder, label string) []byte { switch label { - case "{type}": - return append(b, state.Type()...) - case "{name}": - return append(b, state.Name()...) - case "{class}": - return append(b, state.Class()...) - case "{proto}": - return append(b, state.Proto()...) - case "{size}": - return strconv.AppendInt(b, int64(state.Req.Len()), 10) - case "{remote}": - return appendAddrToRFC3986(b, state.IP()) - case "{port}": - return append(b, state.Port()...) - case "{local}": - return appendAddrToRFC3986(b, state.LocalIP()) - // Header placeholders (case-insensitive). - case headerReplacer + "id}": - return strconv.AppendInt(b, int64(state.Req.Id), 10) - case headerReplacer + "opcode}": - return strconv.AppendInt(b, int64(state.Req.Opcode), 10) - case headerReplacer + "do}": - return strconv.AppendBool(b, state.Do()) - case headerReplacer + "bufsize}": - return strconv.AppendInt(b, int64(state.Size()), 10) // Recorded replacements. case "{rcode}": if rr == nil || rr.Msg == nil { @@ -109,6 +84,38 @@ func appendValue(b []byte, state request.Request, rr *dnstest.Recorder, label st return appendFlags(b, rr.Msg.MsgHdr) } return append(b, EmptyValue...) + } + + if (request.Request{}) == state { + return append(b, EmptyValue...) + } + + switch label { + case "{type}": + return append(b, state.Type()...) + case "{name}": + return append(b, state.Name()...) + case "{class}": + return append(b, state.Class()...) + case "{proto}": + return append(b, state.Proto()...) + case "{size}": + return strconv.AppendInt(b, int64(state.Req.Len()), 10) + case "{remote}": + return appendAddrToRFC3986(b, state.IP()) + case "{port}": + return append(b, state.Port()...) + case "{local}": + return appendAddrToRFC3986(b, state.LocalIP()) + // Header placeholders (case-insensitive). + case headerReplacer + "id}": + return strconv.AppendInt(b, int64(state.Req.Id), 10) + case headerReplacer + "opcode}": + return strconv.AppendInt(b, int64(state.Req.Opcode), 10) + case headerReplacer + "do}": + return strconv.AppendBool(b, state.Do()) + case headerReplacer + "bufsize}": + return strconv.AppendInt(b, int64(state.Size()), 10) default: return append(b, EmptyValue...) } diff --git a/plugin/pkg/replacer/replacer_test.go b/plugin/pkg/replacer/replacer_test.go index 28bb08d7a..aa8ac6fd2 100644 --- a/plugin/pkg/replacer/replacer_test.go +++ b/plugin/pkg/replacer/replacer_test.go @@ -256,6 +256,12 @@ func TestLabels(t *testing.T) { if repl != expect[lbl] { t.Errorf("Expected value %q, got %q", expect[lbl], repl) } + + // test empty state and nil recorder won't panic + repl_empty := replacer.Replace(ctx, request.Request{}, nil, lbl) + if repl_empty != EmptyValue { + t.Errorf("Expected empty value %q, got %q", EmptyValue, repl_empty) + } } }