Easier way to dnstap? (#1496)

* Easier way to dnstap?

* Remove unnecessary function parameter from Tapper

* golint

* golint 2

* Proxy dnstap tests

* README.md & doc

* net.IP

* Proxy test was incorrect

* Small changes

* Update README.md

* Was not reporting dnstap errors + test

* Wasn't working at all, now it's ok

* Thanks Travis
This commit is contained in:
varyoo 2018-03-01 03:19:01 +01:00 committed by Miek Gieben
parent f697b33283
commit 6bb08ffee4
12 changed files with 274 additions and 255 deletions

View file

@ -49,19 +49,43 @@ at Github: <https://github.com/dnstap/golang-dnstap>. It's written in Go.
The following command listens on the given socket and decodes messages to stdout.
~~~ sh
% dnstap -u /tmp/dnstap.sock
$ dnstap -u /tmp/dnstap.sock
~~~
The following command listens on the given socket and saves message payloads to a binary dnstap-format log file.
~~~ sh
% dnstap -u /tmp/dnstap.sock -w /tmp/test.dnstap
$ dnstap -u /tmp/dnstap.sock -w /tmp/test.dnstap
~~~
Listen for dnstap messages on port 6000.
~~~ sh
% dnstap -l 127.0.0.1:6000
$ dnstap -l 127.0.0.1:6000
~~~
## Using Dnstap in your plugin
~~~ Go
import (
"github.com/coredns/coredns/plugin/dnstap"
"github.com/coredns/coredns/plugin/dnstap/msg"
)
func (h Dnstap) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
// log client query to Dnstap
if t := dnstap.TapperFromContext(ctx); t != nil {
b := msg.New().Time(time.Now()).Addr(w.RemoteAddr())
if t.Pack() {
b.Msg(r)
}
if m, err := b.ToClientQuery(); err == nil {
t.TapMessage(m)
}
}
// ...
}
~~~
## See Also

View file

@ -1,11 +1,9 @@
package dnstap
import (
"fmt"
"io"
"time"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/dnstap/msg"
"github.com/coredns/coredns/plugin/dnstap/taprw"
tap "github.com/dnstap/golang-dnstap"
@ -17,7 +15,9 @@ import (
type Dnstap struct {
Next plugin.Handler
IO IORoutine
Pack bool
// Set to true to include the relevant raw DNS message into the dnstap messages.
JoinRawMessage bool
}
type (
@ -27,8 +27,8 @@ type (
}
// Tapper is implemented by the Context passed by the dnstap handler.
Tapper interface {
TapMessage(*tap.Message) error
TapBuilder() msg.Builder
TapMessage(message *tap.Message)
Pack() bool
}
tapContext struct {
context.Context
@ -50,24 +50,18 @@ func TapperFromContext(ctx context.Context) (t Tapper) {
return
}
func tapMessageTo(w io.Writer, m *tap.Message) error {
frame, err := msg.Marshal(m)
if err != nil {
return fmt.Errorf("marshal: %s", err)
}
_, err = w.Write(frame)
return err
}
// TapMessage implements Tapper.
func (h Dnstap) TapMessage(m *tap.Message) error {
h.IO.Dnstap(msg.Wrap(m))
return nil
func (h *Dnstap) TapMessage(m *tap.Message) {
t := tap.Dnstap_MESSAGE
h.IO.Dnstap(tap.Dnstap{
Type: &t,
Message: m,
})
}
// TapBuilder implements Tapper.
func (h Dnstap) TapBuilder() msg.Builder {
return msg.Builder{Full: h.Pack}
// Pack returns true if the raw DNS message should be included into the dnstap messages.
func (h Dnstap) Pack() bool {
return h.JoinRawMessage
}
// ServeDNS logs the client query and response to dnstap and passes the dnstap Context.
@ -78,8 +72,13 @@ func (h Dnstap) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
sendOption := taprw.SendOption{Cq: true, Cr: true}
newCtx := context.WithValue(ctx, DnstapSendOption, &sendOption)
rw := &taprw.ResponseWriter{ResponseWriter: w, Tapper: &h, Query: r, Send: &sendOption}
rw.SetQueryEpoch()
rw := &taprw.ResponseWriter{
ResponseWriter: w,
Tapper: &h,
Query: r,
Send: &sendOption,
QueryEpoch: time.Now(),
}
code, err := plugin.NextOrFailure(h.Name(), h.Next, tapContext{newCtx, h}, rw, r)
if err != nil {
@ -87,7 +86,7 @@ func (h Dnstap) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg)
return code, err
}
if err := rw.DnstapError(); err != nil {
if err = rw.DnstapError(); err != nil {
return code, plugin.Error("dnstap", err)
}

View file

@ -1,8 +1,12 @@
package dnstap
import (
"errors"
"net"
"strings"
"testing"
"github.com/coredns/coredns/plugin"
"github.com/coredns/coredns/plugin/dnstap/test"
mwtest "github.com/coredns/coredns/plugin/test"
@ -20,8 +24,8 @@ func testCase(t *testing.T, tapq, tapr *tap.Message, q, r *dns.Msg) {
return 0, w.WriteMsg(r)
}),
IO: &w,
Pack: false,
IO: &w,
JoinRawMessage: false,
}
_, err := h.ServeDNS(context.TODO(), &mwtest.ResponseWriter{}, q)
if err != nil {
@ -52,7 +56,62 @@ func TestDnstap(t *testing.T) {
mwtest.A("example.org. 3600 IN A 10.0.0.1"),
},
}.Msg()
tapq := test.TestingData().ToClientQuery()
tapr := test.TestingData().ToClientResponse()
tapq, _ := test.TestingData().ToClientQuery()
tapr, _ := test.TestingData().ToClientResponse()
testCase(t, tapq, tapr, q, r)
}
type noWriter struct {
}
func (n noWriter) Dnstap(d tap.Dnstap) {
}
func endWith(c int, err error) plugin.Handler {
return mwtest.HandlerFunc(func(_ context.Context, w dns.ResponseWriter, _ *dns.Msg) (int, error) {
w.WriteMsg(nil) // trigger plugin dnstap to log client query and response
// maybe dnstap should log the client query when no message is written...
return c, err
})
}
type badAddr struct {
}
func (bad badAddr) Network() string {
return "bad network"
}
func (bad badAddr) String() string {
return "bad address"
}
type badRW struct {
dns.ResponseWriter
}
func (bad *badRW) RemoteAddr() net.Addr {
return badAddr{}
}
func TestError(t *testing.T) {
h := Dnstap{
Next: endWith(0, nil),
IO: noWriter{},
JoinRawMessage: false,
}
rw := &badRW{&mwtest.ResponseWriter{}}
// the dnstap error will show only if there is no plugin error
_, err := h.ServeDNS(context.TODO(), rw, nil)
if err == nil || !strings.HasPrefix(err.Error(), "plugin/dnstap") {
t.Fatal("must return the dnstap error but have:", err)
}
// plugin errors will always overwrite dnstap errors
pluginErr := errors.New("plugin error")
h.Next = endWith(0, pluginErr)
_, err = h.ServeDNS(context.TODO(), rw, nil)
if err != pluginErr {
t.Fatal("must return the plugin error but have:", err)
}
}

View file

@ -1,157 +1,153 @@
// Package msg helps to build a dnstap Message.
package msg
import (
"errors"
"net"
"strconv"
"time"
tap "github.com/dnstap/golang-dnstap"
"github.com/miekg/dns"
)
// Builder helps to build Data by being aware of the dnstap plugin configuration.
// Builder helps to build a Dnstap message.
type Builder struct {
Full bool
Data
}
// AddrMsg parses the info of net.Addr and dns.Msg.
func (b *Builder) AddrMsg(a net.Addr, m *dns.Msg) (err error) {
err = b.RemoteAddr(a)
if err != nil {
return
}
return b.Msg(m)
}
// Msg parses the info of dns.Msg.
func (b *Builder) Msg(m *dns.Msg) (err error) {
if b.Full {
err = b.Pack(m)
}
return
}
// Data helps to build a dnstap Message.
// It can be transformed into the actual Message using this package.
type Data struct {
Packed []byte
SocketProto tap.SocketProtocol
SocketFam tap.SocketFamily
Address []byte
Address net.IP
Port uint32
TimeSec uint64
err error
}
// HostPort decodes into Data any string returned by dnsutil.ParseHostPortOrFile.
func (d *Data) HostPort(addr string) error {
// New returns a new Builder
func New() *Builder {
return &Builder{}
}
// Addr adds the remote address to the message.
func (b *Builder) Addr(remote net.Addr) *Builder {
if b.err != nil {
return b
}
switch addr := remote.(type) {
case *net.TCPAddr:
b.Address = addr.IP
b.Port = uint32(addr.Port)
b.SocketProto = tap.SocketProtocol_TCP
case *net.UDPAddr:
b.Address = addr.IP
b.Port = uint32(addr.Port)
b.SocketProto = tap.SocketProtocol_UDP
default:
b.err = errors.New("unknown remote address type")
return b
}
if b.Address.To4() != nil {
b.SocketFam = tap.SocketFamily_INET
} else {
b.SocketFam = tap.SocketFamily_INET6
}
return b
}
// Msg adds the raw DNS message to the dnstap message.
func (b *Builder) Msg(m *dns.Msg) *Builder {
if b.err != nil {
return b
}
b.Packed, b.err = m.Pack()
return b
}
// HostPort adds the remote address as encoded by dnsutil.ParseHostPortOrFile to the message.
func (b *Builder) HostPort(addr string) *Builder {
ip, port, err := net.SplitHostPort(addr)
if err != nil {
return err
b.err = err
return b
}
p, err := strconv.ParseUint(port, 10, 32)
if err != nil {
return err
b.err = err
return b
}
d.Port = uint32(p)
b.Port = uint32(p)
if ip := net.ParseIP(ip); ip != nil {
d.Address = []byte(ip)
b.Address = []byte(ip)
if ip := ip.To4(); ip != nil {
d.SocketFam = tap.SocketFamily_INET
b.SocketFam = tap.SocketFamily_INET
} else {
d.SocketFam = tap.SocketFamily_INET6
b.SocketFam = tap.SocketFamily_INET6
}
return nil
return b
}
return errors.New("not an ip address")
b.err = errors.New("not an ip address")
return b
}
// RemoteAddr parses the information about the remote address into Data.
func (d *Data) RemoteAddr(remote net.Addr) error {
switch addr := remote.(type) {
case *net.TCPAddr:
d.Address = addr.IP
d.Port = uint32(addr.Port)
d.SocketProto = tap.SocketProtocol_TCP
case *net.UDPAddr:
d.Address = addr.IP
d.Port = uint32(addr.Port)
d.SocketProto = tap.SocketProtocol_UDP
default:
return errors.New("unknown remote address type")
}
if a := net.IP(d.Address); a.To4() != nil {
d.SocketFam = tap.SocketFamily_INET
} else {
d.SocketFam = tap.SocketFamily_INET6
}
return nil
}
// Pack encodes the DNS message into Data.
func (d *Data) Pack(m *dns.Msg) error {
packed, err := m.Pack()
if err != nil {
return err
}
d.Packed = packed
return nil
// Time adds the timestamp to the message.
func (b *Builder) Time(ts time.Time) *Builder {
b.TimeSec = uint64(ts.Unix())
return b
}
// ToClientResponse transforms Data into a client response message.
func (d *Data) ToClientResponse() *tap.Message {
func (b *Builder) ToClientResponse() (*tap.Message, error) {
t := tap.Message_CLIENT_RESPONSE
return &tap.Message{
Type: &t,
SocketFamily: &d.SocketFam,
SocketProtocol: &d.SocketProto,
ResponseTimeSec: &d.TimeSec,
ResponseMessage: d.Packed,
QueryAddress: d.Address,
QueryPort: &d.Port,
}
SocketFamily: &b.SocketFam,
SocketProtocol: &b.SocketProto,
ResponseTimeSec: &b.TimeSec,
ResponseMessage: b.Packed,
QueryAddress: b.Address,
QueryPort: &b.Port,
}, b.err
}
// ToClientQuery transforms Data into a client query message.
func (d *Data) ToClientQuery() *tap.Message {
func (b *Builder) ToClientQuery() (*tap.Message, error) {
t := tap.Message_CLIENT_QUERY
return &tap.Message{
Type: &t,
SocketFamily: &d.SocketFam,
SocketProtocol: &d.SocketProto,
QueryTimeSec: &d.TimeSec,
QueryMessage: d.Packed,
QueryAddress: d.Address,
QueryPort: &d.Port,
}
SocketFamily: &b.SocketFam,
SocketProtocol: &b.SocketProto,
QueryTimeSec: &b.TimeSec,
QueryMessage: b.Packed,
QueryAddress: b.Address,
QueryPort: &b.Port,
}, b.err
}
// ToOutsideQuery transforms the data into a forwarder or resolver query message.
func (d *Data) ToOutsideQuery(t tap.Message_Type) *tap.Message {
func (b *Builder) ToOutsideQuery(t tap.Message_Type) (*tap.Message, error) {
return &tap.Message{
Type: &t,
SocketFamily: &d.SocketFam,
SocketProtocol: &d.SocketProto,
QueryTimeSec: &d.TimeSec,
QueryMessage: d.Packed,
ResponseAddress: d.Address,
ResponsePort: &d.Port,
}
SocketFamily: &b.SocketFam,
SocketProtocol: &b.SocketProto,
QueryTimeSec: &b.TimeSec,
QueryMessage: b.Packed,
ResponseAddress: b.Address,
ResponsePort: &b.Port,
}, b.err
}
// ToOutsideResponse transforms the data into a forwarder or resolver response message.
func (d *Data) ToOutsideResponse(t tap.Message_Type) *tap.Message {
func (b *Builder) ToOutsideResponse(t tap.Message_Type) (*tap.Message, error) {
return &tap.Message{
Type: &t,
SocketFamily: &d.SocketFam,
SocketProtocol: &d.SocketProto,
ResponseTimeSec: &d.TimeSec,
ResponseMessage: d.Packed,
ResponseAddress: d.Address,
ResponsePort: &d.Port,
}
SocketFamily: &b.SocketFam,
SocketProtocol: &b.SocketProto,
ResponseTimeSec: &b.TimeSec,
ResponseMessage: b.Packed,
ResponseAddress: b.Address,
ResponsePort: &b.Port,
}, b.err
}

View file

@ -12,12 +12,9 @@ import (
"github.com/miekg/dns"
)
func testRequest(t *testing.T, expected Data, r request.Request) {
d := Data{}
if err := d.RemoteAddr(r.W.RemoteAddr()); err != nil {
t.Fail()
return
}
func testRequest(t *testing.T, expected Builder, r request.Request) {
d := Builder{}
d.Addr(r.W.RemoteAddr())
if d.SocketProto != expected.SocketProto ||
d.SocketFam != expected.SocketFam ||
!reflect.DeepEqual(d.Address, expected.Address) ||
@ -27,7 +24,7 @@ func testRequest(t *testing.T, expected Data, r request.Request) {
}
}
func TestRequest(t *testing.T) {
testRequest(t, Data{
testRequest(t, Builder{
SocketProto: tap.SocketProtocol_UDP,
SocketFam: tap.SocketFamily_INET,
Address: net.ParseIP("10.240.0.1"),

View file

@ -1,28 +0,0 @@
package msg
import (
"fmt"
lib "github.com/dnstap/golang-dnstap"
"github.com/golang/protobuf/proto"
)
// Wrap a dnstap message in the top-level dnstap type.
func Wrap(m *lib.Message) lib.Dnstap {
t := lib.Dnstap_MESSAGE
return lib.Dnstap{
Type: &t,
Message: m,
}
}
// Marshal encodes the message to a binary dnstap payload.
func Marshal(m *lib.Message) (data []byte, err error) {
payload := Wrap(m)
data, err = proto.Marshal(&payload)
if err != nil {
err = fmt.Errorf("proto: %s", err)
return
}
return
}

View file

@ -66,7 +66,7 @@ func setup(c *caddy.Controller) error {
}
dio := dnstapio.New(conf.target, conf.socket)
dnstap := Dnstap{IO: dio, Pack: conf.full}
dnstap := Dnstap{IO: dio, JoinRawMessage: conf.full}
c.OnStartup(func() error {
dio.Connect()

View file

@ -21,67 +21,59 @@ type SendOption struct {
// Tapper is what ResponseWriter needs to log to dnstap.
type Tapper interface {
TapMessage(m *tap.Message) error
TapBuilder() msg.Builder
TapMessage(*tap.Message)
Pack() bool
}
// ResponseWriter captures the client response and logs the query to dnstap.
// Single request use.
// SendOption configures Dnstap to selectively send Dnstap messages. Default is send all.
type ResponseWriter struct {
queryEpoch uint64
QueryEpoch time.Time
Query *dns.Msg
dns.ResponseWriter
Tapper
err error
Send *SendOption
dnstapErr error
}
// DnstapError check if a dnstap error occurred during Write and returns it.
func (w ResponseWriter) DnstapError() error {
return w.err
}
// SetQueryEpoch sets the query epoch as reported by dnstap.
func (w *ResponseWriter) SetQueryEpoch() {
w.queryEpoch = uint64(time.Now().Unix())
func (w *ResponseWriter) DnstapError() error {
return w.dnstapErr
}
// WriteMsg writes back the response to the client and THEN works on logging the request
// and response to dnstap.
// Dnstap errors are to be checked by DnstapError.
func (w *ResponseWriter) WriteMsg(resp *dns.Msg) (writeErr error) {
writeErr = w.ResponseWriter.WriteMsg(resp)
writeEpoch := uint64(time.Now().Unix())
writeEpoch := time.Now()
b := w.TapBuilder()
b.TimeSec = w.queryEpoch
b := msg.New().Time(w.QueryEpoch).Addr(w.RemoteAddr())
if w.Send == nil || w.Send.Cq {
if err := func() (err error) {
err = b.AddrMsg(w.ResponseWriter.RemoteAddr(), w.Query)
if err != nil {
return
}
return w.TapMessage(b.ToClientQuery())
}(); err != nil {
w.err = fmt.Errorf("client query: %s", err)
// don't forget to call DnstapError later
if w.Pack() {
b.Msg(w.Query)
}
if m, err := b.ToClientQuery(); err != nil {
w.dnstapErr = fmt.Errorf("client query: %s", err)
} else {
w.TapMessage(m)
}
}
if w.Send == nil || w.Send.Cr {
if writeErr == nil {
if err := func() (err error) {
b.TimeSec = writeEpoch
if err = b.Msg(resp); err != nil {
return
}
return w.TapMessage(b.ToClientResponse())
}(); err != nil {
w.err = fmt.Errorf("client response: %s", err)
if w.Pack() {
b.Msg(resp)
}
if m, err := b.Time(writeEpoch).ToClientResponse(); err != nil {
w.dnstapErr = fmt.Errorf("client response: %s", err)
} else {
w.TapMessage(m)
}
}
}
return
return writeErr
}

View file

@ -1,41 +1,14 @@
package taprw
import (
"errors"
"testing"
"github.com/coredns/coredns/plugin/dnstap/msg"
"github.com/coredns/coredns/plugin/dnstap/test"
mwtest "github.com/coredns/coredns/plugin/test"
tap "github.com/dnstap/golang-dnstap"
"github.com/miekg/dns"
)
type TapFailer struct {
}
func (TapFailer) TapMessage(*tap.Message) error {
return errors.New("failed")
}
func (TapFailer) TapBuilder() msg.Builder {
return msg.Builder{Full: true}
}
func TestDnstapError(t *testing.T) {
rw := ResponseWriter{
Query: new(dns.Msg),
ResponseWriter: &mwtest.ResponseWriter{},
Tapper: TapFailer{},
}
if err := rw.WriteMsg(new(dns.Msg)); err != nil {
t.Errorf("dnstap error during Write: %s", err)
}
if rw.DnstapError() == nil {
t.Fatal("no dnstap error")
}
}
func testingMsg() (m *dns.Msg) {
m = new(dns.Msg)
m.SetQuestion("example.com.", dns.TypeA)
@ -69,12 +42,18 @@ func TestClientQueryResponse(t *testing.T) {
t.Fatalf("%d msg trapped", l)
return
}
want := d.ToClientQuery()
want, err := d.ToClientQuery()
if err != nil {
t.Fatal("Testing data must build", err)
}
have := trapper.Trap[0]
if !test.MsgEqual(want, have) {
t.Fatalf("query: want: %v\nhave: %v", want, have)
}
want = d.ToClientResponse()
want, err = d.ToClientResponse()
if err != nil {
t.Fatal("Testing data must build", err)
}
have = trapper.Trap[1]
if !test.MsgEqual(want, have) {
t.Fatalf("response: want: %v\nhave: %v", want, have)

View file

@ -17,8 +17,8 @@ type Context struct {
}
// TestingData returns the Data matching coredns/test.ResponseWriter.
func TestingData() (d *msg.Data) {
d = &msg.Data{
func TestingData() (d *msg.Builder) {
d = &msg.Builder{
SocketFam: tap.SocketFamily_INET,
SocketProto: tap.SocketProtocol_UDP,
Address: net.ParseIP("10.240.0.1"),
@ -68,13 +68,12 @@ type TrapTapper struct {
Full bool
}
// TapMessage adds the message to the trap.
func (t *TrapTapper) TapMessage(m *tap.Message) error {
t.Trap = append(t.Trap, m)
return nil
// Pack returns field Full.
func (t *TrapTapper) Pack() bool {
return t.Full
}
// TapBuilder returns a test msg.Builder.
func (t *TrapTapper) TapBuilder() msg.Builder {
return msg.Builder{Full: t.Full}
// TapMessage adds the message to the trap.
func (t *TrapTapper) TapMessage(m *tap.Message) {
t.Trap = append(t.Trap, m)
}

View file

@ -4,6 +4,7 @@ import (
"time"
"github.com/coredns/coredns/plugin/dnstap"
"github.com/coredns/coredns/plugin/dnstap/msg"
"github.com/coredns/coredns/request"
tap "github.com/dnstap/golang-dnstap"
@ -18,11 +19,7 @@ func toDnstap(ctx context.Context, host string, ex Exchanger, state request.Requ
}
// Query
b := tapper.TapBuilder()
b.TimeSec = uint64(start.Unix())
if err := b.HostPort(host); err != nil {
return err
}
b := msg.New().Time(start).HostPort(host)
t := ex.Transport()
if t == "" {
@ -34,21 +31,26 @@ func toDnstap(ctx context.Context, host string, ex Exchanger, state request.Requ
b.SocketProto = tap.SocketProtocol_UDP
}
if err := b.Msg(state.Req); err != nil {
return err
}
if err := tapper.TapMessage(b.ToOutsideQuery(tap.Message_FORWARDER_QUERY)); err != nil {
if tapper.Pack() {
b.Msg(state.Req)
}
m, err := b.ToOutsideQuery(tap.Message_FORWARDER_QUERY)
if err != nil {
return err
}
tapper.TapMessage(m)
// Response
if reply != nil {
b.TimeSec = uint64(time.Now().Unix())
if err := b.Msg(reply); err != nil {
if tapper.Pack() {
b.Msg(reply)
}
m, err := b.Time(time.Now()).ToOutsideResponse(tap.Message_FORWARDER_RESPONSE)
if err != nil {
return err
}
return tapper.TapMessage(b.ToOutsideResponse(tap.Message_FORWARDER_RESPONSE))
tapper.TapMessage(m)
}
return nil
}

View file

@ -14,9 +14,9 @@ import (
"golang.org/x/net/context"
)
func testCase(t *testing.T, ex Exchanger, q, r *dns.Msg, datq, datr *msg.Data) {
tapq := datq.ToOutsideQuery(tap.Message_FORWARDER_QUERY)
tapr := datr.ToOutsideResponse(tap.Message_FORWARDER_RESPONSE)
func testCase(t *testing.T, ex Exchanger, q, r *dns.Msg, datq, datr *msg.Builder) {
tapq, _ := datq.ToOutsideQuery(tap.Message_FORWARDER_QUERY)
tapr, _ := datr.ToOutsideResponse(tap.Message_FORWARDER_RESPONSE)
ctx := test.Context{}
err := toDnstap(&ctx, "10.240.0.1:40212", ex,
request.Request{W: &mwtest.ResponseWriter{}, Req: q}, r, time.Now())