TrueCloudLab/distribution
13
0
Fork 3
Code Issues 3 Pull requests Projects Releases 2 Packages Wiki Activity Actions

Merge pull request #3240 from olegburov/bump-logrus-logstash-hook

Browse source 
Upgrade Logstash hook for Logrus to 1.0.0
This commit is contained in:
João Pereira 2021-02-21 13:09:51 +00:00 committed by GitHub
commit f89664f7d1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 157 additions and 289 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -9,7 +9,7 @@ require (
github.com/aws/aws-sdk-go v1.34.9
github.com/bitly/go-simplejson v0.5.0 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/bshuster-repo/logrus-logstash-hook v0.4.1
github.com/bshuster-repo/logrus-logstash-hook v1.0.0
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b // indirect
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 // indirect

4
go.sum
View file

@ -18,8 +18,8 @@ github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkN
github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1 h1:pgAtgj+A31JBVtEHu2uHuEx0n+2ukqUJnS2vVe5pQNA=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70=
github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd h1:rFt+Y/IK1aEZkEHchZRSq9OQbsSzIT/OrI8YFFmRIng=
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=

3
registry/registry.go
View file

@ -18,6 +18,7 @@ import (
"github.com/bugsnag/bugsnag-go"
"github.com/docker/go-metrics"
gorhandlers "github.com/gorilla/handlers"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/yvasiyarov/gorelic"
@ -285,7 +286,7 @@ func configureLogging(ctx context.Context, config *configuration.Configuration)
})
case "logstash":
log.SetFormatter(&logstash.LogstashFormatter{
TimestampFormat: time.RFC3339Nano,
Formatter: &logrus.JSONFormatter{TimestampFormat: time.RFC3339Nano},
})
default:
// just let the library use default on empty string.

13
vendor/github.com/bshuster-repo/logrus-logstash-hook/.travis.yml generated vendored
View file

@ -1,13 +1,10 @@
language: go
sudo: false
matrix:
include:
- go: 1.3
- go: 1.4
- go: 1.5
- go: 1.6
- go: tip
go:
- "1.11.x"
- "1.12.x"
- "tip"
install:
- # Skip
@ -15,5 +12,5 @@ install:
script:
- go get -t -v ./...
- diff -u <(echo -n) <(gofmt -d .)
- go tool vet .
- go vet .
- go test -v -race ./...

6
vendor/github.com/bshuster-repo/logrus-logstash-hook/CHANGELOG.md generated vendored
View file

@ -1,5 +1,11 @@
# Changelog
## 1.0
* Remove the old API: `NewConnWith`, `WithPrefix` and etc and move to a simple `New` function.
* Prefix is no longer supported in this package.
* Change the Hook structure to have only two members: `logrus.Formatter` and `io.Writer`.
## 0.4
* Update the name of the package from `logrus_logstash` to `logrustash`

74
vendor/github.com/bshuster-repo/logrus-logstash-hook/README.md generated vendored
View file

@ -1,30 +1,35 @@
# Logstash hook for logrus <img src="http://i.imgur.com/hTeVwmJ.png" width="40" height="40" alt=":walrus:" class="emoji" title=":walrus:" /> [![Build Status](https://travis-ci.org/bshuster-repo/logrus-logstash-hook.svg?branch=master)](https://travis-ci.org/bshuster-repo/logrus-logstash-hook)
Use this hook to send the logs to [Logstash](https://www.elastic.co/products/logstash) over both UDP and TCP.
# Logstash hook for logrus <img src="http://i.imgur.com/hTeVwmJ.png" width="40" height="40" alt=":walrus:" class="emoji" title=":walrus:" />
[![Build Status](https://travis-ci.org/bshuster-repo/logrus-logstash-hook.svg?branch=master)](https://travis-ci.org/bshuster-repo/logrus-logstash-hook)
[![Go Report Status](https://goreportcard.com/badge/github.com/bshuster-repo/logrus-logstash-hook)](https://goreportcard.com/report/github.com/bshuster-repo/logrus-logstash-hook)
## Usage
Use this hook to send the logs to [Logstash](https://www.elastic.co/products/logstash).
# Usage
```go
package main
import (
"github.com/sirupsen/logrus"
"github.com/bshuster-repo/logrus-logstash-hook"
"github.com/sirupsen/logrus"
"net"
)
func main() {
log := logrus.New()
hook, err := logrustash.NewHook("tcp", "172.17.0.2:9999", "myappName")
conn, err := net.Dial("tcp", "logstash.mycompany.net:8911")
if err != nil {
log.Fatal(err)
}
hook := logrustash.New(conn, logrustash.DefaultFormatter(logrus.Fields{"type": "myappName"}))
log.Hooks.Add(hook)
ctx := log.WithFields(logrus.Fields{
"method": "main",
})
...
ctx.Info("Hello World!")
}
```
This is how it will look like:
@ -41,61 +46,8 @@ This is how it will look like:
"type" => "myappName"
}
```
## Hook Fields
Fields can be added to the hook, which will always be in the log context.
This can be done when creating the hook:
```go
hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{
"hostname": os.Hostname(),
"serviceName": "myServiceName",
})
```
Or afterwards:
```go
hook.WithFields(logrus.Fields{
"hostname": os.Hostname(),
"serviceName": "myServiceName",
})
```
This allows you to set up the hook so logging is available immediately, and add important fields as they become available.
Single fields can be added/updated using 'WithField':
```go
hook.WithField("status", "running")
```
## Field prefix
The hook allows you to send logging to logstash and also retain the default std output in text format.
However to keep this console output readable some fields might need to be omitted from the default non-hooked log output.
Each hook can be configured with a prefix used to identify fields which are only to be logged to the logstash connection.
For example if you don't want to see the hostname and serviceName on each log line in the console output you can add a prefix:
```go
hook, err := logrustash.NewHookWithFields("tcp", "172.17.0.2:9999", "myappName", logrus.Fields{
"_hostname": os.Hostname(),
"_serviceName": "myServiceName",
})
...
hook.WithPrefix("_")
```
There are also constructors available which allow you to specify the prefix from the start.
The std-out will not have the '\_hostname' and '\_servicename' fields, and the logstash output will, but the prefix will be dropped from the name.
# Authors
# Maintainers
Name | Github | Twitter |
------------ | --------- | ---------- |

126
vendor/github.com/bshuster-repo/logrus-logstash-hook/hook.go generated vendored Normal file
View file

@ -0,0 +1,126 @@
package logrustash
import (
"io"
"sync"
"github.com/sirupsen/logrus"
)
// Hook represents a Logstash hook.
// It has two fields: writer to write the entry to Logstash and
// formatter to format the entry to a Logstash format before sending.
//
// To initialize it use the `New` function.
//
type Hook struct {
writer io.Writer
formatter logrus.Formatter
}
// New returns a new logrus.Hook for Logstash.
//
// To create a new hook that sends logs to `tcp://logstash.corp.io:9999`:
//
// conn, _ := net.Dial("tcp", "logstash.corp.io:9999")
// hook := logrustash.New(conn, logrustash.DefaultFormatter())
func New(w io.Writer, f logrus.Formatter) logrus.Hook {
return Hook{
writer: w,
formatter: f,
}
}
// Fire takes, formats and sends the entry to Logstash.
// Hook's formatter is used to format the entry into Logstash format
// and Hook's writer is used to write the formatted entry to the Logstash instance.
func (h Hook) Fire(e *logrus.Entry) error {
dataBytes, err := h.formatter.Format(e)
if err != nil {
return err
}
_, err = h.writer.Write(dataBytes)
return err
}
// Levels returns all logrus levels.
func (h Hook) Levels() []logrus.Level {
return logrus.AllLevels
}
// Using a pool to re-use of old entries when formatting Logstash messages.
// It is used in the Fire function.
var entryPool = sync.Pool{
New: func() interface{} {
return &logrus.Entry{}
},
}
// copyEntry copies the entry `e` to a new entry and then adds all the fields in `fields` that are missing in the new entry data.
// It uses `entryPool` to re-use allocated entries.
func copyEntry(e *logrus.Entry, fields logrus.Fields) *logrus.Entry {
ne := entryPool.Get().(*logrus.Entry)
ne.Message = e.Message
ne.Level = e.Level
ne.Time = e.Time
ne.Data = logrus.Fields{}
for k, v := range fields {
ne.Data[k] = v
}
for k, v := range e.Data {
ne.Data[k] = v
}
return ne
}
// releaseEntry puts the given entry back to `entryPool`. It must be called if copyEntry is called.
func releaseEntry(e *logrus.Entry) {
entryPool.Put(e)
}
// LogstashFormatter represents a Logstash format.
// It has logrus.Formatter which formats the entry and logrus.Fields which
// are added to the JSON message if not given in the entry data.
//
// Note: use the `DefaultFormatter` function to set a default Logstash formatter.
type LogstashFormatter struct {
logrus.Formatter
logrus.Fields
}
var (
logstashFields = logrus.Fields{"@version": "1", "type": "log"}
logstashFieldMap = logrus.FieldMap{
logrus.FieldKeyTime: "@timestamp",
logrus.FieldKeyMsg: "message",
}
)
// DefaultFormatter returns a default Logstash formatter:
// A JSON format with "@version" set to "1" (unless set differently in `fields`,
// "type" to "log" (unless set differently in `fields`),
// "@timestamp" to the log time and "message" to the log message.
//
// Note: to set a different configuration use the `LogstashFormatter` structure.
func DefaultFormatter(fields logrus.Fields) logrus.Formatter {
for k, v := range logstashFields {
if _, ok := fields[k]; !ok {
fields[k] = v
}
}
return LogstashFormatter{
Formatter: &logrus.JSONFormatter{FieldMap: logstashFieldMap},
Fields: fields,
}
}
// Format formats an entry to a Logstash format according to the given Formatter and Fields.
//
// Note: the given entry is copied and not changed during the formatting process.
func (f LogstashFormatter) Format(e *logrus.Entry) ([]byte, error) {
ne := copyEntry(e, f.Fields)
dataBytes, err := f.Formatter.Format(ne)
releaseEntry(ne)
return dataBytes, err
}

133
vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash.go generated vendored
View file

@ -1,133 +0,0 @@
package logrustash
import (
"net"
"strings"
"github.com/sirupsen/logrus"
)
// Hook represents a connection to a Logstash instance
type Hook struct {
conn net.Conn
appName string
alwaysSentFields logrus.Fields
hookOnlyPrefix string
TimeFormat string
}
// NewHook creates a new hook to a Logstash instance, which listens on
// `protocol`://`address`.
func NewHook(protocol, address, appName string) (*Hook, error) {
return NewHookWithFields(protocol, address, appName, make(logrus.Fields))
}
// NewHookWithConn creates a new hook to a Logstash instance, using the supplied connection
func NewHookWithConn(conn net.Conn, appName string) (*Hook, error) {
return NewHookWithFieldsAndConn(conn, appName, make(logrus.Fields))
}
// NewHookWithFields creates a new hook to a Logstash instance, which listens on
// `protocol`://`address`. alwaysSentFields will be sent with every log entry.
func NewHookWithFields(protocol, address, appName string, alwaysSentFields logrus.Fields) (*Hook, error) {
return NewHookWithFieldsAndPrefix(protocol, address, appName, alwaysSentFields, "")
}
// NewHookWithFieldsAndPrefix creates a new hook to a Logstash instance, which listens on
// `protocol`://`address`. alwaysSentFields will be sent with every log entry. prefix is used to select fields to filter
func NewHookWithFieldsAndPrefix(protocol, address, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) {
conn, err := net.Dial(protocol, address)
if err != nil {
return nil, err
}
return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, prefix)
}
// NewHookWithFieldsAndConn creates a new hook to a Logstash instance using the supplied connection
func NewHookWithFieldsAndConn(conn net.Conn, appName string, alwaysSentFields logrus.Fields) (*Hook, error) {
return NewHookWithFieldsAndConnAndPrefix(conn, appName, alwaysSentFields, "")
}
//NewHookWithFieldsAndConnAndPrefix creates a new hook to a Logstash instance using the suppolied connection and prefix
func NewHookWithFieldsAndConnAndPrefix(conn net.Conn, appName string, alwaysSentFields logrus.Fields, prefix string) (*Hook, error) {
return &Hook{conn: conn, appName: appName, alwaysSentFields: alwaysSentFields, hookOnlyPrefix: prefix}, nil
}
//NewFilterHook makes a new hook which does not forward to logstash, but simply enforces the prefix rules
func NewFilterHook() *Hook {
return NewFilterHookWithPrefix("")
}
//NewFilterHookWithPrefix make a new hook which does not forward to logstash, but simply enforces the specified prefix
func NewFilterHookWithPrefix(prefix string) *Hook {
return &Hook{conn: nil, appName: "", alwaysSentFields: make(logrus.Fields), hookOnlyPrefix: prefix}
}
func (h *Hook) filterHookOnly(entry *logrus.Entry) {
if h.hookOnlyPrefix != "" {
for key := range entry.Data {
if strings.HasPrefix(key, h.hookOnlyPrefix) {
delete(entry.Data, key)
}
}
}
}
//WithPrefix sets a prefix filter to use in all subsequent logging
func (h *Hook) WithPrefix(prefix string) {
h.hookOnlyPrefix = prefix
}
func (h *Hook) WithField(key string, value interface{}) {
h.alwaysSentFields[key] = value
}
func (h *Hook) WithFields(fields logrus.Fields) {
//Add all the new fields to the 'alwaysSentFields', possibly overwriting exising fields
for key, value := range fields {
h.alwaysSentFields[key] = value
}
}
func (h *Hook) Fire(entry *logrus.Entry) error {
//make sure we always clear the hookonly fields from the entry
defer h.filterHookOnly(entry)
// Add in the alwaysSentFields. We don't override fields that are already set.
for k, v := range h.alwaysSentFields {
if _, inMap := entry.Data[k]; !inMap {
entry.Data[k] = v
}
}
//For a filteringHook, stop here
if h.conn == nil {
return nil
}
formatter := LogstashFormatter{Type: h.appName}
if h.TimeFormat != "" {
formatter.TimestampFormat = h.TimeFormat
}
dataBytes, err := formatter.FormatWithPrefix(entry, h.hookOnlyPrefix)
if err != nil {
return err
}
if _, err = h.conn.Write(dataBytes); err != nil {
return err
}
return nil
}
func (h *Hook) Levels() []logrus.Level {
return []logrus.Level{
logrus.PanicLevel,
logrus.FatalLevel,
logrus.ErrorLevel,
logrus.WarnLevel,
logrus.InfoLevel,
logrus.DebugLevel,
}
}

81
vendor/github.com/bshuster-repo/logrus-logstash-hook/logstash_formatter.go generated vendored
View file

@ -1,81 +0,0 @@
package logrustash
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/sirupsen/logrus"
)
// Formatter generates json in logstash format.
// Logstash site: http://logstash.net/
type LogstashFormatter struct {
Type string // if not empty use for logstash type field.
// TimestampFormat sets the format used for timestamps.
TimestampFormat string
}
func (f *LogstashFormatter) Format(entry *logrus.Entry) ([]byte, error) {
return f.FormatWithPrefix(entry, "")
}
func (f *LogstashFormatter) FormatWithPrefix(entry *logrus.Entry, prefix string) ([]byte, error) {
fields := make(logrus.Fields)
for k, v := range entry.Data {
//remvove the prefix when sending the fields to logstash
if prefix != "" && strings.HasPrefix(k, prefix) {
k = strings.TrimPrefix(k, prefix)
}
switch v := v.(type) {
case error:
// Otherwise errors are ignored by `encoding/json`
// https://github.com/Sirupsen/logrus/issues/377
fields[k] = v.Error()
default:
fields[k] = v
}
}
fields["@version"] = "1"
timeStampFormat := f.TimestampFormat
if timeStampFormat == "" {
timeStampFormat = time.RFC3339
}
fields["@timestamp"] = entry.Time.Format(timeStampFormat)
// set message field
v, ok := entry.Data["message"]
if ok {
fields["fields.message"] = v
}
fields["message"] = entry.Message
// set level field
v, ok = entry.Data["level"]
if ok {
fields["fields.level"] = v
}
fields["level"] = entry.Level.String()
// set type field
if f.Type != "" {
v, ok = entry.Data["type"]
if ok {
fields["fields.type"] = v
}
fields["type"] = f.Type
}
serialized, err := json.Marshal(fields)
if err != nil {
return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err)
}
return append(serialized, '\n'), nil
}

2
vendor/modules.txt vendored
View file

@ -57,7 +57,7 @@ github.com/aws/aws-sdk-go/service/sts
github.com/aws/aws-sdk-go/service/sts/stsiface
# github.com/beorn7/perks v1.0.1
github.com/beorn7/perks/quantile
# github.com/bshuster-repo/logrus-logstash-hook v0.4.1
# github.com/bshuster-repo/logrus-logstash-hook v1.0.0
github.com/bshuster-repo/logrus-logstash-hook
# github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd
github.com/bugsnag/bugsnag-go