TrueCloudLab/frostfs-http-gw
17
1
Fork 18
Code Issues 12 Pull requests 3 Projects Releases 16 Packages 1 Activity Actions

[#195] Add tags support
All checks were successful
/ DCO (pull_request) Successful in 37s
Details
/ Vulncheck (pull_request) Successful in 51s
Details
/ Builds (pull_request) Successful in 1m5s
Details
/ OCI image (pull_request) Successful in 1m29s
Details
/ Lint (pull_request) Successful in 1m51s
Details
/ Tests (pull_request) Successful in 52s
Details

Browse source 
Signed-off-by: Aleksey Kravchenko <al.kravchenko@yadro.com>
This commit is contained in:
Aleksey Kravchenko 2025-01-21 09:44:13 +03:00
parent 4b782cf124
commit a51358bb16
22 changed files with 577 additions and 311 deletions
Download patch file Download diff file Expand all files Collapse all files

4
internal/cache/buckets.go vendored
View file

@ -72,7 +72,7 @@ func (o *BucketCache) GetByCID(cnrID cid.ID) *data.BucketInfo {
key, ok := entry.(string)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", key)))
zap.String("expected", fmt.Sprintf("%T", key)), logs.TagField(logs.TagDatapath))
return nil
}
@ -88,7 +88,7 @@ func (o *BucketCache) get(key string) *data.BucketInfo {
result, ok := entry.(*data.BucketInfo)
if !ok {
o.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil
}

2
internal/cache/netmap.go vendored
View file

@ -53,7 +53,7 @@ func (c *NetmapCache) Get() *netmap.NetMap {
result, ok := entry.(netmap.NetMap)
if !ok {
c.logger.Warn(logs.InvalidCacheEntryType, zap.String("actual", fmt.Sprintf("%T", entry)),
zap.String("expected", fmt.Sprintf("%T", result)))
zap.String("expected", fmt.Sprintf("%T", result)), logs.TagField(logs.TagDatapath))
return nil
}

17
internal/handler/browse.go
View file

@ -230,7 +230,7 @@ func (h *Handler) getDirObjectsNative(ctx context.Context, bucketInfo *data.Buck
}
for objExt := range resp {
if objExt.Error != nil {
log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error))
log.Error(logs.FailedToHeadObject, zap.Error(objExt.Error), logs.TagField(logs.TagExternalStorage))
result.hasErrors = true
continue
}
@ -273,7 +273,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re
})
if err != nil {
wg.Done()
log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err))
log.Warn(logs.FailedToSumbitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
}
select {
case <-ctx.Done():
@ -283,7 +283,7 @@ func (h *Handler) headDirObjects(ctx context.Context, cnrID cid.ID, objectIDs Re
}
})
if err != nil {
log.Error(logs.FailedToIterateOverResponse, zap.Error(err))
log.Error(logs.FailedToIterateOverResponse, zap.Error(err), logs.TagField(logs.TagDatapath))
}
wg.Wait()
}()
@ -332,6 +332,11 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
const S3Protocol = "s3"
const FrostfsProtocol = "frostfs"
logTag := logs.TagExternalStorageTree
if p.isNative {
logTag = logs.TagExternalStorage
}
ctx := utils.GetContextFromRequest(c)
reqLog := utils.GetReqLogOrDefault(ctx, h.log)
log := reqLog.With(
@ -341,7 +346,7 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
)
resp, err := p.listObjects(ctx, p.bucketInfo, p.prefix)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logTag)
return
}
@ -360,7 +365,7 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
"parentDir": parentDir,
}).Parse(h.config.IndexPageTemplate())
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logTag)
return
}
bucketName := p.bucketInfo.Name
@ -376,7 +381,7 @@ func (h *Handler) browseObjects(c *fasthttp.RequestCtx, p browseParams) {
Protocol: protocol,
HasErrors: resp.hasErrors,
}); err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logTag)
return
}
}

20
internal/handler/download.go
View file

@ -36,13 +36,13 @@ func (h *Handler) DownloadByAddressOrBucketName(c *fasthttp.RequestCtx) {
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
logAndSendBucketError(c, log, checkS3Err)
logAndSendBucketError(c, log, checkS3Err, logs.TagExternalStorageTree)
return
}
@ -118,7 +118,8 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
prefix, err := url.QueryUnescape(prefix)
if err != nil {
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix), zap.Error(err))
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", scid), zap.String("prefix", prefix),
zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not unescape prefix: "+err.Error(), fasthttp.StatusBadRequest)
return
}
@ -127,13 +128,13 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
bktInfo, err := h.getBucketInfo(ctx, scid, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}
resSearch, err := h.search(ctx, bktInfo.CID, object.AttributeFilePath, prefix, object.MatchCommonPrefix)
if err != nil {
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage))
ResponseError(c, "could not search for objects: "+err.Error(), fasthttp.StatusBadRequest)
return
}
@ -165,19 +166,20 @@ func (h *Handler) DownloadZipped(c *fasthttp.RequestCtx) {
addr.SetObject(id)
if err = h.zipObject(ctx, zipWriter, addr, btoken, bufZip); err != nil {
log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err))
log.Error(logs.FailedToAddObjectToArchive, zap.String("oid", id.EncodeToString()), zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return false
})
if errIter != nil {
log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter))
log.Error(logs.IteratingOverSelectedObjectsFailed, zap.Error(errIter), logs.TagField(logs.TagExternalStorage))
} else if !called {
log.Error(logs.ObjectsNotFound)
log.Error(logs.ObjectsNotFound, logs.TagField(logs.TagExternalStorage))
}
if err = zipWriter.Close(); err != nil {
log.Error(logs.CloseZipWriter, zap.Error(err))
log.Error(logs.CloseZipWriter, zap.Error(err), logs.TagField(logs.TagDatapath))
}
})
}

3
internal/handler/filter.go
View file

@ -50,7 +50,8 @@ func filterHeaders(l *zap.Logger, header *fasthttp.RequestHeader) (map[string]st
l.Debug(logs.AddAttributeToResultObject,
zap.String("key", k),
zap.String("val", v))
zap.String("val", v),
logs.TagField(logs.TagDatapath))
})
return result, err

27
internal/handler/handler.go
View file

@ -206,11 +206,11 @@ func (h *Handler) byS3Path(ctx context.Context, req request, cnrID cid.ID, path
foundOID, err := h.tree.GetLatestVersion(ctx, &cnrID, path)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorageTree)
return
}
if foundOID.IsDeleteMarker {
log.Error(logs.ObjectWasDeleted)
log.Error(logs.ObjectWasDeleted, logs.TagField(logs.TagExternalStorageTree))
ResponseError(c, "object deleted", fasthttp.StatusNotFound)
return
}
@ -230,14 +230,16 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte
key, err := url.QueryUnescape(key)
if err != nil {
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key), zap.Error(err))
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_key", key),
zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not unescape attr_key: "+err.Error(), fasthttp.StatusBadRequest)
return
}
val, err = url.QueryUnescape(val)
if err != nil {
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val), zap.Error(err))
log.Error(logs.FailedToUnescapeQuery, zap.String("cid", cidParam), zap.String("attr_val", val),
zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not unescape attr_val: "+err.Error(), fasthttp.StatusBadRequest)
return
}
@ -246,7 +248,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}
@ -271,7 +273,7 @@ func (h *Handler) byAttribute(c *fasthttp.RequestCtx, handler func(context.Conte
func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cnrID cid.ID, attrKey, attrVal string) (oid.ID, error) {
res, err := h.search(ctx, cnrID, attrKey, attrVal, object.MatchStringEqual)
if err != nil {
log.Error(logs.CouldNotSearchForObjects, zap.Error(err))
log.Error(logs.CouldNotSearchForObjects, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return oid.ID{}, fmt.Errorf("could not search for objects: %w", err)
}
defer res.Close()
@ -282,13 +284,13 @@ func (h *Handler) findObjectByAttribute(ctx context.Context, log *zap.Logger, cn
if n == 0 {
switch {
case errors.Is(err, io.EOF) && h.needSearchByFileName(attrKey, attrVal):
log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName)
log.Debug(logs.ObjectNotFoundByFilePathTrySearchByFileName, logs.TagField(logs.TagExternalStorage))
return h.findObjectByAttribute(ctx, log, cnrID, attrFileName, attrVal)
case errors.Is(err, io.EOF):
log.Error(logs.ObjectNotFound, zap.Error(err))
log.Error(logs.ObjectNotFound, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return oid.ID{}, fmt.Errorf("object not found: %w", err)
default:
log.Error(logs.ReadObjectListFailed, zap.Error(err))
log.Error(logs.ReadObjectListFailed, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return oid.ID{}, fmt.Errorf("read object list failed: %w", err)
}
}
@ -342,7 +344,8 @@ func (h *Handler) getBucketInfo(ctx context.Context, containerName string, log *
log.Warn(logs.CouldntPutBucketIntoCache,
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
return bktInfo, nil
@ -386,13 +389,13 @@ func (h *Handler) browseIndex(c *fasthttp.RequestCtx, isNativeList bool) {
unescapedKey, err := url.QueryUnescape(oidURLParam)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}
bktInfo, err := h.getBucketInfo(ctx, cidURLParam, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}

9
internal/handler/head.go
View file

@ -67,7 +67,8 @@ func (h *Handler) headObject(ctx context.Context, req request, objectAddress oid
req.log.Info(logs.CouldntParseCreationDate,
zap.String("key", key),
zap.String("val", val),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagDatapath))
continue
}
req.Response.Header.Set(fasthttp.HeaderLastModified, time.Unix(value, 0).UTC().Format(http.TimeFormat))
@ -126,12 +127,12 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
bktInfo, err := h.getBucketInfo(ctx, cidParam, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorage)
return
}
checkS3Err := h.tree.CheckSettingsNodeExists(ctx, bktInfo)
if checkS3Err != nil && !errors.Is(checkS3Err, layer.ErrNodeNotFound) {
logAndSendBucketError(c, log, checkS3Err)
logAndSendBucketError(c, log, checkS3Err, logs.TagExternalStorageTree)
return
}
@ -143,7 +144,7 @@ func (h *Handler) HeadByAddressOrBucketName(c *fasthttp.RequestCtx) {
} else if err = objID.DecodeString(oidParam); err == nil {
h.byNativeAddress(ctx, req, bktInfo.CID, objID, h.headObject)
} else {
logAndSendBucketError(c, log, checkS3Err)
logAndSendBucketError(c, log, checkS3Err, logs.TagExternalStorageTree)
return
}
}

5
internal/handler/multipart.go
View file

@ -33,7 +33,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF
name := part.FormName()
if name == "" {
l.Debug(logs.IgnorePartEmptyFormName)
l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath))
continue
}
@ -41,8 +41,7 @@ func fetchMultipartFile(l *zap.Logger, r io.Reader, boundary string) (MultipartF
// ignore multipart/form-data values
if filename == "" {
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name))
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath))
continue
}

5
internal/handler/multipart_test.go
View file

@ -112,7 +112,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul
name := part.FormName()
if name == "" {
l.Debug(logs.IgnorePartEmptyFormName)
l.Debug(logs.IgnorePartEmptyFormName, logs.TagField(logs.TagDatapath))
continue
}
@ -120,8 +120,7 @@ func fetchMultipartFileDefault(l *zap.Logger, r io.Reader, boundary string) (Mul
// ignore multipart/form-data values
if filename == "" {
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name))
l.Debug(logs.IgnorePartEmptyFilename, zap.String("form", name), logs.TagField(logs.TagDatapath))
continue
}

5
internal/handler/reader.go
View file

@ -110,7 +110,8 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A
if err = req.setTimestamp(val); err != nil {
req.log.Error(logs.CouldntParseCreationDate,
zap.String("val", val),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
case object.AttributeContentType:
contentType = val
@ -144,7 +145,7 @@ func (h *Handler) receiveFile(ctx context.Context, req request, objAddress oid.A
return payload, nil
}, filename)
if err != nil && err != io.EOF {
req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err))
req.log.Error(logs.CouldNotDetectContentTypeFromPayload, zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(req.RequestCtx, "could not detect Content-Type from payload: "+err.Error(), fasthttp.StatusBadRequest)
return
}

15
internal/handler/upload.go
View file

@ -59,7 +59,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) {
bktInfo, err := h.getBucketInfo(ctx, scid, log)
if err != nil {
logAndSendBucketError(c, log, err)
logAndSendBucketError(c, log, err, logs.TagExternalStorageTree)
return
}
@ -74,19 +74,20 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) {
zap.Stringer("address", addr),
zap.String("filename", file.FileName()),
zap.Error(err),
logs.TagField(logs.TagDatapath),
)
}()
boundary := string(c.Request.Header.MultipartFormBoundary())
if file, err = fetchMultipartFile(log, bodyStream, boundary); err != nil {
log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err))
log.Error(logs.CouldNotReceiveMultipartForm, zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not receive multipart/form: "+err.Error(), fasthttp.StatusBadRequest)
return
}
filtered, err := filterHeaders(log, &c.Request.Header)
if err != nil {
log.Error(logs.CouldNotProcessHeaders, zap.Error(err))
log.Error(logs.CouldNotProcessHeaders, zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, err.Error(), fasthttp.StatusBadRequest)
return
}
@ -94,14 +95,15 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) {
now := time.Now()
if rawHeader := c.Request.Header.Peek(fasthttp.HeaderDate); rawHeader != nil {
if parsed, err := time.Parse(http.TimeFormat, string(rawHeader)); err != nil {
log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err))
log.Warn(logs.CouldNotParseClientTime, zap.String("Date header", string(rawHeader)), zap.Error(err),
logs.TagField(logs.TagDatapath))
} else {
now = parsed
}
}
if err = utils.PrepareExpirationHeader(c, h.frostfs, filtered, now); err != nil {
log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err))
log.Error(logs.CouldNotPrepareExpirationHeader, zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not prepare expiration header: "+err.Error(), fasthttp.StatusBadRequest)
return
}
@ -155,7 +157,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) {
// Try to return the response, otherwise, if something went wrong, throw an error.
if err = newPutResponse(addr).encode(c); err != nil {
log.Error(logs.CouldNotEncodeResponse, zap.Error(err))
log.Error(logs.CouldNotEncodeResponse, zap.Error(err), logs.TagField(logs.TagDatapath))
ResponseError(c, "could not encode response", fasthttp.StatusBadRequest)
return
@ -180,6 +182,7 @@ func (h *Handler) Upload(c *fasthttp.RequestCtx) {
func (h *Handler) handlePutFrostFSErr(r *fasthttp.RequestCtx, err error, log *zap.Logger) {
statusCode, msg, additionalFields := formErrorResponse("could not store file in frostfs", err)
logFields := append([]zap.Field{zap.Error(err)}, additionalFields...)
logFields = append(logFields, logs.TagField(logs.TagExternalStorage))
log.Error(logs.CouldNotStoreFileInFrostfs, logFields...)
ResponseError(r, msg, statusCode)

5
internal/handler/utils.go
View file

@ -28,6 +28,7 @@ func (r *request) handleFrostFSErr(err error, start time.Time) {
logFields := []zap.Field{
zap.Stringer("elapsed", time.Since(start)),
zap.Error(err),
logs.TagField(logs.TagExternalStorage),
}
statusCode, msg, additionalFields := formErrorResponse("could not receive object", err)
logFields = append(logFields, additionalFields...)
@ -77,8 +78,8 @@ func isValidValue(s string) bool {
return true
}
func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error) {
log.Error(logs.CouldntGetBucket, zap.Error(err))
func logAndSendBucketError(c *fasthttp.RequestCtx, log *zap.Logger, err error, tag string) {
log.Error(logs.CouldntGetBucket, zap.Error(err), logs.TagField(tag))
if client.IsErrContainerNotFound(err) {
ResponseError(c, "Not Found", fasthttp.StatusNotFound)

217
internal/logs/logs.go
View file

@ -1,97 +1,128 @@
package logs
import "go.uber.org/zap"
const (
CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/*
CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go
CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go
WrongObjectID = "wrong object id" // Error in ../../downloader/download.go
GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go
ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go
CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go
ObjectNotFound = "object not found" // Error in ../../downloader/download.go
ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go
FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go
ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go
CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go
ServiceIsRunning = "service is running" // Info in ../../metrics/service.go
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go
ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go
CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go
CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go
IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go
IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go
CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go
CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go
CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go
CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go
CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go
CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go
CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go
AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go
FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go
FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go
SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go
ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go
MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go
NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go
StartingApplication = "starting application" // Info in ../../app.go
StartingServer = "starting server" // Info in ../../app.go
ListenAndServe = "listen and serve" // Fatal in ../../app.go
ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go
FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go
LogLevelWontBeUpdated = "log level won't be updated" // Warn in ../../app.go
FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go
AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go
AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go
AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go
AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go
Request = "request" // Info in ../../app.go
CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go
FailedToAddServer = "failed to add server" // Warn in ../../app.go
AddServer = "add server" // Info in ../../app.go
NoHealthyServers = "no healthy servers" // Fatal in ../../app.go
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go
TracingConfigUpdated = "tracing config updated" // Info in ../../app.go
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go
UsingCredentials = "using credentials" // Info in ../../settings.go
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go
AddedStoragePeer = "added storage peer" // Info in ../../settings.go
CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go
CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go
FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go
FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go
FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../cmd/http-gw/settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../cmd/http-gw/settings.go
FailedToUnescapeQuery = "failed to unescape query"
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
ServerReconnecting = "reconnecting server..."
ServerReconnectedSuccessfully = "server reconnected successfully"
ServerReconnectFailed = "failed to reconnect server"
WarnDuplicateAddress = "duplicate address"
MultinetDialSuccess = "multinet dial successful"
MultinetDialFail = "multinet dial failed"
FailedToLoadMultinetConfig = "failed to load multinet config"
MultinetConfigWontBeUpdated = "multinet config won't be updated"
ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName"
CouldntCacheNetmap = "couldn't cache netmap"
TagFieldName = "tag"
TagApp = "app"
TagConfig = "config"
TagDatapath = "datapath"
TagExternalStorage = "external_storage"
TagExternalStorageTree = "external_storage_tree"
)
func TagField(tag string) zap.Field {
return zap.String(TagFieldName, tag)
}
const (
ServiceIsRunning = "service is running" // Info in ../../metrics/service.go
ServiceCouldntStartOnConfiguredPort = "service couldn't start on configured port" // Warn in ../../metrics/service.go
ServiceHasntStartedSinceItsDisabled = "service hasn't started since it's disabled" // Info in ../../metrics/service.go
ShuttingDownService = "shutting down service" // Info in ../../metrics/service.go
CantShutDownService = "can't shut down service" // Panic in ../../metrics/service.go
CantGracefullyShutDownService = "can't gracefully shut down service, force stop" // Error in ../../metrics/service.go
FailedToCreateResolver = "failed to create resolver" // Fatal in ../../app.go
FailedToCreateWorkerPool = "failed to create worker pool" // Fatal in ../../app.go
FailedToReadIndexPageTemplate = "failed to read index page template" // Error in ../../app.go
SetCustomIndexPageTemplate = "set custom index page template" // Info in ../../app.go
StartingApplication = "starting application" // Info in ../../app.go
StartingServer = "starting server" // Info in ../../app.go
ListenAndServe = "listen and serve" // Fatal in ../../app.go
ShuttingDownWebServer = "shutting down web server" // Info in ../../app.go
FailedToShutdownTracing = "failed to shutdown tracing" // Warn in ../../app.go
AddedPathUploadCid = "added path /upload/{cid}" // Info in ../../app.go
AddedPathGetCidOid = "added path /get/{cid}/{oid}" // Info in ../../app.go
AddedPathGetByAttributeCidAttrKeyAttrVal = "added path /get_by_attribute/{cid}/{attr_key}/{attr_val:*}" // Info in ../../app.go
AddedPathZipCidPrefix = "added path /zip/{cid}/{prefix}" // Info in ../../app.go
FailedToAddServer = "failed to add server" // Warn in ../../app.go
AddServer = "add server" // Info in ../../app.go
NoHealthyServers = "no healthy servers" // Fatal in ../../app.go
FailedToInitializeTracing = "failed to initialize tracing" // Warn in ../../app.go
RuntimeSoftMemoryDefinedWithGOMEMLIMIT = "soft runtime memory defined with GOMEMLIMIT environment variable, config value skipped" // Warn in ../../app.go
RuntimeSoftMemoryLimitUpdated = "soft runtime memory limit value updated" // Info in ../../app.go
CouldNotLoadFrostFSPrivateKey = "could not load FrostFS private key" // Fatal in ../../settings.go
UsingCredentials = "using credentials" // Info in ../../settings.go
FailedToCreateConnectionPool = "failed to create connection pool" // Fatal in ../../settings.go
FailedToDialConnectionPool = "failed to dial connection pool" // Fatal in ../../settings.go
FailedToCreateTreePool = "failed to create tree pool" // Fatal in ../../settings.go
FailedToDialTreePool = "failed to dial tree pool" // Fatal in ../../settings.go
ServerReconnecting = "reconnecting server..." // Info in ../../app.go
ServerReconnectedSuccessfully = "server reconnected successfully" // Info in ../../app.go
ServerReconnectFailed = "failed to reconnect server" // Warn in ../../app.go
)
const (
ContainerResolverWillBeDisabledBecauseOfResolversResolverOrderIsEmpty = "container resolver will be disabled because of resolvers 'resolver_order' is empty" // Info in ../../app.go
MetricsAreDisabled = "metrics are disabled" // Warn in ../../app.go
NoWalletPathSpecifiedCreatingEphemeralKeyAutomaticallyForThisRun = "no wallet path specified, creating ephemeral key automatically for this run" // Info in ../../app.go
SIGHUPConfigReloadStarted = "SIGHUP config reload started" // Info in ../../app.go
FailedToReloadConfigBecauseItsMissed = "failed to reload config because it's missed" // Warn in ../../app.go
FailedToReloadConfig = "failed to reload config" // Warn in ../../app.go
TagsLogConfigWontBeUpdated = "tags log config won't be updated" // Warn in ../../app.go
FailedToUpdateResolvers = "failed to update resolvers" // Warn in ../../app.go
FailedToReloadServerParameters = "failed to reload server parameters" // Warn in ../../app.go
SIGHUPConfigReloadCompleted = "SIGHUP config reload completed" // Info in ../../app.go
TracingConfigUpdated = "tracing config updated" // Info in ../../app.go
ResolverNNSWontBeUsedSinceRPCEndpointIsntProvided = "resolver nns won't be used since rpc_endpoint isn't provided" // Warn in ../../app.go
AddedStoragePeer = "added storage peer" // Info in ../../settings.go
InvalidLifetimeUsingDefaultValue = "invalid lifetime, using default value (in seconds)" // Error in ../../settings.go
InvalidCacheSizeUsingDefaultValue = "invalid cache size, using default value" // Error in ../../settings.go
WarnDuplicateAddress = "duplicate address" // Warn in ../../settings.go
FailedToLoadMultinetConfig = "failed to load multinet config" // Fatal in ../../settings.go
MultinetConfigWontBeUpdated = "multinet config won't be updated" // Warn in ../../app.go
)
const (
CouldntParseCreationDate = "couldn't parse creation date" // Info in ../../downloader/*
CouldNotDetectContentTypeFromPayload = "could not detect Content-Type from payload" // Error in ../../downloader/download.go
FailedToAddObjectToArchive = "failed to add object to archive" // Error in ../../downloader/download.go
CloseZipWriter = "close zip writer" // Error in ../../downloader/download.go
IgnorePartEmptyFormName = "ignore part, empty form name" // Debug in ../../uploader/upload.go
IgnorePartEmptyFilename = "ignore part, empty filename" // Debug in ../../uploader/upload.go
CloseTemporaryMultipartFormFile = "close temporary multipart/form file" // Debug in ../../uploader/upload.go
CouldNotProcessHeaders = "could not process headers" // Error in ../../uploader/upload.go
CouldNotParseClientTime = "could not parse client time" // Warn in ../../uploader/upload.go
CouldNotPrepareExpirationHeader = "could not prepare expiration header" // Error in ../../uploader/upload.go
CouldNotEncodeResponse = "could not encode response" // Error in ../../uploader/upload.go
AddAttributeToResultObject = "add attribute to result object" // Debug in ../../uploader/filter.go
Request = "request" // Info in ../../app.go
CouldNotFetchAndStoreBearerToken = "could not fetch and store bearer token" // Error in ../../app.go
CouldntPutBucketIntoCache = "couldn't put bucket info into cache" // Warn in ../handler/handler.go
FailedToSumbitTaskToPool = "failed to submit task to pool" // Error in ../handler/browse.go
FailedToIterateOverResponse = "failed to iterate over search response" // Error in ../handler/browse.go
InvalidCacheEntryType = "invalid cache entry type" // Warn in ../cache/buckets.go
FailedToUnescapeQuery = "failed to unescape query" // Error in ../handler/handler.go
MultinetDialSuccess = "multinet dial successful" // Debug in ../net/event_handler.go
MultinetDialFail = "multinet dial failed" // Debug in ../net/event_handler.go
CouldntCacheNetmap = "couldn't cache netmap" // Warn in ../service/frostfs/source.go
)
const (
CouldNotReceiveObject = "could not receive object" // Error in ../../downloader/download.go
CouldNotSearchForObjects = "could not search for objects" // Error in ../../downloader/download.go
ObjectNotFound = "object not found" // Error in ../../downloader/download.go
ReadObjectListFailed = "read object list failed" // Error in ../../downloader/download.go
IteratingOverSelectedObjectsFailed = "iterating over selected objects failed" // Error in ../../downloader/download.go
ObjectsNotFound = "objects not found" // Error in ../../downloader/download.go
CouldNotReceiveMultipartForm = "could not receive multipart/form" // Error in ../../uploader/upload.go
CouldNotStoreFileInFrostfs = "could not store file in frostfs" // Error in ../../uploader/upload.go
FailedToHeadObject = "failed to head object" // Error in ../handler/browse.go
ObjectNotFoundByFilePathTrySearchByFileName = "object not found by filePath attribute, try search by fileName" // Debug in ../handler/handler.go
)
const (
ObjectWasDeleted = "object was deleted" // Error in ../../downloader/download.go
CouldntGetBucket = "could not get bucket" // Error in ../handler/utils.go
)
const (
WrongObjectID = "wrong object id" // Error in ../../downloader/download.go
GetLatestObjectVersion = "get latest object version" // Error in ../../downloader/download.go
FailedToParseAddressInTreeNode = "failed to parse object addr in tree node"
SettingsNodeInvalidOwnerKey = "settings node: invalid owner key"
SystemNodeHasMultipleIDs = "system node has multiple ids"
FailedToRemoveOldSystemNode = "failed to remove old system node"
BucketSettingsNodeHasMultipleIDs = "bucket settings node has multiple ids"
)

6
internal/net/event_handler.go
View file

@ -17,9 +17,11 @@ func (l LogEventHandler) DialPerformed(sourceIP net.Addr, _, address string, err
sourceIPString = sourceIP.Network() + "://" + sourceIP.String()
}
if err == nil {
l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString), zap.String("destination", address))
l.logger.Debug(logs.MultinetDialSuccess, zap.String("source", sourceIPString),
zap.String("destination", address), logs.TagField(logs.TagDatapath))
} else {
l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString), zap.String("destination", address), zap.Error(err))
l.logger.Debug(logs.MultinetDialFail, zap.String("source", sourceIPString),
zap.String("destination", address), zap.Error(err), logs.TagField(logs.TagDatapath))
}
}

2
internal/service/frostfs/source.go
View file

@ -40,7 +40,7 @@ func (s *Source) NetMapSnapshot(ctx context.Context) (netmap.NetMap, error) {
}
if err = s.netmapCache.Put(netmapSnapshot); err != nil {
s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err))
s.log.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath))
}
return netmapSnapshot, nil