realloc/frostfs-s3-gw
1
0
Fork 0
forked from TrueCloudLab/frostfs-s3-gw
Code Issues Pull requests Projects Releases Packages Wiki Activity

[#606] Support log tagging

Browse source 
Signed-off-by: Nikita Zinkevich <n.zinkevich@yadro.com>
This commit is contained in:
Nikita Zinkevich 2025-02-06 13:18:16 +03:00
parent ffac62e8b4
commit e7f620f137
40 changed files with 850 additions and 497 deletions
Download patch file Download diff file Expand all files Collapse all files

6
api/handler/acl.go
View file

@ -99,7 +99,7 @@ func (h *handler) encodePrivateCannedACL(ctx context.Context, bktInfo *data.Buck
ownerEncodedID := ownerDisplayName
if settings.OwnerKey == nil {
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()))
h.reqLogger(ctx).Warn(logs.BucketOwnerKeyIsMissing, zap.String("owner", bktInfo.Owner.String()), logs.TagField(logs.TagDatapath))
} else {
ownerDisplayName = settings.OwnerKey.Address()
ownerEncodedID = hex.EncodeToString(settings.OwnerKey.Bytes())
@ -150,7 +150,7 @@ func (h *handler) putBucketACLAPEHandler(w http.ResponseWriter, r *http.Request,
defer func() {
if errBody := r.Body.Close(); errBody != nil {
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody))
h.reqLogger(ctx).Warn(logs.CouldNotCloseRequestBody, zap.Error(errBody), logs.TagField(logs.TagDatapath))
}
}()
@ -382,7 +382,7 @@ func (h *handler) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request)
h.logAndSendError(ctx, w, "could not convert s3 policy to native chain policy", reqInfo, err)
return
} else {
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules)
h.reqLogger(ctx).Warn(logs.PolicyCouldntBeConvertedToNativeRules, logs.TagField(logs.TagDatapath))
}
chainsToSave := []*chain.Chain{s3Chain}

2
api/handler/copy.go
View file

@ -244,7 +244,7 @@ func (h *handler) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
}
}
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID))
h.reqLogger(ctx).Info(logs.ObjectIsCopied, zap.Stringer("object_id", dstObjInfo.ID), logs.TagField(logs.TagExternalStorage))
if dstEncryptionParams.Enabled() {
addSSECHeaders(w.Header(), r.Header)

4
api/handler/cors.go
View file

@ -108,13 +108,13 @@ func (h *handler) AppendCORSHeaders(w http.ResponseWriter, r *http.Request) {
}
bktInfo, err := h.getBucketInfo(ctx, reqInfo.BucketName)
if err != nil {
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err))
h.reqLogger(ctx).Warn(logs.GetBucketInfo, zap.Error(err), logs.TagField(logs.TagExternalBlockchain))
return
}
cors, err := h.obj.GetBucketCORS(ctx, bktInfo, h.cfg.NewXMLDecoder)
if err != nil {
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err))
h.reqLogger(ctx).Warn(logs.GetBucketCors, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
return
}

4
api/handler/get.go
View file

@ -296,12 +296,12 @@ func parseConditionalHeaders(headers http.Header, log *zap.Logger) *conditionalA
if httpTime, err := parseHTTPTime(headers.Get(api.IfModifiedSince)); err == nil {
args.IfModifiedSince = httpTime
} else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err))
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfModifiedSince, headers.Get(api.IfModifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
args.IfUnmodifiedSince = httpTime
} else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
}
return args

2
api/handler/patch.go
View file

@ -142,7 +142,7 @@ func parsePatchConditionalHeaders(headers http.Header, log *zap.Logger) *conditi
if httpTime, err := parseHTTPTime(headers.Get(api.IfUnmodifiedSince)); err == nil {
args.IfUnmodifiedSince = httpTime
} else {
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err))
log.Warn(logs.FailedToParseHTTPTime, zap.String(api.IfUnmodifiedSince, headers.Get(api.IfUnmodifiedSince)), zap.Error(err), logs.TagField(logs.TagDatapath))
}
return args

6
api/handler/put.go
View file

@ -466,7 +466,7 @@ func (h *handler) isTLSCheckRequired(r *http.Request) bool {
tlsTermination, err := strconv.ParseBool(tlsTerminationStr)
if err != nil {
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err))
h.reqLogger(r.Context()).Warn(logs.WarnInvalidTypeTLSTerminationHeader, zap.String("header", tlsTerminationStr), zap.Error(err), logs.TagField(logs.TagDatapath))
return true
}
@ -828,7 +828,7 @@ func (h *handler) createBucketHandlerPolicy(w http.ResponseWriter, r *http.Reque
h.logAndSendError(ctx, w, "could not create bucket", reqInfo, err)
return
}
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID))
h.reqLogger(ctx).Info(logs.BucketIsCreated, zap.Stringer("container_id", bktInfo.CID), logs.TagField(logs.TagExternalStorage))
chains := bucketCannedACLToAPERules(cannedACL, reqInfo, bktInfo.CID)
if err = h.ape.SaveACLChains(bktInfo.CID.EncodeToString(), chains); err != nil {
@ -1041,7 +1041,7 @@ func isLockEnabled(log *zap.Logger, header http.Header) bool {
lockEnabled, err := strconv.ParseBool(lockEnabledStr)
if err != nil {
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err))
log.Warn(logs.InvalidBucketObjectLockEnabledHeader, zap.String("header", lockEnabledStr), zap.Error(err), logs.TagField(logs.TagDatapath))
}
return lockEnabled

5
api/handler/util.go
View file

@ -39,9 +39,10 @@ func (h *handler) logAndSendError(ctx context.Context, w http.ResponseWriter, lo
zap.String("object", reqInfo.ObjectName),
zap.String("description", logText),
zap.String("user", reqInfo.User),
zap.Error(err)}
zap.Error(err),
}
fields = append(fields, additional...)
h.reqLogger(ctx).Error(logs.RequestFailed, fields...)
h.reqLogger(ctx).Error(logs.RequestFailed, append(fields, logs.TagField(logs.TagDatapath))...)
}
func handleDeleteMarker(w http.ResponseWriter, err error) error {

45
api/layer/cache.go
View file

@ -54,7 +54,7 @@ func DefaultCachesConfigs(logger *zap.Logger) *CachesConfig {
func NewCache(cfg *CachesConfig) *Cache {
return &Cache{
logger: cfg.Logger,
logger: cfg.Logger.With(logs.TagField(logs.TagDatapath)),
listsCache: cache.NewObjectsListCache(cfg.ObjectsList),
sessionListCache: cache.NewListSessionCache(cfg.SessionList),
objCache: cache.New(cfg.Objects),
@ -76,7 +76,8 @@ func (c *Cache) PutBucket(bktInfo *data.BucketInfo) {
zap.String("zone", bktInfo.Zone),
zap.String("bucket name", bktInfo.Name),
zap.Stringer("bucket cid", bktInfo.CID),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
}
@ -118,11 +119,12 @@ func (c *Cache) PutObject(owner user.ID, extObjInfo *data.ExtendedObjectInfo) {
if err := c.objCache.PutObject(extObjInfo); err != nil {
c.logger.Warn(logs.CouldntAddObjectToCache, zap.Error(err),
zap.String("object_name", extObjInfo.ObjectInfo.Name), zap.String("bucket_name", extObjInfo.ObjectInfo.Bucket),
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()))
zap.String("cid", extObjInfo.ObjectInfo.CID.EncodeToString()), zap.String("oid", extObjInfo.ObjectInfo.ID.EncodeToString()),
logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, extObjInfo.ObjectInfo.Address().EncodeToString()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -132,7 +134,8 @@ func (c *Cache) PutObjectWithName(owner user.ID, extObjInfo *data.ExtendedObject
if err := c.namesCache.Put(extObjInfo.ObjectInfo.NiceName(), extObjInfo.ObjectInfo.Address()); err != nil {
c.logger.Warn(logs.CouldntPutObjAddressToNameCache,
zap.String("obj nice name", extObjInfo.ObjectInfo.NiceName()),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagDatapath))
}
}
@ -146,11 +149,11 @@ func (c *Cache) GetList(owner user.ID, key cache.ObjectsListKey) []*data.NodeVer
func (c *Cache) PutList(owner user.ID, key cache.ObjectsListKey, list []*data.NodeVersion) {
if err := c.listsCache.PutVersions(key, list); err != nil {
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err))
c.logger.Warn(logs.CouldntCacheListOfObjects, zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key.String()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -164,11 +167,11 @@ func (c *Cache) GetListSession(owner user.ID, key cache.ListSessionKey) *data.Li
func (c *Cache) PutListSession(owner user.ID, key cache.ListSessionKey, session *data.ListSession) {
if err := c.sessionListCache.PutListSession(key, session); err != nil {
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err))
c.logger.Warn(logs.CouldntCacheListSession, zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key.String()); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -187,11 +190,11 @@ func (c *Cache) GetTagging(owner user.ID, key string) map[string]string {
func (c *Cache) PutTagging(owner user.ID, key string, tags map[string]string) {
if err := c.systemCache.PutTagging(key, tags); err != nil {
c.logger.Error(logs.CouldntCacheTags, zap.Error(err))
c.logger.Error(logs.CouldntCacheTags, zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -209,11 +212,11 @@ func (c *Cache) GetLockInfo(owner user.ID, key string) *data.LockInfo {
func (c *Cache) PutLockInfo(owner user.ID, key string, lockInfo *data.LockInfo) {
if err := c.systemCache.PutLockInfo(key, lockInfo); err != nil {
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err))
c.logger.Error(logs.CouldntCacheLockInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -230,11 +233,11 @@ func (c *Cache) GetSettings(owner user.ID, bktInfo *data.BucketInfo) *data.Bucke
func (c *Cache) PutSettings(owner user.ID, bktInfo *data.BucketInfo, settings *data.BucketSettings) {
key := bktInfo.Name + bktInfo.SettingsObjectName()
if err := c.systemCache.PutSettings(key, settings); err != nil {
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheBucketSettings, zap.String("bucket", bktInfo.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -252,11 +255,11 @@ func (c *Cache) PutCORS(owner user.ID, bkt *data.BucketInfo, cors *data.CORSConf
key := bkt.CORSObjectName()
if err := c.systemCache.PutCORS(key, cors); err != nil {
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheCors, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -278,11 +281,11 @@ func (c *Cache) PutLifecycleConfiguration(owner user.ID, bkt *data.BucketInfo, c
key := bkt.LifecycleConfigurationObjectName()
if err := c.systemCache.PutLifecycleConfiguration(key, cfg); err != nil {
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err))
c.logger.Warn(logs.CouldntCacheLifecycleConfiguration, zap.String("bucket", bkt.Name), zap.Error(err), logs.TagField(logs.TagDatapath))
}
if err := c.accessCache.Put(owner, key); err != nil {
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err))
c.logger.Warn(logs.CouldntCacheAccessControlOperation, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -296,7 +299,7 @@ func (c *Cache) GetNetworkInfo() *netmap.NetworkInfo {
func (c *Cache) PutNetworkInfo(info netmap.NetworkInfo) {
if err := c.networkCache.PutNetworkInfo(info); err != nil {
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err))
c.logger.Warn(logs.CouldntCacheNetworkInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}
@ -306,7 +309,7 @@ func (c *Cache) GetNetmap() *netmap.NetMap {
func (c *Cache) PutNetmap(nm netmap.NetMap) {
if err := c.networkCache.PutNetmap(nm); err != nil {
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err))
c.logger.Warn(logs.CouldntCacheNetmap, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}

5
api/layer/container.go
View file

@ -64,6 +64,7 @@ func (n *Layer) containerInfo(ctx context.Context, prm frostfs.PrmContainer) (*d
log.Error(logs.CouldNotParseContainerObjectLockEnabledAttribute,
zap.String("lock_enabled", attrLockEnabled),
zap.Error(err),
logs.TagField(logs.TagDatapath),
)
}
}
@ -88,7 +89,7 @@ func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams)
res, err := n.frostFS.UserContainers(ctx, prm)
if err != nil {
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err))
n.reqLogger(ctx).Error(logs.CouldNotListUserContainers, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil, err
}
@ -100,7 +101,7 @@ func (n *Layer) containerList(ctx context.Context, listParams ListBucketsParams)
}
info, err := n.containerInfo(ctx, getPrm)
if err != nil {
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err))
n.reqLogger(ctx).Error(logs.CouldNotFetchContainerInfo, zap.Error(err), logs.TagField(logs.TagExternalStorage))
continue
}

3
api/layer/cors.go
View file

@ -92,7 +92,8 @@ func (n *Layer) deleteCORSObject(ctx context.Context, bktInfo *data.BucketInfo,
if err := n.objectDeleteWithAuth(ctx, corsBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteCorsObject, zap.Error(err),
zap.String("cnrID", corsBkt.CID.EncodeToString()),
zap.String("objID", addr.Object().EncodeToString()))
zap.String("objID", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage))
}
}

22
api/layer/layer.go
View file

@ -543,7 +543,9 @@ func (n *Layer) GetExtendedObjectInfo(ctx context.Context, p *HeadObjectParams)
n.reqLogger(ctx).Debug(logs.GetObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", objInfo.ObjectInfo.ID))
zap.Stringer("oid", objInfo.ObjectInfo.ID),
logs.TagField(logs.TagDatapath),
)
return objInfo, nil
}
@ -596,8 +598,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
return obj
}
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
}
if obj.Error = n.treeService.RemoveVersion(ctx, bkt, nodeVersion.ID); obj.Error != nil {
@ -635,8 +637,8 @@ func (n *Layer) deleteObject(ctx context.Context, bkt *data.BucketInfo, settings
if !client.IsErrObjectAlreadyRemoved(obj.Error) && !client.IsErrObjectNotFound(obj.Error) {
return obj
}
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting,
zap.Stringer("cid", bkt.CID), zap.String("oid", obj.VersionID), zap.Error(obj.Error))
n.reqLogger(ctx).Debug(logs.CouldntDeleteObjectFromStorageContinueDeleting, zap.Stringer("cid", bkt.CID),
zap.String("oid", obj.VersionID), zap.Error(obj.Error), logs.TagField(logs.TagExternalStorage))
}
}
@ -751,7 +753,7 @@ func (n *Layer) getNodeVersionsToDelete(ctx context.Context, bkt *data.BucketInf
return nil, fmt.Errorf("%w: there isn't tree node with requested version id", apierr.GetAPIError(apierr.ErrNoSuchVersion))
}
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids))
n.reqLogger(ctx).Debug(logs.GetTreeNodeToDelete, zap.Stringer("cid", bkt.CID), zap.Strings("oids", oids), logs.TagField(logs.TagDatapath))
return versionsToDelete, nil
}
@ -821,7 +823,7 @@ func (n *Layer) DeleteObjects(ctx context.Context, p *DeleteObjectParams) []*Ver
for i, obj := range p.Objects {
p.Objects[i] = n.deleteObject(ctx, p.BktInfo, p.Settings, obj, p.NetworkInfo)
if p.IsMultiple && p.Objects[i].Error != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error))
n.reqLogger(ctx).Error(logs.CouldntDeleteObject, zap.String("object", obj.String()), zap.Error(p.Objects[i].Error), logs.TagField(logs.TagExternalStorage))
}
}
@ -851,7 +853,7 @@ func (n *Layer) ResolveBucket(ctx context.Context, zone, name string) (cid.ID, e
return cid.ID{}, err
}
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID))
n.reqLogger(ctx).Info(logs.ResolveBucket, zap.Stringer("cid", cnrID), logs.TagField(logs.TagExternalBlockchain))
}
return cnrID, nil
@ -876,12 +878,12 @@ func (n *Layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error {
corsObj, err := n.treeService.GetBucketCORS(ctx, p.BktInfo)
if err != nil {
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err))
n.reqLogger(ctx).Error(logs.GetBucketCors, zap.Error(err), logs.TagField(logs.TagExternalStorageTree))
}
lifecycleObj, treeErr := n.treeService.GetBucketLifecycleConfiguration(ctx, p.BktInfo)
if treeErr != nil {
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr))
n.reqLogger(ctx).Error(logs.GetBucketLifecycle, zap.Error(treeErr), logs.TagField(logs.TagExternalStorageTree))
}
err = n.frostFS.DeleteContainer(ctx, p.BktInfo.CID, p.SessionToken)

4
api/layer/lifecycle.go
View file

@ -79,7 +79,9 @@ func (n *Layer) deleteLifecycleObject(ctx context.Context, bktInfo *data.BucketI
if err := n.objectDeleteWithAuth(ctx, lifecycleBkt, addr.Object(), prmAuth); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteLifecycleObject, zap.Error(err),
zap.String("cid", lifecycleBkt.CID.EncodeToString()),
zap.String("oid", addr.Object().EncodeToString()))
zap.String("oid", addr.Object().EncodeToString()),
logs.TagField(logs.TagExternalStorage),
)
}
}

6
api/layer/listing.go
View file

@ -541,7 +541,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
realSize, err := GetObjectSize(oi)
if err != nil {
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err))
reqLog.Debug(logs.FailedToGetRealObjectSize, zap.Error(err), logs.TagField(logs.TagDatapath))
realSize = oi.Size
}
@ -554,7 +554,7 @@ func (n *Layer) initWorkerPool(ctx context.Context, size int, p commonVersionsLi
})
if err != nil {
wg.Done()
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
reqLog.Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
}
}(node)
}
@ -645,7 +645,7 @@ func (n *Layer) objectInfoFromObjectsCacheOrFrostFS(ctx context.Context, bktInfo
meta, err := n.objectHead(ctx, bktInfo, node.OID)
if err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err))
n.reqLogger(ctx).Warn(logs.CouldNotFetchObjectMeta, zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil
}

32
api/layer/multipart_upload.go
View file

@ -213,7 +213,7 @@ func (n *Layer) UploadPart(ctx context.Context, p *UploadPartParams) (string, er
func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInfo, p *UploadPartParams) (*data.ObjectInfo, error) {
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err := p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
}
@ -266,7 +266,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
n.prepareAuthParameters(ctx, &prm.PrmAuth, bktInfo.Owner)
err = n.frostFS.DeleteObject(ctx, prm)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", bktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
}
return nil, apierr.GetAPIError(apierr.ErrInvalidDigest)
}
@ -283,7 +286,10 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, bktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", bktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
}
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
@ -291,7 +297,7 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
n.reqLogger(ctx).Debug(logs.UploadPart,
zap.String("multipart upload", p.Info.UploadID), zap.Int("part number", p.PartNumber),
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID))
zap.Stringer("cid", bktInfo.CID), zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagDatapath))
partInfo := &data.PartInfo{
Key: p.Info.Key,
@ -314,7 +320,8 @@ func (n *Layer) uploadPart(ctx context.Context, multipartInfo *data.MultipartInf
if err = n.objectDelete(ctx, bktInfo, oldPartID); err != nil {
n.reqLogger(ctx).Error(logs.CouldntDeleteOldPartObject, zap.Error(err),
zap.String("cid", bktInfo.CID.EncodeToString()),
zap.String("oid", oldPartID.EncodeToString()))
zap.String("oid", oldPartID.EncodeToString()),
logs.TagField(logs.TagExternalStorage))
}
}
}
@ -481,7 +488,8 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
n.reqLogger(ctx).Error(logs.CouldNotPutCompletedObject,
zap.String("uploadID", p.Info.UploadID),
zap.String("uploadKey", p.Info.Key),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagExternalStorage))
return nil, nil, apierr.GetAPIError(apierr.ErrInternalError)
}
@ -493,7 +501,8 @@ func (n *Layer) CompleteMultipartUpload(ctx context.Context, p *CompleteMultipar
if err = n.objectDelete(ctx, p.Info.Bkt, partInfo.OID); err != nil {
n.reqLogger(ctx).Warn(logs.CouldNotDeleteUploadPart,
zap.Stringer("cid", p.Info.Bkt.CID), zap.Stringer("oid", &partInfo.OID),
zap.Error(err))
zap.Error(err),
logs.TagField(logs.TagExternalStorage))
}
addr.SetObject(partInfo.OID)
n.cache.DeleteObject(addr)
@ -587,7 +596,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
oids, err := relations.ListAllRelations(ctx, n.frostFS.Relations(), bkt.CID, info.OID, tokens)
if err != nil {
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", bkt.CID.EncodeToString()),
zap.String("oid", info.OID.EncodeToString()), zap.Error(err))
zap.String("oid", info.OID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
continue
}
members = append(members, append(oids, info.OID)...)
@ -596,7 +605,7 @@ func (n *Layer) deleteUploadedParts(ctx context.Context, bkt *data.BucketInfo, p
err := n.putTombstones(ctx, bkt, networkInfo, members)
if err != nil {
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err))
n.reqLogger(ctx).Warn(logs.FailedToPutTombstones, zap.Error(err), logs.TagField(logs.TagExternalStorage))
}
}
@ -609,7 +618,7 @@ func (n *Layer) ListParts(ctx context.Context, p *ListPartsParams) (*ListPartsIn
encInfo := FormEncryptionInfo(multipartInfo.Meta)
if err = p.Info.Encryption.MatchObjectEncryption(encInfo); err != nil {
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err))
n.reqLogger(ctx).Warn(logs.MismatchedObjEncryptionInfo, zap.Error(err), logs.TagField(logs.TagDatapath))
return nil, apierr.GetAPIError(apierr.ErrInvalidEncryptionParameters)
}
@ -701,7 +710,8 @@ func (n *Layer) getUploadParts(ctx context.Context, p *UploadInfoParams) (*data.
zap.Stringer("cid", p.Bkt.CID),
zap.String("upload id", p.UploadID),
zap.Ints("part numbers", partsNumbers),
zap.Strings("oids", oids))
zap.Strings("oids", oids),
logs.TagField(logs.TagDatapath))
return multipartInfo, res, nil
}

16
api/layer/object.go
View file

@ -295,7 +295,11 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if !bytes.Equal(headerMd5Hash, createdObj.MD5Sum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage),
)
}
return nil, apierr.GetAPIError(apierr.ErrBadDigest)
}
@ -309,13 +313,17 @@ func (n *Layer) PutObject(ctx context.Context, p *PutObjectParams) (*data.Extend
if !bytes.Equal(contentHashBytes, createdObj.HashSum) {
err = n.objectDelete(ctx, p.BktInfo, createdObj.ID)
if err != nil {
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
n.reqLogger(ctx).Debug(logs.FailedToDeleteObject,
zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID),
logs.TagField(logs.TagExternalStorage))
}
return nil, apierr.GetAPIError(apierr.ErrContentSHA256Mismatch)
}
}
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID), zap.Stringer("oid", createdObj.ID))
n.reqLogger(ctx).Debug(logs.PutObject, zap.Stringer("cid", p.BktInfo.CID),
zap.Stringer("oid", createdObj.ID), logs.TagField(logs.TagExternalStorage))
now := TimeNow(ctx)
newVersion := &data.NodeVersion{
BaseNodeVersion: data.BaseNodeVersion{
@ -540,7 +548,7 @@ func (n *Layer) objectPutAndHash(ctx context.Context, prm frostfs.PrmObjectCreat
func (n *Layer) payloadDiscard(ctx context.Context, payload io.Reader) {
if payload != nil {
if _, errDiscard := io.Copy(io.Discard, payload); errDiscard != nil {
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard))
n.reqLogger(ctx).Warn(logs.FailedToDiscardPutPayloadProbablyGoroutineLeaks, zap.Error(errDiscard), logs.TagField(logs.TagExternalStorage))
}
}
}

2
api/layer/system_object.go
View file

@ -217,7 +217,7 @@ func (n *Layer) GetBucketSettings(ctx context.Context, bktInfo *data.BucketInfo)
return nil, err
}
settings = &data.BucketSettings{Versioning: data.VersioningUnversioned}
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults)
n.reqLogger(ctx).Debug(logs.BucketSettingsNotFoundUseDefaults, logs.TagField(logs.TagDatapath))
}
n.cache.PutSettings(owner, bktInfo, settings)

3
api/layer/tagging.go
View file

@ -168,7 +168,8 @@ func (n *Layer) getNodeVersion(ctx context.Context, objVersion *data.ObjectVersi
if err == nil && version != nil && !version.IsDeleteMarker {
n.reqLogger(ctx).Debug(logs.GetTreeNode,
zap.Stringer("cid", objVersion.BktInfo.CID), zap.Stringer("oid", version.OID))
zap.Stringer("cid", objVersion.BktInfo.CID),
zap.Stringer("oid", version.OID), logs.TagField(logs.TagExternalStorageTree))
}
return version, err

6
api/layer/tombstone.go
View file

@ -65,13 +65,13 @@ func (n *Layer) submitPutTombstone(ctx context.Context, bkt *data.BucketInfo, me
defer wg.Done()
if err := n.putTombstoneObject(ctx, tomb, bkt); err != nil {
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err))
n.reqLogger(ctx).Warn(logs.FailedToPutTombstoneObject, zap.String("cid", bkt.CID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
errCh <- fmt.Errorf("put tombstone object: %w", err)
}
})
if err != nil {
wg.Done()
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err))
n.reqLogger(ctx).Warn(logs.FailedToSubmitTaskToPool, zap.Error(err), logs.TagField(logs.TagDatapath))
errCh <- fmt.Errorf("submit task to pool: %w", err)
}
}
@ -106,7 +106,7 @@ func (n *Layer) getMembers(ctx context.Context, cnrID cid.ID, objID oid.ID, toke
}
n.reqLogger(ctx).Warn(logs.FailedToListAllObjectRelations, zap.String("cid", cnrID.EncodeToString()),
zap.String("oid", objID.EncodeToString()), zap.Error(err))
zap.String("oid", objID.EncodeToString()), zap.Error(err), logs.TagField(logs.TagExternalStorage))
return nil, nil
}
return append(oids, objID), nil

2
api/middleware/address_style.go
View file

@ -110,7 +110,7 @@ func preparePathStyleAddress(reqInfo *ReqInfo, r *http.Request, reqLogger *zap.L
// https://github.com/go-chi/chi/issues/641
// https://github.com/go-chi/chi/issues/642
if obj, err := url.PathUnescape(reqInfo.ObjectName); err != nil {
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err))
reqLogger.Warn(logs.FailedToUnescapeObjectName, zap.Error(err), logs.TagField(logs.TagDatapath))
} else {
reqInfo.ObjectName = obj
}

14
api/middleware/auth.go
View file

@ -53,15 +53,15 @@ func Auth(center Center, log *zap.Logger) Func {
box, err := center.Authenticate(r)
if err != nil {
if errors.Is(err, ErrNoAuthorizationHeader) {
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err))
reqLogOrDefault(ctx, log).Debug(logs.CouldntReceiveAccessBoxForGateKeyRandomKeyWillBeUsed, zap.Error(err), logs.TagField(logs.TagDatapath))
} else {
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err))
reqLogOrDefault(ctx, log).Error(logs.FailedToPassAuthentication, zap.Error(err), logs.TagField(logs.TagDatapath))
err = apierr.TransformToS3Error(err)
if err.(apierr.Error).ErrCode == apierr.ErrInternalError {
err = apierr.GetAPIError(apierr.ErrAccessDenied)
}
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
}
return
}
@ -71,7 +71,7 @@ func Auth(center Center, log *zap.Logger) Func {
if box.AccessBox.Gate.BearerToken != nil {
reqInfo.User = bearer.ResolveIssuer(*box.AccessBox.Gate.BearerToken).String()
}
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID))
reqLogOrDefault(ctx, log).Debug(logs.SuccessfulAuth, zap.String("accessKeyID", box.AuthHeaders.AccessKeyID), logs.TagField(logs.TagDatapath))
}
h.ServeHTTP(w, r.WithContext(ctx))
@ -89,15 +89,15 @@ func FrostfsIDValidation(frostfsID FrostFSIDValidator, log *zap.Logger) Func {
ctx := r.Context()
bd, err := GetBoxData(ctx)
if err != nil || bd.Gate.BearerToken == nil {
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation)
reqLogOrDefault(ctx, log).Debug(logs.AnonRequestSkipFrostfsIDValidation, logs.TagField(logs.TagDatapath))
h.ServeHTTP(w, r)
return
}
if err = validateBearerToken(frostfsID, bd.Gate.BearerToken); err != nil {
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err))
reqLogOrDefault(ctx, log).Error(logs.FrostfsIDValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
if _, wrErr := WriteErrorResponse(w, GetReqInfo(r.Context()), err); wrErr != nil {
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
reqLogOrDefault(ctx, log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
}
return
}

2
api/middleware/log_http_stub.go
View file

@ -23,7 +23,7 @@ type (
)
func LogHTTP(l *zap.Logger, _ LogHTTPSettings) Func {
l.Warn(logs.LogHTTPDisabledInThisBuild)
l.Warn(logs.LogHTTPDisabledInThisBuild, logs.TagField(logs.TagApp))
return func(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
h.ServeHTTP(w, r)

2
api/middleware/metrics.go
View file

@ -139,7 +139,7 @@ func resolveCID(log *zap.Logger, resolveContainerID ContainerIDResolveFunc) cidR
containerID, err := resolveContainerID(ctx, reqInfo.BucketName)
if err != nil {
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err))
reqLogOrDefault(ctx, log).Debug(logs.FailedToResolveCID, zap.Error(err), logs.TagField(logs.TagExternalBlockchain))
return ""
}

8
api/middleware/policy.go
View file

@ -90,10 +90,10 @@ func PolicyCheck(cfg PolicyConfig) Func {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if err := policyCheck(r, cfg); err != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err))
reqLogOrDefault(ctx, cfg.Log).Error(logs.PolicyValidationFailed, zap.Error(err), logs.TagField(logs.TagDatapath))
err = apierr.TransformToS3Error(err)
if _, wrErr := WriteErrorResponse(w, GetReqInfo(ctx), err); wrErr != nil {
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr))
reqLogOrDefault(ctx, cfg.Log).Error(logs.FailedToWriteResponse, zap.Error(wrErr), logs.TagField(logs.TagDatapath))
}
return
}
@ -200,7 +200,9 @@ func getPolicyRequest(r *http.Request, cfg PolicyConfig, reqType ReqType, bktNam
reqLogOrDefault(r.Context(), cfg.Log).Debug(logs.PolicyRequest, zap.String("action", op),
zap.String("resource", res), zap.Any("request properties", requestProps),
zap.Any("resource properties", resourceProps))
zap.Any("resource properties", resourceProps),
logs.TagField(logs.TagDatapath),
)
return testutil.NewRequest(op, testutil.NewResource(res, resourceProps), requestProps), pk, groups, nil
}

5
api/middleware/reqinfo.go
View file

@ -166,7 +166,7 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
// generate random UUIDv4
id, err := uuid.NewRandom()
if err != nil {
log.Error(logs.FailedToGenerateRequestID, zap.Error(err))
log.Error(logs.FailedToGenerateRequestID, zap.Error(err), logs.TagField(logs.TagDatapath))
}
// set request id into response header
@ -198,7 +198,8 @@ func Request(log *zap.Logger, settings RequestSettings) Func {
r = r.WithContext(SetReqLogger(ctx, reqLogger))
reqLogger.Info(logs.RequestStart, zap.String("host", r.Host),
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace))
zap.String("remote_host", reqInfo.RemoteHost), zap.String("namespace", reqInfo.Namespace),
logs.TagField(logs.TagDatapath))
// continue execution
h.ServeHTTP(lw, r)

2
api/middleware/response.go
View file

@ -311,7 +311,7 @@ func LogSuccessResponse(l *zap.Logger) Func {
}
ctx := r.Context()
reqLogger := reqLogOrDefault(ctx, l)
reqLogger := reqLogOrDefault(ctx, l).With(logs.TagField(logs.TagDatapath))
reqInfo := GetReqInfo(ctx)
fields := make([]zap.Field, 0, 6)

6
api/router.go
View file

@ -245,13 +245,14 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
zap.String("method", reqInfo.API),
zap.String("http method", r.Method),
zap.String("url", r.RequestURI),
logs.TagField(logs.TagDatapath),
}
if wrErr != nil {
fields = append(fields, zap.NamedError("write_response_error", wrErr))
}
log.Error(logs.RequestUnmatched, fields...)
log.Error(logs.RequestUnmatched, append(fields, logs.TagField(logs.TagDatapath))...)
}
}
@ -266,13 +267,14 @@ func notSupportedHandler() http.HandlerFunc {
fields := []zap.Field{
zap.String("http method", r.Method),
zap.String("url", r.RequestURI),
logs.TagField(logs.TagDatapath),
}
if wrErr != nil {
fields = append(fields, zap.NamedError("write_response_error", wrErr))
}
log.Error(logs.NotSupported, fields...)
log.Error(logs.NotSupported, append(fields, logs.TagField(logs.TagDatapath))...)
}
}
}