diff --git a/api/handler/list.go b/api/handler/list.go index ec55228a..82401b07 100644 --- a/api/handler/list.go +++ b/api/handler/list.go @@ -39,6 +39,19 @@ type ListMultipartUploadsResult struct { var maxObjectList = 10000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. +func (h *handler) registerAndSendError(w http.ResponseWriter, r *http.Request, err error, logText string) { + rid := api.GetRequestID(r.Context()) + h.log.Error(logText, + zap.String("request_id", rid), + zap.Error(err)) + + api.WriteErrorResponse(r.Context(), w, api.Error{ + Code: api.GetAPIError(api.ErrBadRequest).Code, + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) +} + // ListBucketsHandler handles bucket listing requests. func (h *handler) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { var ( @@ -345,3 +358,92 @@ func (h *handler) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Req }, r.URL) } } + +func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { + p, err := parseListObjectVersionsRequest(r) + if err != nil { + h.registerAndSendError(w, r, err, "failed to parse request ") + return + } + + info, err := h.obj.ListObjectVersions(r.Context(), p) + if err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + return + } + + response := encodeListObjectVersionsToResponse(info, p.Bucket) + if err := api.EncodeToResponse(w, response); err != nil { + h.registerAndSendError(w, r, err, "something went wrong") + } +} + +func parseListObjectVersionsRequest(r *http.Request) (*layer.ListObjectVersionsParams, error) { + var ( + err error + res layer.ListObjectVersionsParams + ) + + if r.URL.Query().Get("max-keys") == "" { + res.MaxKeys = maxObjectList + } else if res.MaxKeys, err = strconv.Atoi(r.URL.Query().Get("max-keys")); err != nil || res.MaxKeys <= 0 { + return nil, api.GetAPIError(api.ErrInvalidMaxKeys) + } + + res.Prefix = r.URL.Query().Get("prefix") + res.KeyMarker = r.URL.Query().Get("marker") + res.Delimiter = r.URL.Query().Get("delimiter") + res.Encode = r.URL.Query().Get("encoding-type") + res.VersionIDMarker = r.URL.Query().Get("version-id-marker") + + if info := api.GetReqInfo(r.Context()); info != nil { + res.Bucket = info.BucketName + } + + return &res, nil +} + +func encodeListObjectVersionsToResponse(info *layer.ListObjectVersionsInfo, bucketName string) *ListObjectsVersionsResponse { + res := ListObjectsVersionsResponse{ + Name: bucketName, + IsTruncated: info.IsTruncated, + KeyMarker: info.KeyMarker, + NextKeyMarker: info.NextKeyMarker, + NextVersionIDMarker: info.NextVersionIDMarker, + VersionIDMarker: info.VersionIDMarker, + } + + for _, prefix := range info.CommonPrefixes { + res.CommonPrefixes = append(res.CommonPrefixes, CommonPrefix{Prefix: *prefix}) + } + + for _, ver := range info.Version { + res.Version = append(res.Version, ObjectVersionResponse{ + IsLatest: ver.IsLatest, + Key: ver.Object.Name, + LastModified: ver.Object.Created.Format(time.RFC3339), + Owner: Owner{ + ID: ver.Object.Owner.String(), + DisplayName: ver.Object.Owner.String(), + }, + Size: ver.Object.Size, + VersionID: ver.VersionID, + ETag: ver.Object.HashSum, + }) + } + // this loop is not starting till versioning is not implemented + for _, del := range info.DeleteMarker { + res.DeleteMarker = append(res.DeleteMarker, DeleteMarkerEntry{ + IsLatest: del.IsLatest, + Key: del.Key, + LastModified: del.LastModified, + Owner: Owner{ + ID: del.Owner.String(), + DisplayName: del.Owner.String(), + }, + VersionID: del.VersionID, + }) + } + + return &res +} diff --git a/api/handler/response.go b/api/handler/response.go index cccdd23d..fda7f60d 100644 --- a/api/handler/response.go +++ b/api/handler/response.go @@ -109,6 +109,27 @@ type Object struct { UserMetadata StringMap `xml:"UserMetadata,omitempty"` } +// ObjectVersionResponse container for object version in the response of ListBucketObjectVersionsHandler. +type ObjectVersionResponse struct { + ETag string `xml:"ETag"` + IsLatest bool `xml:"IsLatest"` + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + Owner Owner `xml:"Owner"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass,omitempty"` // is empty!! + VersionID string `xml:"VersionId"` +} + +// DeleteMarkerEntry container for deleted object's version in the response of ListBucketObjectVersionsHandler. +type DeleteMarkerEntry struct { + IsLatest bool `xml:"IsLatest"` + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + Owner Owner `xml:"Owner"` + VersionID string `xml:"VersionId"` +} + // StringMap is a map[string]string. type StringMap map[string]string @@ -125,6 +146,21 @@ type CopyObjectResponse struct { ETag string // md5sum of the copied object. } +// ListObjectsVersionsResponse is a response of ListBucketObjectVersionsHandler. +type ListObjectsVersionsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"` + EncodingType string `xml:"EncodingType,omitempty"` + Name string `xml:"Name"` + IsTruncated bool `xml:"IsTruncated"` + KeyMarker string `xml:"KeyMarker"` + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + NextVersionIDMarker string `xml:"NextVersionIdMarker,omitempty"` + VersionIDMarker string `xml:"VersionIdMarker"` + DeleteMarker []DeleteMarkerEntry `xml:"DeleteMarker"` + Version []ObjectVersionResponse `xml:"Version"` + CommonPrefixes []CommonPrefix `xml:"CommonPrefixes"` +} + // MarshalXML - StringMap marshals into XML. func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { tokens := []xml.Token{start} diff --git a/api/handler/unimplemented.go b/api/handler/unimplemented.go index 716eb34e..34bf13da 100644 --- a/api/handler/unimplemented.go +++ b/api/handler/unimplemented.go @@ -279,14 +279,6 @@ func (h *handler) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) }, r.URL) } -func (h *handler) ListBucketObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { - api.WriteErrorResponse(r.Context(), w, api.Error{ - Code: "XNeoFSUnimplemented", - Description: "implement me " + mux.CurrentRoute(r).GetName(), - HTTPStatusCode: http.StatusNotImplemented, - }, r.URL) -} - func (h *handler) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { api.WriteErrorResponse(r.Context(), w, api.Error{ Code: "XNeoFSUnimplemented", diff --git a/api/layer/layer.go b/api/layer/layer.go index 3b9f4f44..6dbd7375 100644 --- a/api/layer/layer.go +++ b/api/layer/layer.go @@ -82,6 +82,16 @@ type ( DeleteBucketParams struct { Name string } + // ListObjectVersionsParams stores list objects versions parameters. + ListObjectVersionsParams struct { + Bucket string + Delimiter string + KeyMarker string + MaxKeys int + Prefix string + VersionIDMarker string + Encode string + } // NeoFS provides basic NeoFS interface. NeoFS interface { @@ -105,6 +115,7 @@ type ( CopyObject(ctx context.Context, p *CopyObjectParams) (*ObjectInfo, error) ListObjects(ctx context.Context, p *ListObjectsParams) (*ListObjectsInfo, error) + ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) DeleteObject(ctx context.Context, bucket, object string) error DeleteObjects(ctx context.Context, bucket string, objects []string) []error @@ -118,8 +129,11 @@ var ( ErrObjectNotExists = errors.New("object not exists") ) -// ETag (hex encoded md5sum) of empty string. -const emptyETag = "d41d8cd98f00b204e9800998ecf8427e" +const ( + // ETag (hex encoded md5sum) of empty string. + emptyETag = "d41d8cd98f00b204e9800998ecf8427e" + unversionedObjectVersionID = "null" +) // NewLayer creates instance of layer. It checks credentials // and establishes gRPC connection with node. @@ -488,3 +502,73 @@ func (n *layer) DeleteBucket(ctx context.Context, p *DeleteBucketParams) error { return n.deleteContainer(ctx, bucketInfo.CID) } + +func (n *layer) ListObjectVersions(ctx context.Context, p *ListObjectVersionsParams) (*ListObjectVersionsInfo, error) { + var ( + res = ListObjectVersionsInfo{} + err error + bkt *BucketInfo + ids []*object.ID + uniqNames = make(map[string]bool) + ) + + if bkt, err = n.GetBucketInfo(ctx, p.Bucket); err != nil { + return nil, err + } else if ids, err = n.objectSearch(ctx, &findParams{cid: bkt.CID}); err != nil { + return nil, err + } + + versions := make([]*ObjectVersionInfo, 0, len(ids)) + // todo: deletemarkers is empty now, we will use it after proper realization of versioning + deleted := make([]*DeletedObjectInfo, 0, len(ids)) + res.DeleteMarker = deleted + + for _, id := range ids { + addr := object.NewAddress() + addr.SetObjectID(id) + addr.SetContainerID(bkt.CID) + + meta, err := n.objectHead(ctx, addr) + if err != nil { + n.log.Warn("could not fetch object meta", zap.Error(err)) + continue + } + if ov := objectVersionInfoFromMeta(bkt, meta, p.Prefix, p.Delimiter); ov != nil { + if _, ok := uniqNames[ov.Object.Name]; ok { + continue + } + if len(p.KeyMarker) > 0 && ov.Object.Name <= p.KeyMarker { + continue + } + uniqNames[ov.Object.Name] = ov.Object.isDir + versions = append(versions, ov) + } + } + + sort.Slice(versions, func(i, j int) bool { + return versions[i].Object.Name < versions[j].Object.Name + }) + + if len(versions) > p.MaxKeys { + res.IsTruncated = true + + lastVersion := versions[p.MaxKeys-1] + res.KeyMarker = lastVersion.Object.Name + res.VersionIDMarker = lastVersion.VersionID + + nextVersion := versions[p.MaxKeys] + res.NextKeyMarker = nextVersion.Object.Name + res.NextVersionIDMarker = nextVersion.VersionID + + versions = versions[:p.MaxKeys] + } + + for _, ov := range versions { + if isDir := uniqNames[ov.Object.Name]; isDir { + res.CommonPrefixes = append(res.CommonPrefixes, &ov.Object.Name) + } else { + res.Version = append(res.Version, ov) + } + } + return &res, nil +} diff --git a/api/layer/util.go b/api/layer/util.go index 1b1c70af..c4f355b4 100644 --- a/api/layer/util.go +++ b/api/layer/util.go @@ -53,6 +53,34 @@ type ( // List of prefixes for this request. Prefixes []string } + + // ObjectVersionInfo stores info about objects versions. + ObjectVersionInfo struct { + Object *ObjectInfo + IsLatest bool + VersionID string + } + + // DeletedObjectInfo stores info about deleted versions of objects. + DeletedObjectInfo struct { + Owner *owner.ID + Key string + VersionID string + IsLatest bool + LastModified string + } + + // ListObjectVersionsInfo stores info and list of objects' versions. + ListObjectVersionsInfo struct { + CommonPrefixes []*string + IsTruncated bool + KeyMarker string + NextKeyMarker string + NextVersionIDMarker string + Version []*ObjectVersionInfo + DeleteMarker []*DeletedObjectInfo + VersionIDMarker string + } ) // PathSeparator is a path components separator string. @@ -123,6 +151,14 @@ func objectInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter } } +func objectVersionInfoFromMeta(bkt *BucketInfo, meta *object.Object, prefix, delimiter string) *ObjectVersionInfo { + oi := objectInfoFromMeta(bkt, meta, prefix, delimiter) + if oi == nil { + return nil + } + return &ObjectVersionInfo{Object: oi, IsLatest: true, VersionID: unversionedObjectVersionID} +} + func filenameFromObject(o *object.Object) string { var name = o.ID().String() for _, attr := range o.Attributes() { diff --git a/go.sum b/go.sum index 65362f12..623e46b4 100644 --- a/go.sum +++ b/go.sum @@ -349,7 +349,6 @@ github.com/nspcc-dev/neo-go v0.95.3/go.mod h1:t15xRFDVhz5o/pstptdoW9N9JJBNn1hZ6A github.com/nspcc-dev/neofs-api-go v1.22.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.24.0/go.mod h1:G7dqincfdjBrAbL5nxVp82emF05fSVEqe59ICsoRDI8= github.com/nspcc-dev/neofs-api-go v1.27.0/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= -github.com/nspcc-dev/neofs-api-go v1.27.1 h1:ONdKOnm0/hK6m38VTUliCHY6RTxg+IpAzY4G+BeOZG4= github.com/nspcc-dev/neofs-api-go v1.27.1/go.mod h1:i0Cwgvcu9A4M4e58pydbXFisUhSxpfljmuWFPIp2btE= github.com/nspcc-dev/neofs-api-go v1.28.0 h1:q3FNNz0as4XDqm/AJlAIWMX8YH0nLNbi+G9bONhnKnU= github.com/nspcc-dev/neofs-api-go v1.28.0/go.mod h1:YRIzUqBj/lGbmFm8mmCh54ZOzcJKkEIhv2s7ZvSLv3M=