frostfs-node/cmd/neofs-cli/modules/object.go
Alex Vanin 4c0ca23b34 [] Print split ID in object.Head output of neofs-cli
Signed-off-by: Alex Vanin <alexey@nspcc.ru>
2020-11-30 10:44:15 +03:00

762 lines
19 KiB
Go

package cmd
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/nspcc-dev/neofs-api-go/pkg/client"
"github.com/nspcc-dev/neofs-api-go/pkg/container"
"github.com/nspcc-dev/neofs-api-go/pkg/object"
"github.com/nspcc-dev/neofs-api-go/pkg/owner"
"github.com/nspcc-dev/neofs-api-go/pkg/token"
"github.com/spf13/cobra"
)
const (
getRangeCmdUse = "range"
getRangeCmdShortDesc = "Get payload range data of an object"
getRangeCmdLongDesc = "Get payload range data of an object"
)
var (
// objectCmd represents the object command
objectCmd = &cobra.Command{
Use: "object",
Short: "Operations with Objects",
Long: `Operations with Objects`,
}
objectPutCmd = &cobra.Command{
Use: "put",
Short: "Put object to NeoFS",
Long: "Put object to NeoFS",
RunE: putObject,
}
objectGetCmd = &cobra.Command{
Use: "get",
Short: "Get object from NeoFS",
Long: "Get object from NeoFS",
RunE: getObject,
}
objectDelCmd = &cobra.Command{
Use: "delete",
Aliases: []string{"del"},
Short: "Delete object from NeoFS",
Long: "Delete object from NeoFS",
RunE: deleteObject,
}
objectSearchCmd = &cobra.Command{
Use: "search",
Short: "Search object",
Long: "Search object",
RunE: searchObject,
}
objectHeadCmd = &cobra.Command{
Use: "head",
Short: "Get object header",
Long: "Get object header",
RunE: getObjectHeader,
}
objectHashCmd = &cobra.Command{
Use: "hash",
Short: "Get object hash",
Long: "Get object hash",
RunE: getObjectHash,
}
objectRangeCmd = &cobra.Command{
Use: getRangeCmdUse,
Short: getRangeCmdShortDesc,
Long: getRangeCmdLongDesc,
RunE: getObjectRange,
}
)
const (
hashSha256 = "sha256"
hashTz = "tz"
rangeSep = ":"
)
const searchOIDFlag = "oid"
func init() {
rootCmd.AddCommand(objectCmd)
objectCmd.PersistentFlags().String("bearer", "", "File with signed JSON or binary encoded bearer token")
objectCmd.AddCommand(objectPutCmd)
objectPutCmd.Flags().String("file", "", "File with object payload")
_ = objectPutCmd.MarkFlagFilename("file")
_ = objectPutCmd.MarkFlagRequired("file")
objectPutCmd.Flags().String("cid", "", "Container ID")
_ = objectPutCmd.MarkFlagRequired("cid")
objectPutCmd.Flags().String("attributes", "", "User attributes in form of Key1=Value1,Key2=Value2")
objectPutCmd.Flags().Bool("disable-filename", false, "Do not set well-known filename attribute")
objectPutCmd.Flags().Bool("disable-timestamp", false, "Do not set well-known timestamp attribute")
objectCmd.AddCommand(objectDelCmd)
objectDelCmd.Flags().String("cid", "", "Container ID")
_ = objectDelCmd.MarkFlagRequired("cid")
objectDelCmd.Flags().String("oid", "", "Object ID")
_ = objectDelCmd.MarkFlagRequired("oid")
objectCmd.AddCommand(objectGetCmd)
objectGetCmd.Flags().String("file", "", "File to write object payload to. Default: stdout.")
objectGetCmd.Flags().String("header", "", "File to write header to. Default: stdout.")
objectGetCmd.Flags().String("cid", "", "Container ID")
_ = objectGetCmd.MarkFlagRequired("cid")
objectGetCmd.Flags().String("oid", "", "Object ID")
_ = objectGetCmd.MarkFlagRequired("oid")
objectCmd.AddCommand(objectSearchCmd)
objectSearchCmd.Flags().String("cid", "", "Container ID")
_ = objectSearchCmd.MarkFlagRequired("cid")
objectSearchCmd.Flags().String("filters", "", "Filters in the form hdrName=value,...")
objectSearchCmd.Flags().Bool("root", false, "Search for user objects")
objectSearchCmd.Flags().Bool("phy", false, "Search physically stored objects")
objectSearchCmd.Flags().String(searchOIDFlag, "", "Search object by identifier")
objectCmd.AddCommand(objectHeadCmd)
objectHeadCmd.Flags().String("file", "", "File to write header to. Default: stdout.")
objectHeadCmd.Flags().String("cid", "", "Container ID")
_ = objectHeadCmd.MarkFlagRequired("cid")
objectHeadCmd.Flags().String("oid", "", "Object ID")
_ = objectHeadCmd.MarkFlagRequired("oid")
objectHeadCmd.Flags().Bool("main-only", false, "Return only main fields")
objectHeadCmd.Flags().Bool("json", false, "Marshal output in JSON")
objectHeadCmd.Flags().Bool("proto", false, "Marshal output in Protobuf")
objectCmd.AddCommand(objectHashCmd)
objectHashCmd.Flags().String("cid", "", "Container ID")
_ = objectHashCmd.MarkFlagRequired("cid")
objectHashCmd.Flags().String("oid", "", "Object ID")
_ = objectHashCmd.MarkFlagRequired("oid")
objectHashCmd.Flags().String("range", "", "Range to take hash from in the form offset1:length1,...")
objectHashCmd.Flags().String("type", hashSha256, "Hash type. Either 'sha256' or 'tz'")
objectCmd.AddCommand(objectRangeCmd)
objectRangeCmd.Flags().String("cid", "", "Container ID")
_ = objectRangeCmd.MarkFlagRequired("cid")
objectRangeCmd.Flags().String("oid", "", "Object ID")
_ = objectRangeCmd.MarkFlagRequired("oid")
objectRangeCmd.Flags().String("range", "", "Range to take data from in the form offset:length")
objectRangeCmd.Flags().String("file", "", "File to write object payload to. Default: stdout.")
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// objectCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// objectCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
func initSession(ctx context.Context) (*client.Client, *token.SessionToken, error) {
cli, err := getSDKClient()
if err != nil {
return nil, nil, fmt.Errorf("can't create client: %w", err)
}
tok, err := cli.CreateSession(ctx, math.MaxUint64)
if err != nil {
return nil, nil, fmt.Errorf("can't create session: %w", err)
}
return cli, tok, nil
}
func putObject(cmd *cobra.Command, _ []string) error {
ownerID, err := getOwnerID()
if err != nil {
return err
}
cid, err := getCID(cmd)
if err != nil {
return err
}
filename := cmd.Flag("file").Value.String()
f, err := os.OpenFile(filename, os.O_RDONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("can't open file '%s': %w", filename, err)
}
attrs, err := parseObjectAttrs(cmd)
if err != nil {
return fmt.Errorf("can't parse object attributes: %w", err)
}
obj := object.NewRaw()
obj.SetContainerID(cid)
obj.SetOwnerID(ownerID)
obj.SetAttributes(attrs...)
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
oid, err := cli.PutObject(ctx,
new(client.PutObjectParams).
WithObject(obj.Object()).
WithPayloadReader(f),
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return fmt.Errorf("can't put object: %w", err)
}
cmd.Printf("[%s] Object successfully stored\n", filename)
cmd.Printf(" ID: %s\n CID: %s\n", oid, cid)
return nil
}
func deleteObject(cmd *cobra.Command, _ []string) error {
objAddr, err := getObjectAddress(cmd)
if err != nil {
return err
}
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
err = cli.DeleteObject(ctx,
new(client.DeleteObjectParams).WithAddress(objAddr),
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return err
}
cmd.Println("Object removed successfully.")
cmd.Printf(" ID: %s\n CID: %s\n", objAddr.ObjectID(), objAddr.ContainerID())
return nil
}
func getObject(cmd *cobra.Command, _ []string) error {
objAddr, err := getObjectAddress(cmd)
if err != nil {
return err
}
var out io.Writer
filename := cmd.Flag("file").Value.String()
if filename == "" {
out = os.Stdout
} else {
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("can't open file '%s': %w", filename, err)
}
defer f.Close()
out = f
}
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
obj, err := cli.GetObject(ctx,
new(client.GetObjectParams).
WithAddress(objAddr).
WithPayloadWriter(out),
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return fmt.Errorf("can't put object: %w", err)
}
if filename != "" {
cmd.Printf("[%s] Object successfully saved\n", filename)
}
// Print header only if file is not streamed to stdout.
hdrFile := cmd.Flag("header").Value.String()
if filename != "" || hdrFile != "" {
return printHeader(cmd, obj, hdrFile)
}
return nil
}
func getObjectHeader(cmd *cobra.Command, _ []string) error {
objAddr, err := getObjectAddress(cmd)
if err != nil {
return err
}
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
ps := new(client.ObjectHeaderParams).WithAddress(objAddr)
if ok, _ := cmd.Flags().GetBool("main-only"); ok {
ps = ps.WithMainFields()
}
obj, err := cli.GetObjectHeader(ctx, ps,
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return fmt.Errorf("can't put object: %w", err)
}
return printHeader(cmd, obj, cmd.Flag("file").Value.String())
}
func searchObject(cmd *cobra.Command, _ []string) error {
cid, err := getCID(cmd)
if err != nil {
return err
}
sf, err := parseSearchFilters(cmd)
if err != nil {
return err
}
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
ps := new(client.SearchObjectParams).WithContainerID(cid).WithSearchFilters(sf)
ids, err := cli.SearchObject(ctx, ps,
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return fmt.Errorf("can't put object: %w", err)
}
cmd.Printf("Found %d objects.\n", len(ids))
for _, id := range ids {
cmd.Println(id)
}
return nil
}
func getObjectHash(cmd *cobra.Command, _ []string) error {
objAddr, err := getObjectAddress(cmd)
if err != nil {
return err
}
ranges, err := getRangeList(cmd)
if err != nil {
return err
}
typ, err := getHashType(cmd)
if err != nil {
return err
}
ctx := context.Background()
cli, tok, err := initSession(ctx)
if err != nil {
return err
}
btok, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
if len(ranges) == 0 { // hash of full payload
obj, err := cli.GetObjectHeader(ctx,
new(client.ObjectHeaderParams).WithAddress(objAddr),
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return fmt.Errorf("can't get object: %w", err)
}
switch typ {
case hashSha256:
cmd.Println(hex.EncodeToString(obj.PayloadChecksum().Sum()))
case hashTz:
cmd.Println(hex.EncodeToString(obj.PayloadHomomorphicHash().Sum()))
}
return nil
}
ps := new(client.RangeChecksumParams).WithAddress(objAddr).WithRangeList(ranges...)
switch typ {
case hashSha256:
res, err := cli.ObjectPayloadRangeSHA256(ctx, ps,
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return err
}
for i := range res {
cmd.Printf("Offset=%d (Length=%d)\t: %s\n", ranges[i].GetOffset(), ranges[i].GetLength(),
hex.EncodeToString(res[i][:]))
}
case hashTz:
res, err := cli.ObjectPayloadRangeTZ(ctx, ps,
client.WithTTL(getTTL()),
client.WithSession(tok),
client.WithBearer(btok))
if err != nil {
return err
}
for i := range res {
cmd.Printf("Offset=%d (Length=%d)\t: %s\n", ranges[i].GetOffset(), ranges[i].GetLength(),
hex.EncodeToString(res[i][:]))
}
}
return nil
}
func getOwnerID() (*owner.ID, error) {
key, err := getKey()
if err != nil {
return nil, err
}
w, err := owner.NEO3WalletFromPublicKey(&key.PublicKey)
if err != nil {
return nil, err
}
ownerID := owner.NewID()
ownerID.SetNeo3Wallet(w)
return ownerID, nil
}
func parseSearchFilters(cmd *cobra.Command) (object.SearchFilters, error) {
var fs object.SearchFilters
if raw := cmd.Flag("filters").Value.String(); len(raw) != 0 {
rawFs := strings.Split(raw, ",")
for i := range rawFs {
kv := strings.SplitN(rawFs[i], "=", 2)
if len(kv) != 2 {
return nil, fmt.Errorf("invalid filter format: %s", rawFs[i])
}
fs.AddFilter(kv[0], kv[1], object.MatchStringEqual)
}
}
root, _ := cmd.Flags().GetBool("root")
if root {
fs.AddRootFilter()
}
phy, _ := cmd.Flags().GetBool("phy")
if phy {
fs.AddPhyFilter()
}
oid, _ := cmd.Flags().GetString(searchOIDFlag)
if oid != "" {
id := object.NewID()
if err := id.Parse(oid); err != nil {
return nil, err
}
fs.AddObjectIDFilter(object.MatchStringEqual, id)
}
return fs, nil
}
func parseObjectAttrs(cmd *cobra.Command) ([]*object.Attribute, error) {
var rawAttrs []string
raw := cmd.Flag("attributes").Value.String()
if len(raw) != 0 {
rawAttrs = strings.Split(raw, ",")
}
attrs := make([]*object.Attribute, 0, len(rawAttrs)+2) // name + timestamp attributes
for i := range rawAttrs {
kv := strings.SplitN(rawAttrs[i], "=", 2)
if len(kv) != 2 {
return nil, fmt.Errorf("invalid attribute format: %s", rawAttrs[i])
}
attr := object.NewAttribute()
attr.SetKey(kv[0])
attr.SetValue(kv[1])
attrs = append(attrs, attr)
}
disableFilename, _ := cmd.Flags().GetBool("disable-filename")
if !disableFilename {
filename := filepath.Base(cmd.Flag("file").Value.String())
attr := object.NewAttribute()
attr.SetKey(object.AttributeFileName)
attr.SetValue(filename)
attrs = append(attrs, attr)
}
disableTime, _ := cmd.Flags().GetBool("disable-timestamp")
if !disableTime {
attr := object.NewAttribute()
attr.SetKey(object.AttributeTimestamp)
attr.SetValue(strconv.FormatInt(time.Now().Unix(), 10))
attrs = append(attrs, attr)
}
return attrs, nil
}
func getCID(cmd *cobra.Command) (*container.ID, error) {
cid := container.NewID()
err := cid.Parse(cmd.Flag("cid").Value.String())
return cid, err
}
func getOID(cmd *cobra.Command) (*object.ID, error) {
oid := object.NewID()
err := oid.Parse(cmd.Flag("oid").Value.String())
return oid, err
}
func getObjectAddress(cmd *cobra.Command) (*object.Address, error) {
cid, err := getCID(cmd)
if err != nil {
return nil, err
}
oid, err := getOID(cmd)
if err != nil {
return nil, err
}
objAddr := object.NewAddress()
objAddr.SetContainerID(cid)
objAddr.SetObjectID(oid)
return objAddr, nil
}
func getRangeList(cmd *cobra.Command) ([]*object.Range, error) {
v := cmd.Flag("range").Value.String()
if len(v) == 0 {
return nil, nil
}
vs := strings.Split(v, ",")
rs := make([]*object.Range, len(vs))
for i := range vs {
r := strings.Split(vs[i], rangeSep)
if len(r) != 2 {
return nil, fmt.Errorf("invalid range specifier: %s", vs[i])
}
offset, err := strconv.ParseUint(r[0], 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid range specifier: %s", vs[i])
}
length, err := strconv.ParseUint(r[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("invalid range specifier: %s", vs[i])
}
rs[i] = object.NewRange()
rs[i].SetOffset(offset)
rs[i].SetLength(length)
}
return rs, nil
}
func getHashType(cmd *cobra.Command) (string, error) {
rawType := cmd.Flag("type").Value.String()
switch typ := strings.ToLower(rawType); typ {
case hashSha256, hashTz:
return typ, nil
default:
return "", fmt.Errorf("invalid hash type: %s", typ)
}
}
func printHeader(cmd *cobra.Command, obj *object.Object, filename string) error {
bs, err := marshalHeader(cmd, obj)
if err != nil {
return err
}
if len(bs) != 0 {
if filename == "" {
cmd.Println(string(bs))
return nil
}
err := ioutil.WriteFile(filename, bs, os.ModePerm)
if err != nil {
return err
}
cmd.Printf("[%s] Header successfully saved.", filename)
}
cmd.Printf("ID: %s\n", obj.ID())
cmd.Printf("CID: %s\n", obj.ContainerID())
cmd.Printf("Owner: %s\n", obj.OwnerID())
cmd.Printf("CreatedAt: %d\n", obj.CreationEpoch())
cmd.Printf("Size: %d\n", obj.PayloadSize())
cmd.Printf("HomoHash: %s\n", hex.EncodeToString(obj.PayloadHomomorphicHash().Sum()))
cmd.Printf("Checksum: %s\n", hex.EncodeToString(obj.PayloadChecksum().Sum()))
switch obj.Type() {
case object.TypeRegular:
cmd.Println("Type: regular")
case object.TypeTombstone:
cmd.Println("Type: tombstone")
case object.TypeStorageGroup:
cmd.Println("Type: storage group")
default:
cmd.Println("Type: unknown")
}
cmd.Println("Attributes:")
for _, attr := range obj.Attributes() {
if attr.Key() == object.AttributeTimestamp {
cmd.Printf(" %s=%s (%s)\n",
attr.Key(),
attr.Value(),
prettyPrintUnixTime(attr.Value()))
continue
}
cmd.Printf(" %s=%s\n", attr.Key(), attr.Value())
}
if obj.SplitID() != nil {
cmd.Printf("SplitID: %s\n", obj.SplitID().String())
}
return nil
}
func marshalHeader(cmd *cobra.Command, hdr *object.Object) ([]byte, error) {
toJson, _ := cmd.Flags().GetBool("json")
toProto, _ := cmd.Flags().GetBool("proto")
switch {
case toJson && toProto:
return nil, errors.New("'--json' and '--proto' flags are mutually exclusive")
case toJson:
return hdr.MarshalJSON()
case toProto:
return hdr.Marshal()
default:
return nil, nil
}
}
func getBearerToken(cmd *cobra.Command, flagname string) (*token.BearerToken, error) {
path, err := cmd.Flags().GetString(flagname)
if err != nil || len(path) == 0 {
return nil, nil
}
data, err := ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("can't read bearer token file: %w", err)
}
tok := token.NewBearerToken()
if err := tok.UnmarshalJSON(data); err != nil {
if err = tok.Unmarshal(data); err != nil {
return nil, fmt.Errorf("can't decode bearer token: %w", err)
}
printVerbose("Using binary encoded bearer token")
} else {
printVerbose("Using JSON encoded bearer token")
}
return tok, nil
}
func getObjectRange(cmd *cobra.Command, _ []string) error {
objAddr, err := getObjectAddress(cmd)
if err != nil {
return err
}
ranges, err := getRangeList(cmd)
if err != nil {
return err
} else if len(ranges) != 1 {
return errors.New("exactly one range must be specified")
}
var out io.Writer
filename := cmd.Flag("file").Value.String()
if filename == "" {
out = os.Stdout
} else {
f, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY, os.ModePerm)
if err != nil {
return fmt.Errorf("can't open file '%s': %w", filename, err)
}
defer f.Close()
out = f
}
ctx := context.Background()
c, sessionToken, err := initSession(ctx)
if err != nil {
return err
}
bearerToken, err := getBearerToken(cmd, "bearer")
if err != nil {
return err
}
_, err = c.ObjectPayloadRangeData(ctx,
new(client.RangeDataParams).
WithAddress(objAddr).
WithRange(ranges[0]).
WithDataWriter(out),
client.WithTTL(getTTL()),
client.WithSession(sessionToken),
client.WithBearer(bearerToken))
if err != nil {
return fmt.Errorf("can't get object payload range: %w", err)
}
if filename != "" {
cmd.Printf("[%s] Payload successfully saved\n", filename)
}
return nil
}