[#20] Client: Optimize memory usage #21
46 changed files with 596 additions and 372 deletions
25
src/FrostFS.SDK.ClientV2/Cache.cs
Normal file
25
src/FrostFS.SDK.ClientV2/Cache.cs
Normal file
|
@ -0,0 +1,25 @@
|
|||
|
||||
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2
|
||||
{
|
||||
internal static class Cache
|
||||
{
|
||||
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
|
||||
{
|
||||
// TODO: get from options?
|
||||
SizeLimit = 256
|
||||
});
|
||||
|
||||
private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions
|
||||
{
|
||||
// TODO: get from options?
|
||||
SizeLimit = 1024
|
||||
});
|
||||
|
||||
internal static IMemoryCache Owners => _ownersCache;
|
||||
|
||||
internal static IMemoryCache Containers => _containersCache;
|
||||
}
|
||||
}
|
|
@ -125,7 +125,7 @@ public class Client : IFrostFSClient
|
|||
|
||||
public IAsyncEnumerable<ContainerId> ListContainersAsync(PrmContainerGetAll? args = null)
|
||||
{
|
||||
args = args ?? new PrmContainerGetAll();
|
||||
args ??= new PrmContainerGetAll();
|
||||
var service = GetContainerService(args);
|
||||
return service.ListContainersAsync(args);
|
||||
}
|
||||
|
@ -223,7 +223,10 @@ public class Client : IFrostFSClient
|
|||
#region ToolsImplementation
|
||||
public ObjectId CalculateObjectId(ObjectHeader header)
|
||||
{
|
||||
return new ObjectTools(ClientCtx).CalculateObjectId(header);
|
||||
if (header == null)
|
||||
throw new ArgumentNullException(nameof(header));
|
||||
|
||||
return ObjectTools.CalculateObjectId(header, ClientCtx);
|
||||
}
|
||||
#endregion
|
||||
|
||||
|
@ -236,7 +239,6 @@ public class Client : IFrostFSClient
|
|||
if (!localNodeInfo.Version.IsSupported(ClientCtx.Version))
|
||||
{
|
||||
var msg = $"FrostFS {localNodeInfo.Version} is not supported.";
|
||||
Console.WriteLine(msg);
|
||||
throw new ApplicationException(msg);
|
||||
}
|
||||
}
|
||||
|
|
13
src/FrostFS.SDK.ClientV2/CllientKey.cs
Normal file
13
src/FrostFS.SDK.ClientV2/CllientKey.cs
Normal file
|
@ -0,0 +1,13 @@
|
|||
using FrostFS.SDK.Cryptography;
|
||||
using Google.Protobuf;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2
|
||||
{
|
||||
public class ClientKey(ECDsa key)
|
||||
{
|
||||
internal ECDsa ECDsaKey { get; } = key;
|
||||
|
||||
internal ByteString PublicKeyProto { get; } = ByteString.CopyFrom(key.PublicKey());
|
||||
}
|
||||
}
|
|
@ -1,12 +1,9 @@
|
|||
using System;
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
using System;
|
||||
|
||||
public class ResponseException : Exception
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
||||
public class ResponseException(ResponseStatus status) : Exception()
|
||||
{
|
||||
public Status Status { get; set; }
|
||||
|
||||
public ResponseException(Status status) : base()
|
||||
{
|
||||
Status = status;
|
||||
}
|
||||
public ResponseStatus Status { get; set; } = status;
|
||||
}
|
|
@ -12,8 +12,11 @@
|
|||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
|
||||
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
|
||||
<PackageReference Include="System.Runtime.Caching" Version="8.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
|
@ -10,12 +10,12 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class ContainerMapper
|
||||
{
|
||||
public static Container.Container ToGrpcMessage(this ModelsV2.Container container)
|
||||
public static Container.Container ToMessage(this ModelsV2.Container container)
|
||||
{
|
||||
return new Container.Container
|
||||
{
|
||||
BasicAcl = (uint)container.BasicAcl,
|
||||
PlacementPolicy = container.PlacementPolicy.ToGrpcMessage(),
|
||||
PlacementPolicy = container.PlacementPolicy.ToMessage(),
|
||||
Nonce = ByteString.CopyFrom(container.Nonce.ToBytes())
|
||||
};
|
||||
}
|
||||
|
|
|
@ -2,16 +2,29 @@ using FrostFS.Refs;
|
|||
using FrostFS.SDK.Cryptography;
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
using Google.Protobuf;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using System;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||
|
||||
public static class ContainerIdMapper
|
||||
{
|
||||
public static ContainerID ToGrpcMessage(this ContainerId containerId)
|
||||
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
|
||||
.SetSlidingExpiration(TimeSpan.FromHours(1))
|
||||
.SetSize(1);
|
||||
|
||||
public static ContainerID ToMessage(this ContainerId model)
|
||||
{
|
||||
return new ContainerID
|
||||
if (!Cache.Containers.TryGetValue(model, out ContainerID? message))
|
||||
{
|
||||
Value = ByteString.CopyFrom(Base58.Decode(containerId.Value))
|
||||
};
|
||||
message = new ContainerID
|
||||
{
|
||||
Value = ByteString.CopyFrom(Base58.Decode(model.Value))
|
||||
};
|
||||
|
||||
Cache.Containers.Set(model, message, _oneHourExpiration);
|
||||
}
|
||||
|
||||
return message!;
|
||||
}
|
||||
}
|
|
@ -1,23 +1,17 @@
|
|||
using FrostFS.SDK.ModelsV2;
|
||||
using FrostFS.Session;
|
||||
using Version = FrostFS.Refs.Version;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||
|
||||
public static class MetaHeaderMapper
|
||||
{
|
||||
public static RequestMetaHeader ToGrpcMessage(this MetaHeader metaHeader)
|
||||
public static RequestMetaHeader ToMessage(this MetaHeader metaHeader)
|
||||
{
|
||||
return new RequestMetaHeader
|
||||
{
|
||||
Version = new Version
|
||||
{
|
||||
Major = (uint)metaHeader.Version.Major,
|
||||
Minor = (uint)metaHeader.Version.Minor,
|
||||
|
||||
},
|
||||
Version = metaHeader.Version.ToMessage(),
|
||||
Epoch = (uint)metaHeader.Epoch,
|
||||
Ttl = (uint)metaHeader.Ttl
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
|
@ -10,6 +10,8 @@ public static class NetmapMapper
|
|||
{
|
||||
return new NetmapSnapshot(
|
||||
netmap.Body.Netmap.Epoch,
|
||||
netmap.Body.Netmap.Nodes.Select(n => n.ToModel(netmap.MetaHeader.Version)).ToArray());
|
||||
netmap.Body.Netmap.Nodes
|
||||
.Select(n => n.ToModel(netmap.MetaHeader.Version))
|
||||
.ToArray());
|
||||
}
|
||||
}
|
|
@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
|
|||
|
||||
public static class PlacementPolicyMapper
|
||||
{
|
||||
public static PlacementPolicy ToGrpcMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
|
||||
public static PlacementPolicy ToMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
|
||||
{
|
||||
var pp = new PlacementPolicy
|
||||
{
|
||||
|
@ -14,9 +14,10 @@ public static class PlacementPolicyMapper
|
|||
Replicas = { },
|
||||
Unique = placementPolicy.Unique
|
||||
};
|
||||
|
||||
foreach (var replica in placementPolicy.Replicas)
|
||||
{
|
||||
pp.Replicas.Add(replica.ToGrpcMessage());
|
||||
pp.Replicas.Add(replica.ToMessage());
|
||||
}
|
||||
|
||||
return pp;
|
||||
|
|
|
@ -4,7 +4,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
|
|||
|
||||
public static class ReplicaMapper
|
||||
{
|
||||
public static Replica ToGrpcMessage(this ModelsV2.Netmap.Replica replica)
|
||||
public static Replica ToMessage(this ModelsV2.Netmap.Replica replica)
|
||||
{
|
||||
return new Replica
|
||||
{
|
||||
|
|
|
@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class ObjectAttributeMapper
|
||||
{
|
||||
public static Header.Types.Attribute ToGrpcMessage(this ObjectAttribute attribute)
|
||||
public static Header.Types.Attribute ToMessage(this ObjectAttribute attribute)
|
||||
{
|
||||
return new Header.Types.Attribute
|
||||
{
|
||||
|
|
|
@ -7,7 +7,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class ObjectFilterMapper
|
||||
{
|
||||
public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this IObjectFilter filter)
|
||||
public static SearchRequest.Types.Body.Types.Filter ToMessage(this IObjectFilter filter)
|
||||
{
|
||||
var objMatchTypeName = filter.MatchType switch
|
||||
{
|
||||
|
|
|
@ -10,7 +10,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class ObjectHeaderMapper
|
||||
{
|
||||
public static Header ToGrpcMessage(this ObjectHeader header)
|
||||
public static Header ToMessage(this ObjectHeader header)
|
||||
{
|
||||
var objTypeName = header.ObjectType switch
|
||||
{
|
||||
|
@ -22,14 +22,16 @@ public static class ObjectHeaderMapper
|
|||
|
||||
var head = new Header
|
||||
{
|
||||
ContainerId = header.ContainerId.ToGrpcMessage(),
|
||||
OwnerId = header!.OwnerId != null ? header.OwnerId.ToMessage() : null,
|
||||
Version = header!.Version != null ? header.Version.ToMessage() : null,
|
||||
ContainerId = header.ContainerId.ToMessage(),
|
||||
ObjectType = objTypeName,
|
||||
PayloadLength = header.PayloadLength
|
||||
};
|
||||
|
||||
foreach (var attribute in header.Attributes)
|
||||
{
|
||||
head.Attributes.Add(attribute.ToGrpcMessage());
|
||||
head.Attributes.Add(attribute.ToMessage());
|
||||
}
|
||||
|
||||
var split = header.Split;
|
||||
|
@ -37,17 +39,10 @@ public static class ObjectHeaderMapper
|
|||
{
|
||||
head.Split = new Header.Types.Split
|
||||
{
|
||||
Parent = split.Parent?.ToGrpcMessage(),
|
||||
ParentSignature = split.ParentSignature?.ToGrpcMessage(),
|
||||
ParentHeader = split.ParentHeader?.ToGrpcMessage(),
|
||||
Previous = split.Previous?.ToGrpcMessage(),
|
||||
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
||||
};
|
||||
|
||||
if (split.Children != null && split.Children.Count != 0)
|
||||
head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
|
||||
}
|
||||
|
||||
|
||||
return head;
|
||||
}
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class ObjectIdMapper
|
||||
{
|
||||
public static ObjectID ToGrpcMessage(this ObjectId objectId)
|
||||
public static ObjectID ToMessage(this ObjectId objectId)
|
||||
{
|
||||
return new ObjectID
|
||||
{
|
||||
|
|
|
@ -1,21 +1,42 @@
|
|||
using FrostFS.Refs;
|
||||
using FrostFS.SDK.Cryptography;
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
using Google.Protobuf;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using System;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||
|
||||
public static class OwnerIdMapper
|
||||
{
|
||||
public static OwnerID ToGrpcMessage(this OwnerId ownerId)
|
||||
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
|
||||
.SetSlidingExpiration(TimeSpan.FromHours(1))
|
||||
.SetSize(1);
|
||||
|
||||
public static OwnerID ToMessage(this OwnerId model)
|
||||
{
|
||||
return new OwnerID
|
||||
if (!Cache.Owners.TryGetValue(model, out OwnerID? message))
|
||||
{
|
||||
Value = ByteString.CopyFrom(ownerId.ToHash())
|
||||
};
|
||||
message = new OwnerID
|
||||
{
|
||||
Value = ByteString.CopyFrom(model.ToHash())
|
||||
};
|
||||
|
||||
Cache.Owners.Set(model, message, _oneHourExpiration);
|
||||
}
|
||||
|
||||
return message!;
|
||||
}
|
||||
|
||||
public static OwnerId ToModel(this OwnerID ownerId)
|
||||
public static OwnerId ToModel(this OwnerID message)
|
||||
{
|
||||
return new OwnerId(ownerId.ToString());
|
||||
if (!Cache.Owners.TryGetValue(message, out OwnerId? model))
|
||||
{
|
||||
model = new OwnerId(Base58.Encode(message.Value.ToByteArray()));
|
||||
|
||||
Cache.Owners.Set(message, model, _oneHourExpiration);
|
||||
}
|
||||
|
||||
return model!;
|
||||
}
|
||||
}
|
|
@ -6,14 +6,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class SignatureMapper
|
||||
{
|
||||
public static Refs.Signature ToGrpcMessage(this Signature signature)
|
||||
public static Refs.Signature ToMessage(this Signature signature)
|
||||
{
|
||||
var scheme = signature.Scheme switch
|
||||
{
|
||||
SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256,
|
||||
SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect,
|
||||
SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512,
|
||||
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme))
|
||||
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}",
|
||||
paramName: nameof(signature.Scheme))
|
||||
};
|
||||
|
||||
return new Refs.Signature
|
||||
|
|
|
@ -5,13 +5,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
|||
|
||||
public static class StatusMapper
|
||||
{
|
||||
public static ModelsV2.Status ToModel(this Status.Status status)
|
||||
public static ModelsV2.ResponseStatus ToModel(this Status.Status status)
|
||||
{
|
||||
if (status is null) return new ModelsV2.Status(StatusCode.Success);
|
||||
if (status is null)
|
||||
return new ModelsV2.ResponseStatus(StatusCode.Success);
|
||||
|
||||
var codeName = Enum.GetName(typeof(StatusCode), status.Code);
|
||||
|
||||
return codeName is null
|
||||
? throw new ArgumentException($"Unknown StatusCode. Value: '{status.Code}'.")
|
||||
: new ModelsV2.Status((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
|
||||
: new ModelsV2.ResponseStatus((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
|
||||
}
|
||||
}
|
|
@ -1,20 +1,74 @@
|
|||
using System.Collections;
|
||||
using System.Threading;
|
||||
using Version = FrostFS.Refs.Version;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||
|
||||
public static class VersionMapper
|
||||
{
|
||||
public static Version ToGrpcMessage(this ModelsV2.Version version)
|
||||
private static readonly Hashtable _cacheMessages = [];
|
||||
private static readonly Hashtable _cacheModels = [];
|
||||
private static SpinLock _spinlock = new();
|
||||
|
||||
public static Version ToMessage(this ModelsV2.Version model)
|
||||
{
|
||||
return new Version
|
||||
var key = model.Major << 16 + model.Minor;
|
||||
|
||||
if (!_cacheMessages.ContainsKey(key))
|
||||
{
|
||||
Major = (uint)version.Major,
|
||||
Minor = (uint)version.Minor
|
||||
};
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
_spinlock.Enter(ref lockTaken);
|
||||
var message = new Version
|
||||
{
|
||||
Major = (uint)model.Major,
|
||||
Minor = (uint)model.Minor
|
||||
};
|
||||
|
||||
_cacheMessages.Add(key, message);
|
||||
return message;
|
||||
}
|
||||
catch (System.ArgumentException)
|
||||
{
|
||||
// ignore attempt to add duplicate error.
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockTaken)
|
||||
_spinlock.Exit(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (Version)_cacheMessages[key];
|
||||
}
|
||||
|
||||
public static ModelsV2.Version ToModel(this Version version)
|
||||
public static ModelsV2.Version ToModel(this Version message)
|
||||
{
|
||||
return new ModelsV2.Version((int)version.Major, (int)version.Minor);
|
||||
var key = (int)message.Major << 16 + (int)message.Minor;
|
||||
|
||||
if (!_cacheModels.ContainsKey(key))
|
||||
{
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
_spinlock.Enter(ref lockTaken);
|
||||
var model = new ModelsV2.Version((int)message.Major, (int)message.Minor);
|
||||
|
||||
_cacheModels.Add(key, model);
|
||||
return model;
|
||||
}
|
||||
catch (System.ArgumentException)
|
||||
{
|
||||
// ignore attempt to add duplicate error.
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockTaken)
|
||||
_spinlock.Exit(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (ModelsV2.Version)_cacheModels[key];
|
||||
}
|
||||
}
|
|
@ -41,4 +41,10 @@ public sealed class PrmObjectPut : IContext, ISessionToken
|
|||
|
||||
/// <inheritdoc />
|
||||
public SessionToken? SessionToken { get; set; }
|
||||
|
||||
internal int MaxObjectSizeCache { get; set; }
|
||||
|
||||
internal ulong CurrentStreamPosition { get; set; } = 0;
|
||||
|
||||
internal ulong FullLength { get; set; } = 0;
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
|||
{
|
||||
internal async Task<ModelsV2.Container> GetContainerAsync(PrmContainerGet args)
|
||||
{
|
||||
GetRequest request = GetContainerRequest(args.ContainerId.ToGrpcMessage(), args.XHeaders);
|
||||
GetRequest request = GetContainerRequest(args.ContainerId.ToMessage(), args.XHeaders);
|
||||
|
||||
var response = await service.GetAsync(request, null, args.Context!.Deadline, args.Context.CancellationToken);
|
||||
|
||||
|
@ -34,12 +34,12 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
|||
{
|
||||
Body = new ListRequest.Types.Body
|
||||
{
|
||||
OwnerId = Context.Owner.ToGrpcMessage()
|
||||
OwnerId = Context.Owner.ToMessage()
|
||||
}
|
||||
};
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await service.ListAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -54,21 +54,21 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
|||
internal async Task<ContainerId> CreateContainerAsync(PrmContainerCreate args)
|
||||
{
|
||||
var ctx = args.Context!;
|
||||
var grpcContainer = args.Container.ToGrpcMessage();
|
||||
grpcContainer.OwnerId = Context.Owner.ToGrpcMessage();
|
||||
grpcContainer.Version = Context.Version.ToGrpcMessage();
|
||||
var grpcContainer = args.Container.ToMessage();
|
||||
grpcContainer.OwnerId = Context.Owner.ToMessage();
|
||||
grpcContainer.Version = Context.Version.ToMessage();
|
||||
|
||||
var request = new PutRequest
|
||||
{
|
||||
Body = new PutRequest.Types.Body
|
||||
{
|
||||
Container = grpcContainer,
|
||||
Signature = Context.Key.SignRFC6979(grpcContainer)
|
||||
Signature = Context.Key.ECDsaKey.SignRFC6979(grpcContainer)
|
||||
}
|
||||
};
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await service.PutAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -86,14 +86,14 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
|||
{
|
||||
Body = new DeleteRequest.Types.Body
|
||||
{
|
||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
||||
Signature = Context.Key.SignRFC6979(args.ContainerId.ToGrpcMessage().Value)
|
||||
ContainerId = args.ContainerId.ToMessage(),
|
||||
Signature = Context.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value)
|
||||
}
|
||||
};
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await service.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -113,7 +113,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
|||
};
|
||||
|
||||
request.AddMetaHeader(xHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
return request;
|
||||
}
|
||||
|
|
|
@ -49,7 +49,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
|||
};
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -63,7 +63,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
|||
var request = new NetworkInfoRequest();
|
||||
|
||||
request.AddMetaHeader(null);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -79,7 +79,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
|||
var request = new NetmapSnapshotRequest();
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
|
|
@ -13,13 +13,13 @@ using FrostFS.Session;
|
|||
using FrostFS.SDK.ModelsV2;
|
||||
using FrostFS.SDK.ClientV2.Extensions;
|
||||
using FrostFS.SDK.ClientV2.Parameters;
|
||||
using System.Buffers;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
||||
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment ctx) : ContextAccessor(ctx), ISessionProvider
|
||||
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment env) : ContextAccessor(env), ISessionProvider
|
||||
{
|
||||
readonly ObjectTools tools = new(ctx);
|
||||
readonly SessionProvider sessions = new (ctx);
|
||||
readonly SessionProvider sessions = new (env);
|
||||
|
||||
public async ValueTask<Session.SessionToken> GetOrCreateSession(ISessionToken args, Context ctx)
|
||||
{
|
||||
|
@ -35,8 +35,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
{
|
||||
Address = new Address
|
||||
{
|
||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
||||
ContainerId = args.ContainerId.ToMessage(),
|
||||
ObjectId = args.ObjectId.ToMessage()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
@ -46,11 +46,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
sessionToken.CreateObjectTokenContext(
|
||||
request.Body.Address,
|
||||
ObjectSessionContext.Types.Verb.Head,
|
||||
Context.Key);
|
||||
Context.Key.ECDsaKey);
|
||||
|
||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await client!.HeadAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -69,8 +69,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
{
|
||||
Address = new Address
|
||||
{
|
||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
||||
ContainerId = args.ContainerId.ToMessage(),
|
||||
ObjectId = args.ObjectId.ToMessage()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
@ -80,11 +80,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
sessionToken.CreateObjectTokenContext(
|
||||
request.Body.Address,
|
||||
ObjectSessionContext.Types.Verb.Get,
|
||||
Context.Key);
|
||||
Context.Key.ECDsaKey);
|
||||
|
||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
return await GetObject(request, ctx);
|
||||
}
|
||||
|
@ -98,8 +98,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
{
|
||||
Address = new Address
|
||||
{
|
||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
||||
ContainerId = args.ContainerId.ToMessage(),
|
||||
ObjectId = args.ObjectId.ToMessage()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
@ -109,10 +109,10 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
sessionToken.CreateObjectTokenContext(
|
||||
request.Body.Address,
|
||||
ObjectSessionContext.Types.Verb.Delete,
|
||||
Context.Key);
|
||||
Context.Key.ECDsaKey);
|
||||
|
||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await client.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -126,24 +126,24 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
{
|
||||
Body = new SearchRequest.Types.Body
|
||||
{
|
||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
||||
ContainerId = args.ContainerId.ToMessage(),
|
||||
Filters = { },
|
||||
Version = 1 // TODO: clarify this param
|
||||
}
|
||||
};
|
||||
|
||||
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToGrpcMessage()));
|
||||
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToMessage()));
|
||||
|
||||
var sessionToken = await GetOrCreateSession(args, ctx);
|
||||
|
||||
sessionToken.CreateObjectTokenContext(
|
||||
new Address { ContainerId = request.Body.ContainerId },
|
||||
ObjectSessionContext.Types.Verb.Search,
|
||||
Context.Key);
|
||||
Context.Key.ECDsaKey);
|
||||
|
||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var objectsIds = SearchObjects(request, ctx);
|
||||
|
||||
|
@ -153,24 +153,31 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
}
|
||||
}
|
||||
|
||||
internal Task<ObjectId> PutObjectAsync(PrmObjectPut args)
|
||||
internal async Task<ObjectId> PutObjectAsync(PrmObjectPut args)
|
||||
{
|
||||
if (args.Header == null)
|
||||
throw new ArgumentException("Value cannot be null", nameof(args.Header));
|
||||
|
||||
if (args.Payload == null)
|
||||
throw new ArgumentException("Value cannot be null", nameof(args.Payload));
|
||||
|
||||
|
||||
if (args.ClientCut)
|
||||
return PutClientCutObject(args);
|
||||
else
|
||||
return PutStreamObject(args);
|
||||
return await PutClientCutObject(args);
|
||||
else
|
||||
{
|
||||
if (args.Header.PayloadLength > 0)
|
||||
args.FullLength = args.Header.PayloadLength;
|
||||
else if (args.Payload.CanSeek)
|
||||
args.FullLength = (ulong)args.Payload.Length;
|
||||
|
||||
return (await PutStreamObject(args)).ObjectId;
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task<ObjectId> PutSingleObjectAsync(PrmSingleObjectPut args)
|
||||
{
|
||||
var ctx = args.Context!;
|
||||
var grpcObject = tools.CreateObject(args.FrostFsObject);
|
||||
var grpcObject = ObjectTools.CreateObject(args.FrostFsObject, env);
|
||||
|
||||
var request = new PutSingleRequest
|
||||
{
|
||||
|
@ -185,11 +192,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
sessionToken.CreateObjectTokenContext(
|
||||
new Address { ContainerId = grpcObject.Header.ContainerId, ObjectId = grpcObject.ObjectId},
|
||||
ObjectSessionContext.Types.Verb.Put,
|
||||
Context.Key);
|
||||
Context.Key.ECDsaKey);
|
||||
|
||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
var response = await client.PutSingleAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||
|
||||
|
@ -201,156 +208,136 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
private async Task<ObjectId> PutClientCutObject(PrmObjectPut args)
|
||||
{
|
||||
var ctx = args.Context!;
|
||||
|
||||
var tokenRaw = await GetOrCreateSession(args, ctx);
|
||||
var token = new ModelsV2.SessionToken(tokenRaw.Serialize());
|
||||
|
||||
args.SessionToken = token;
|
||||
|
||||
var payloadStream = args.Payload!;
|
||||
var header = args.Header!;
|
||||
|
||||
ObjectId? objectId;
|
||||
List<ObjectId> sentObjectIds = [];
|
||||
|
||||
FrostFsObject? currentObject;
|
||||
|
||||
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
|
||||
|
||||
var objectSize = (int)networkSettings.MaxObjectSize;
|
||||
|
||||
var fullLength = header.PayloadLength;
|
||||
|
||||
if (payloadStream.CanSeek)
|
||||
{
|
||||
objectSize = (int)Math.Min(objectSize, payloadStream.Length);
|
||||
|
||||
if (fullLength == 0)
|
||||
fullLength = (ulong)payloadStream.Length;
|
||||
}
|
||||
|
||||
if (fullLength == 0)
|
||||
throw new ArgumentException("Payload stream must be able to seek or PayloadLength must be specified");
|
||||
|
||||
var buffer = new byte[objectSize];
|
||||
if (payloadStream.CanSeek && fullLength == 0)
|
||||
fullLength = (ulong)payloadStream.Length;
|
||||
|
||||
var largeObject = new LargeObject(header.ContainerId);
|
||||
args.FullLength = fullLength;
|
||||
|
||||
var split = new Split();
|
||||
|
||||
while (true)
|
||||
if (args.MaxObjectSizeCache == 0)
|
||||
{
|
||||
var bytesCount = await payloadStream.ReadAsync(buffer, 0, objectSize);
|
||||
|
||||
split.Previous = sentObjectIds.LastOrDefault();
|
||||
|
||||
largeObject.Header.PayloadLength += (ulong)bytesCount;
|
||||
|
||||
currentObject = new FrostFsObject(header.ContainerId)
|
||||
.SetPayload(bytesCount < objectSize ? buffer[..bytesCount] : buffer)
|
||||
.SetSplit(split);
|
||||
|
||||
if (largeObject.PayloadLength == fullLength)
|
||||
break;
|
||||
|
||||
objectId = await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token });
|
||||
|
||||
sentObjectIds.Add(objectId!);
|
||||
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
|
||||
args.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize;
|
||||
}
|
||||
|
||||
if (sentObjectIds.Count != 0)
|
||||
var restBytes = fullLength - args.CurrentStreamPosition;
|
||||
var objectSize = restBytes > 0 ? (int)Math.Min((ulong)args.MaxObjectSizeCache, restBytes) : args.MaxObjectSizeCache;
|
||||
|
||||
//define collection capacity
|
||||
var restPart = (restBytes % (ulong)objectSize) > 0 ? 1 : 0;
|
||||
var objectsCount = fullLength > 0 ? (int)(restBytes / (ulong)objectSize) + restPart : 0;
|
||||
|
||||
List<ObjectId> sentObjectIds = new(objectsCount);
|
||||
|
||||
Split? split = null;
|
||||
|
||||
// keep attributes for the large object
|
||||
var attributes = args.Header!.Attributes;
|
||||
|
||||
// send all parts except the last one as separate Objects
|
||||
while (restBytes > (ulong)args.MaxObjectSizeCache)
|
||||
{
|
||||
largeObject.AddAttributes(args.Header!.Attributes);
|
||||
|
||||
currentObject.SetParent(largeObject);
|
||||
if (split == null)
|
||||
{
|
||||
split = new Split();
|
||||
args.Header!.Attributes = [];
|
||||
}
|
||||
|
||||
var putSingleObjectParams = new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token };
|
||||
split!.Previous = sentObjectIds.LastOrDefault();
|
||||
args.Header!.Split = split;
|
||||
|
||||
objectId = await PutSingleObjectAsync(putSingleObjectParams);
|
||||
var result = await PutStreamObject(args);
|
||||
|
||||
sentObjectIds.Add(objectId);
|
||||
sentObjectIds.Add(result.ObjectId);
|
||||
|
||||
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
|
||||
restBytes -= (ulong)result.ObjectSize;
|
||||
}
|
||||
|
||||
// send the last part and create linkObject
|
||||
if (sentObjectIds.Count > 0)
|
||||
{
|
||||
var largeObjectHeader = new ObjectHeader(header.ContainerId) { PayloadLength = fullLength };
|
||||
|
||||
largeObjectHeader.Attributes.AddRange(attributes);
|
||||
|
||||
args.Header.Split!.ParentHeader = largeObjectHeader;
|
||||
|
||||
var result = await PutStreamObject(args);
|
||||
|
||||
sentObjectIds.Add(result.ObjectId);
|
||||
|
||||
var linkObject = new LinkObject(header.ContainerId, split!.SplitId, largeObjectHeader)
|
||||
.AddChildren(sentObjectIds);
|
||||
|
||||
linkObject.Header.Attributes.Clear();
|
||||
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject) { Context = args.Context});
|
||||
|
||||
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject, ctx){ SessionToken = token });
|
||||
|
||||
return tools.CalculateObjectId(largeObject.Header);
|
||||
return split.Parent!;
|
||||
}
|
||||
|
||||
currentObject
|
||||
.SetSplit(null)
|
||||
.AddAttributes(args.Header!.Attributes);
|
||||
|
||||
return await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx));
|
||||
// We are here if the payload is placed to one Object. It means no cut action, just simple PUT.
|
||||
var singlePartResult = await PutStreamObject(args);
|
||||
|
||||
return singlePartResult.ObjectId;
|
||||
}
|
||||
|
||||
private async Task<ObjectId> PutStreamObject(PrmObjectPut args)
|
||||
struct PutObjectResult(ObjectId objectId, int objectSize)
|
||||
{
|
||||
public ObjectId ObjectId = objectId;
|
||||
public int ObjectSize = objectSize;
|
||||
}
|
||||
|
||||
private async Task<PutObjectResult> PutStreamObject(PrmObjectPut args)
|
||||
{
|
||||
var ctx = args.Context!;
|
||||
var payload = args.Payload!;
|
||||
var header = args.Header!;
|
||||
|
||||
var hdr = header.ToGrpcMessage();
|
||||
hdr.OwnerId = Context.Owner.ToGrpcMessage();
|
||||
hdr.Version = Context.Version.ToGrpcMessage();
|
||||
var chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
||||
|
||||
var oid = new ObjectID { Value = hdr.Sha256() };
|
||||
var restBytes = args.FullLength - args.CurrentStreamPosition;
|
||||
|
||||
var initRequest = new PutRequest
|
||||
chunkSize = (int)Math.Min(restBytes, (ulong)chunkSize);
|
||||
|
||||
var chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
|
||||
var sentBytes = 0;
|
||||
|
||||
// 0 means no limit from client, so server side cut is performed
|
||||
var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0;
|
||||
|
||||
var stream = await GetUploadStream(args, ctx);
|
||||
|
||||
while (objectLimitSize == 0 || sentBytes < objectLimitSize)
|
||||
{
|
||||
Body = new PutRequest.Types.Body
|
||||
{
|
||||
Init = new PutRequest.Types.Body.Types.Init
|
||||
{
|
||||
Header = hdr
|
||||
}
|
||||
}
|
||||
};
|
||||
// send chanks limited to default or user's settings
|
||||
var bufferSize = objectLimitSize > 0 ?
|
||||
(int)Math.Min(objectLimitSize - sentBytes, chunkSize)
|
||||
: chunkSize;
|
||||
|
||||
var sessionToken = await GetOrCreateSession(args, ctx);
|
||||
|
||||
sessionToken.CreateObjectTokenContext(
|
||||
new Address { ContainerId = hdr.ContainerId, ObjectId = oid },
|
||||
ObjectSessionContext.Types.Verb.Put,
|
||||
Context.Key
|
||||
);
|
||||
|
||||
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
initRequest.Sign(Context.Key);
|
||||
|
||||
using var stream = await PutObjectInit(initRequest, ctx);
|
||||
|
||||
var bufferSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
||||
|
||||
if (payload.CanSeek)
|
||||
{
|
||||
bufferSize = (int)Math.Min(payload.Length, bufferSize);
|
||||
}
|
||||
else if (header.PayloadLength > 0)
|
||||
{
|
||||
bufferSize = (int)Math.Min((long)header.PayloadLength, bufferSize);
|
||||
}
|
||||
|
||||
var buffer = new byte[bufferSize];
|
||||
|
||||
while (true)
|
||||
{
|
||||
var bytesCount = await payload.ReadAsync(buffer, 0, bufferSize, ctx.CancellationToken);
|
||||
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken);
|
||||
|
||||
if (bytesCount == 0)
|
||||
break;
|
||||
|
||||
var chunkRequest = new PutRequest(initRequest)
|
||||
sentBytes += bytesCount;
|
||||
|
||||
var chunkRequest = new PutRequest
|
||||
{
|
||||
Body = new PutRequest.Types.Body
|
||||
{
|
||||
Chunk = ByteString.CopyFrom(buffer.AsSpan()[..bytesCount]),
|
||||
},
|
||||
VerifyHeader = null
|
||||
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount)
|
||||
}
|
||||
};
|
||||
|
||||
chunkRequest.Sign(Context.Key);
|
||||
chunkRequest.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
await stream.Write(chunkRequest);
|
||||
}
|
||||
|
@ -358,7 +345,49 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
|||
var response = await stream.Close();
|
||||
Verifier.CheckResponse(response);
|
||||
|
||||
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
|
||||
return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes);
|
||||
}
|
||||
|
||||
private async Task<ObjectStreamer> GetUploadStream(PrmObjectPut args, Context ctx)
|
||||
{
|
||||
var header = args.Header!;
|
||||
|
||||
header.OwnerId = Context.Owner;
|
||||
header.Version = Context.Version;
|
||||
|
||||
var grpcHeader = header.ToMessage();
|
||||
|
||||
if (header.Split != null)
|
||||
{
|
||||
ObjectTools.SetSplitValues(grpcHeader, header.Split, env);
|
||||
}
|
||||
|
||||
var oid = new ObjectID { Value = grpcHeader.Sha256() };
|
||||
|
||||
var initRequest = new PutRequest
|
||||
{
|
||||
Body = new PutRequest.Types.Body
|
||||
{
|
||||
Init = new PutRequest.Types.Body.Types.Init
|
||||
{
|
||||
Header = grpcHeader
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
var sessionToken = await GetOrCreateSession(args, ctx);
|
||||
|
||||
sessionToken.CreateObjectTokenContext(
|
||||
new Address { ContainerId = grpcHeader.ContainerId, ObjectId = oid },
|
||||
ObjectSessionContext.Types.Verb.Put,
|
||||
Context.Key.ECDsaKey
|
||||
);
|
||||
|
||||
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
|
||||
|
||||
initRequest.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
return await PutObjectInit(initRequest, ctx);
|
||||
}
|
||||
|
||||
private async Task<FrostFsObject> GetObject(GetRequest request, Context ctx)
|
||||
|
|
|
@ -21,13 +21,13 @@ internal class SessionServiceProvider : ContextAccessor
|
|||
{
|
||||
Body = new CreateRequest.Types.Body
|
||||
{
|
||||
OwnerId = Context.Owner.ToGrpcMessage(),
|
||||
OwnerId = Context.Owner.ToMessage(),
|
||||
Expiration = args.Expiration
|
||||
}
|
||||
};
|
||||
|
||||
request.AddMetaHeader(args.XHeaders);
|
||||
request.Sign(Context.Key);
|
||||
request.Sign(Context.Key.ECDsaKey);
|
||||
|
||||
return await CreateSession(request, args.Context!);
|
||||
}
|
||||
|
|
|
@ -1,7 +1,9 @@
|
|||
using FrostFS.SDK.ModelsV2;
|
||||
using Google.Protobuf;
|
||||
using Grpc.Net.Client;
|
||||
using System;
|
||||
using System.Security.Cryptography;
|
||||
using FrostFS.SDK.Cryptography;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
||||
|
@ -9,11 +11,12 @@ public class ClientEnvironment(Client client, ECDsa key, OwnerId owner, GrpcChan
|
|||
{
|
||||
internal OwnerId Owner { get; } = owner;
|
||||
internal GrpcChannel Channel { get; private set; } = channel;
|
||||
internal ECDsa Key { get; } = key;
|
||||
internal ModelsV2.Version Version { get; } = version;
|
||||
internal NetworkSettings? NetworkSettings { get; set; }
|
||||
|
||||
internal Client Client { get; set; } = client;
|
||||
internal Client Client { get; } = client;
|
||||
|
||||
internal ClientKey Key { get; } = new ClientKey(key);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
|
|
|
@ -7,54 +7,36 @@ using FrostFS.Refs;
|
|||
using FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||
using FrostFS.SDK.Cryptography;
|
||||
using FrostFS.SDK.ModelsV2;
|
||||
using static FrostFS.Object.Header.Types;
|
||||
|
||||
namespace FrostFS.SDK.ClientV2;
|
||||
|
||||
internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
|
||||
internal class ObjectTools
|
||||
{
|
||||
internal ObjectId CalculateObjectId(ObjectHeader header)
|
||||
internal static ObjectId CalculateObjectId(ObjectHeader header, ClientEnvironment env)
|
||||
{
|
||||
var grpcHeader = CreateHeader(header, []);
|
||||
var grpcHeader = CreateHeader(header, [], env);
|
||||
|
||||
if (header.Split != null)
|
||||
SetSplitValues(grpcHeader, header.Split, env);
|
||||
|
||||
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
|
||||
}
|
||||
|
||||
internal Object.Object CreateObject(FrostFsObject @object)
|
||||
internal static Object.Object CreateObject(FrostFsObject @object, ClientEnvironment env)
|
||||
{
|
||||
var grpcHeader = @object.Header.ToGrpcMessage();
|
||||
|
||||
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
|
||||
grpcHeader.Version = Context.Version.ToGrpcMessage();
|
||||
@object.Header.OwnerId = env.Owner;
|
||||
@object.Header.Version = env.Version;
|
||||
|
||||
var grpcHeader = @object.Header.ToMessage();
|
||||
|
||||
grpcHeader.PayloadLength = (ulong)@object.Payload.Length;
|
||||
grpcHeader.PayloadHash = Sha256Checksum(@object.Payload);
|
||||
|
||||
|
||||
var split = @object.Header.Split;
|
||||
if (split != null)
|
||||
{
|
||||
grpcHeader.Split = new Header.Types.Split
|
||||
{
|
||||
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
||||
};
|
||||
|
||||
if (split.Children != null && split.Children.Count != 0)
|
||||
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
|
||||
|
||||
if (split.ParentHeader is not null)
|
||||
{
|
||||
var grpcParentHeader = CreateHeader(split.ParentHeader, []);
|
||||
|
||||
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
||||
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
||||
grpcHeader.Split.ParentSignature = new Refs.Signature
|
||||
{
|
||||
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
|
||||
Sign = ByteString.CopyFrom(Context.Key.SignData(grpcHeader.Split.Parent.ToByteArray())),
|
||||
};
|
||||
|
||||
split.Parent = grpcHeader.Split.Parent.ToModel();
|
||||
}
|
||||
|
||||
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
|
||||
SetSplitValues(grpcHeader, split, env);
|
||||
}
|
||||
|
||||
var obj = new Object.Object
|
||||
|
@ -66,21 +48,49 @@ internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
|
|||
|
||||
obj.Signature = new Refs.Signature
|
||||
{
|
||||
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
|
||||
Sign = ByteString.CopyFrom(Context.Key.SignData(obj.ObjectId.ToByteArray())),
|
||||
Key = env.Key.PublicKeyProto,
|
||||
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(obj.ObjectId.ToByteArray())),
|
||||
};
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
internal Header CreateHeader(ObjectHeader header, byte[]? payload)
|
||||
internal static void SetSplitValues(Header grpcHeader, ModelsV2.Split split, ClientEnvironment env)
|
||||
{
|
||||
var grpcHeader = header.ToGrpcMessage();
|
||||
|
||||
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
|
||||
grpcHeader.Version = Context.Version.ToGrpcMessage();
|
||||
grpcHeader.Split = new Header.Types.Split
|
||||
{
|
||||
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
||||
};
|
||||
|
||||
if (payload != null)
|
||||
if (split.Children != null && split.Children.Count != 0)
|
||||
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
|
||||
|
||||
if (split.ParentHeader is not null)
|
||||
{
|
||||
var grpcParentHeader = CreateHeader(split.ParentHeader, [], env);
|
||||
|
||||
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
||||
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
||||
grpcHeader.Split.ParentSignature = new Refs.Signature
|
||||
{
|
||||
Key = env.Key.PublicKeyProto,
|
||||
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray())),
|
||||
};
|
||||
|
||||
split.Parent = grpcHeader.Split.Parent.ToModel();
|
||||
}
|
||||
|
||||
grpcHeader.Split.Previous = split.Previous?.ToMessage();
|
||||
}
|
||||
|
||||
internal static Header CreateHeader(ObjectHeader header, byte[]? payload, ClientEnvironment env)
|
||||
{
|
||||
var grpcHeader = header.ToMessage();
|
||||
|
||||
grpcHeader.OwnerId = env.Owner.ToMessage();
|
||||
grpcHeader.Version = env.Version.ToMessage();
|
||||
|
||||
if (payload != null) // && payload.Length > 0
|
||||
grpcHeader.PayloadHash = Sha256Checksum(payload);
|
||||
|
||||
return grpcHeader;
|
||||
|
|
|
@ -17,7 +17,7 @@ public static class RequestConstructor
|
|||
if (request.MetaHeader is not null)
|
||||
return;
|
||||
|
||||
request.MetaHeader = MetaHeader.Default().ToGrpcMessage();
|
||||
request.MetaHeader = MetaHeader.Default().ToMessage();
|
||||
|
||||
if (sessionToken != null)
|
||||
request.MetaHeader.SessionToken = sessionToken;
|
||||
|
|
|
@ -69,7 +69,7 @@ public static class RequestSigner
|
|||
var hash = new byte[65];
|
||||
hash[0] = 0x04;
|
||||
|
||||
key.SignHash(SHA512.Create().ComputeHash(data)).CopyTo(hash, 1);
|
||||
key.SignHash(data.Sha512()).CopyTo(hash, 1);
|
||||
|
||||
return hash;
|
||||
}
|
||||
|
|
|
@ -60,7 +60,7 @@ public static class Verifier
|
|||
|
||||
public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
|
||||
{
|
||||
return key.VerifyHash(SHA512.Create().ComputeHash(data), sig[1..]);
|
||||
return key.VerifyHash(data.Sha512(), sig[1..]);
|
||||
}
|
||||
|
||||
public static bool VerifyMessagePart(this Signature sig, IMessage data)
|
||||
|
@ -101,7 +101,8 @@ public static class Verifier
|
|||
throw new FormatException($"invalid response, type={resp.GetType()}");
|
||||
|
||||
var status = resp.MetaHeader.Status.ToModel();
|
||||
if (!status.IsSuccess)
|
||||
|
||||
if (status != null && !status.IsSuccess)
|
||||
throw new ResponseException(status);
|
||||
}
|
||||
|
||||
|
|
|
@ -51,13 +51,13 @@ public static class Base58
|
|||
int digit = Alphabet.IndexOf(input[i]);
|
||||
if (digit < 0)
|
||||
throw new FormatException($"Invalid Base58 character '{input[i]}' at position {i}");
|
||||
|
||||
|
||||
bi = bi * Alphabet.Length + digit;
|
||||
}
|
||||
|
||||
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
|
||||
var leadingZeros = new byte[leadingZeroCount];
|
||||
|
||||
|
||||
if (bi.IsZero)
|
||||
return leadingZeros;
|
||||
|
||||
|
@ -65,9 +65,9 @@ public static class Base58
|
|||
|
||||
var firstNonZeroIndex = 0;
|
||||
|
||||
while(bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
|
||||
while (bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
|
||||
firstNonZeroIndex++;
|
||||
|
||||
|
||||
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
|
||||
|
||||
return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros);
|
||||
|
|
|
@ -1,11 +1,18 @@
|
|||
using Google.Protobuf;
|
||||
using Org.BouncyCastle.Crypto.Digests;
|
||||
using System.Security.Cryptography;
|
||||
using System.Threading;
|
||||
|
||||
namespace FrostFS.SDK.Cryptography;
|
||||
|
||||
public static class Extentions
|
||||
{
|
||||
private static readonly SHA256 _sha256 = SHA256.Create();
|
||||
private static SpinLock _spinlockSha256 = new();
|
||||
|
||||
private static readonly SHA512 _sha512 = SHA512.Create();
|
||||
private static SpinLock _spinlockSha512 = new();
|
||||
|
||||
internal static byte[] RIPEMD160(this byte[] value)
|
||||
{
|
||||
var hash = new byte[20];
|
||||
|
@ -15,15 +22,41 @@ public static class Extentions
|
|||
digest.DoFinal(hash, 0);
|
||||
return hash;
|
||||
}
|
||||
|
||||
public static byte[] Sha256(this byte[] value)
|
||||
{
|
||||
var sha256 = SHA256.Create();
|
||||
return sha256.ComputeHash(value);
|
||||
}
|
||||
|
||||
public static ByteString Sha256(this IMessage data)
|
||||
{
|
||||
return ByteString.CopyFrom(data.ToByteArray().Sha256());
|
||||
}
|
||||
|
||||
public static byte[] Sha256(this byte[] value)
|
||||
{
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
_spinlockSha256.Enter(ref lockTaken);
|
||||
|
||||
return _sha256.ComputeHash(value);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockTaken)
|
||||
_spinlockSha256.Exit(false);
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] Sha512(this byte[] value)
|
||||
{
|
||||
bool lockTaken = false;
|
||||
try
|
||||
{
|
||||
_spinlockSha512.Enter(ref lockTaken);
|
||||
|
||||
return _sha512.ComputeHash(value);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (lockTaken)
|
||||
_spinlockSha512.Exit(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -133,8 +133,9 @@ public static class KeyExtension
|
|||
public static ECDsa LoadPrivateKey(this byte[] privateKey)
|
||||
{
|
||||
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
||||
var publicKey =
|
||||
secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)).GetEncoded(false)[1..];
|
||||
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
|
||||
.GetEncoded(false)[1..];
|
||||
|
||||
var key = ECDsa.Create(new ECParameters
|
||||
{
|
||||
Curve = ECCurve.NamedCurves.nistP256,
|
||||
|
|
|
@ -112,7 +112,7 @@ internal class Murmur3_128 : HashAlgorithm
|
|||
h2 = seed;
|
||||
}
|
||||
|
||||
private ulong Fimix64(ulong k)
|
||||
private static ulong Fimix64(ulong k)
|
||||
{
|
||||
k ^= k >> 33;
|
||||
k *= 0xff51afd7ed558ccd;
|
||||
|
|
|
@ -77,14 +77,7 @@ namespace System
|
|||
{
|
||||
get
|
||||
{
|
||||
if (_value < 0)
|
||||
{
|
||||
return ~_value;
|
||||
}
|
||||
else
|
||||
{
|
||||
return _value;
|
||||
}
|
||||
return _value < 0 ? ~_value : _value;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -105,12 +98,9 @@ namespace System
|
|||
var offset = _value;
|
||||
if (IsFromEnd)
|
||||
{
|
||||
// offset = length - (~value)
|
||||
// offset = length + (~(~value) + 1)
|
||||
// offset = length + value + 1
|
||||
|
||||
offset += length + 1;
|
||||
}
|
||||
|
||||
return offset;
|
||||
}
|
||||
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj" />
|
||||
<ProjectReference Include="..\FrostFS.SDK.ProtosV2\FrostFS.SDK.ProtosV2.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
using FrostFS.SDK.Cryptography;
|
||||
using System;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
|
@ -8,15 +9,9 @@ public class CheckSum
|
|||
// type is always Sha256
|
||||
public byte[]? Hash { get; set; }
|
||||
|
||||
public static byte[] GetHash(byte[] content)
|
||||
{
|
||||
var sha256 = SHA256.Create();
|
||||
return sha256.ComputeHash(content);
|
||||
}
|
||||
|
||||
public static CheckSum CreateCheckSum(byte[] content)
|
||||
{
|
||||
return new CheckSum { Hash = GetHash(content) };
|
||||
return new CheckSum { Hash = content.Sha256() };
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
|
|
|
@ -56,19 +56,17 @@ public class FrostFsObject
|
|||
/// Applied only for the last Object in chain in case of manual multipart uploading
|
||||
/// </summary>
|
||||
/// <param name="largeObject">Parent for multipart object</param>
|
||||
public void SetParent(LargeObject largeObject)
|
||||
public void SetParent(ObjectHeader largeObjectHeader)
|
||||
{
|
||||
if (Header?.Split == null)
|
||||
throw new Exception("The object is not initialized properly");
|
||||
|
||||
Header.Split.ParentHeader = largeObject.Header;
|
||||
Header.Split.ParentHeader = largeObjectHeader;
|
||||
}
|
||||
}
|
||||
|
||||
public class LargeObject(ContainerId container) : FrostFsObject(container)
|
||||
{
|
||||
private readonly SHA256 payloadHash = SHA256.Create();
|
||||
|
||||
public ulong PayloadLength
|
||||
{
|
||||
get { return Header!.PayloadLength; }
|
||||
|
@ -77,11 +75,11 @@ public class LargeObject(ContainerId container) : FrostFsObject(container)
|
|||
|
||||
public class LinkObject : FrostFsObject
|
||||
{
|
||||
public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId)
|
||||
public LinkObject(ContainerId containerId, SplitId splitId, ObjectHeader largeObjectHeader) : base (containerId)
|
||||
{
|
||||
Header!.Split = new Split(splitId)
|
||||
{
|
||||
ParentHeader = largeObject.Header
|
||||
ParentHeader = largeObjectHeader
|
||||
};
|
||||
}
|
||||
}
|
|
@ -1,5 +1,6 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
|
||||
namespace FrostFS.SDK.ModelsV2;
|
||||
|
||||
public class Split(SplitId splitId)
|
||||
|
@ -19,4 +20,6 @@ public class Split(SplitId splitId)
|
|||
public ObjectHeader? ParentHeader { get; set; }
|
||||
|
||||
public List<ObjectId> Children { get; } = [];
|
||||
|
||||
public Refs.Signature ParentSignatureGrpc { get; set; }
|
||||
}
|
||||
|
|
|
@ -2,7 +2,7 @@ using FrostFS.SDK.ModelsV2.Enums;
|
|||
|
||||
namespace FrostFS.SDK.ModelsV2;
|
||||
|
||||
public class Status(StatusCode code, string? message = null)
|
||||
public class ResponseStatus(StatusCode code, string? message = null)
|
||||
{
|
||||
public StatusCode Code { get; set; } = code;
|
||||
public string Message { get; set; } = message ?? string.Empty;
|
|
@ -110,6 +110,6 @@ public class ContainerTest : ContainerTestsBase
|
|||
|
||||
var request = Mocker.Requests.First();
|
||||
|
||||
Assert.Equal(cid.ToGrpcMessage(), request.Request.Body.ContainerId);
|
||||
Assert.Equal(cid.ToMessage(), request.Request.Body.ContainerId);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -19,14 +19,14 @@ public class AsyncStreamReaderMock(string key, ObjectHeader objectHeader) : Serv
|
|||
{
|
||||
var header = new Header
|
||||
{
|
||||
ContainerId = objectHeader.ContainerId.ToGrpcMessage(),
|
||||
ContainerId = objectHeader.ContainerId.ToMessage(),
|
||||
PayloadLength = objectHeader.PayloadLength,
|
||||
Version = objectHeader.Version!.ToGrpcMessage(),
|
||||
OwnerId = objectHeader.OwnerId!.ToGrpcMessage()
|
||||
Version = objectHeader.Version!.ToMessage(),
|
||||
OwnerId = objectHeader.OwnerId!.ToMessage()
|
||||
};
|
||||
|
||||
foreach (var attr in objectHeader.Attributes)
|
||||
header.Attributes.Add(attr.ToGrpcMessage());
|
||||
header.Attributes.Add(attr.ToMessage());
|
||||
|
||||
var response = new GetResponse
|
||||
{
|
||||
|
|
|
@ -27,7 +27,7 @@ public abstract class ServiceBase(string key)
|
|||
public static BasicAcl DefaultAcl { get; } = BasicAcl.PublicRW;
|
||||
public static PlacementPolicy DefaultPlacementPolicy { get; } = new PlacementPolicy(true, new Replica(1));
|
||||
|
||||
public Metadata ResponseMetaData => [];
|
||||
public static Metadata ResponseMetaData => [];
|
||||
|
||||
protected ResponseVerificationHeader GetResponseVerificationHeader(IResponse response)
|
||||
{
|
||||
|
@ -58,7 +58,7 @@ public abstract class ServiceBase(string key)
|
|||
|
||||
public ResponseMetaHeader ResponseMetaHeader => new()
|
||||
{
|
||||
Version = Version.ToGrpcMessage(),
|
||||
Version = Version.ToMessage(),
|
||||
Epoch = 100,
|
||||
Ttl = 1
|
||||
};
|
||||
|
|
|
@ -19,7 +19,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
|||
{
|
||||
var mock = new Mock<ContainerService.ContainerServiceClient>();
|
||||
|
||||
var grpcVersion = Version.ToGrpcMessage();
|
||||
var grpcVersion = Version.ToMessage();
|
||||
|
||||
PutResponse putResponse = new()
|
||||
{
|
||||
|
@ -32,7 +32,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
|||
},
|
||||
MetaHeader = new ResponseMetaHeader
|
||||
{
|
||||
Version = Version is null ? DefaultVersion.ToGrpcMessage() : Version.ToGrpcMessage(),
|
||||
Version = (Version is null ? DefaultVersion : Version).ToMessage(),
|
||||
Epoch = 100,
|
||||
Ttl = 1
|
||||
}
|
||||
|
@ -69,7 +69,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
|||
Version = grpcVersion,
|
||||
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
|
||||
BasicAcl = (uint)Acl,
|
||||
PlacementPolicy = PlacementPolicy.ToGrpcMessage()
|
||||
PlacementPolicy = PlacementPolicy.ToMessage()
|
||||
}
|
||||
},
|
||||
MetaHeader = ResponseMetaHeader
|
||||
|
|
|
@ -38,12 +38,12 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
|||
HeadResponse ??= new Header
|
||||
{
|
||||
CreationEpoch = 99,
|
||||
ContainerId = ObjectHeader.ContainerId.ToGrpcMessage(),
|
||||
ContainerId = ObjectHeader.ContainerId.ToMessage(),
|
||||
ObjectType = ObjectType.Regular,
|
||||
OwnerId = ObjectHeader.OwnerId!.ToGrpcMessage(),
|
||||
OwnerId = ObjectHeader.OwnerId!.ToMessage(),
|
||||
PayloadLength = 1,
|
||||
PayloadHash = new Refs.Checksum { Type = Refs.ChecksumType.Sha256, Sum = ByteString.CopyFrom(SHA256.HashData([0xff])) },
|
||||
Version = ObjectHeader.Version!.ToGrpcMessage()
|
||||
Version = ObjectHeader.Version!.ToMessage()
|
||||
};
|
||||
|
||||
HeadResponse headResponse = new()
|
||||
|
@ -89,7 +89,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
|||
|
||||
}
|
||||
|
||||
if (ResultObjectId != null)
|
||||
if (ResultObjectIds != null)
|
||||
{
|
||||
PutResponse putResponse = new()
|
||||
{
|
||||
|
@ -97,7 +97,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
|||
{
|
||||
ObjectId = new Refs.ObjectID
|
||||
{
|
||||
Value = ByteString.CopyFrom(ResultObjectId)
|
||||
Value = ByteString.CopyFrom(ResultObjectIds.ElementAt(0))
|
||||
}
|
||||
},
|
||||
MetaHeader = ResponseMetaHeader,
|
||||
|
@ -156,8 +156,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
|||
{
|
||||
Tombstone = new Refs.Address
|
||||
{
|
||||
ContainerId = ObjectHeader!.ContainerId.ToGrpcMessage(),
|
||||
ObjectId = ObjectId.ToGrpcMessage()
|
||||
ContainerId = ObjectHeader!.ContainerId.ToMessage(),
|
||||
ObjectId = ObjectId.ToMessage()
|
||||
}
|
||||
},
|
||||
MetaHeader = ResponseMetaHeader
|
||||
|
@ -195,7 +195,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
|||
|
||||
public Header? HeadResponse { get; set; }
|
||||
|
||||
public byte[]? ResultObjectId { get; set; }
|
||||
public List<byte[]>? ResultObjectIds { get; set; }
|
||||
|
||||
public ClientStreamWriter? ClientStreamWriter { get; private set; } = new ();
|
||||
|
||||
|
|
|
@ -7,6 +7,7 @@ using FrostFS.SDK.Cryptography;
|
|||
using FrostFS.SDK.ModelsV2;
|
||||
using FrostFS.SDK.ModelsV2.Enums;
|
||||
using FrostFS.SDK.ModelsV2.Netmap;
|
||||
using FrostFS.SDK.ProtosV2.Interfaces;
|
||||
using Google.Protobuf;
|
||||
using Microsoft.Extensions.Options;
|
||||
using System.Security.Cryptography;
|
||||
|
@ -82,7 +83,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
Assert.NotNull(result);
|
||||
|
||||
Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), result.Header.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, result.Header.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength);
|
||||
Assert.Single(result.Header.Attributes);
|
||||
Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key);
|
||||
|
@ -92,7 +93,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
[Fact]
|
||||
public async void PutObjectTest()
|
||||
{
|
||||
Mocker.ResultObjectId = SHA256.HashData([]);
|
||||
Mocker.ResultObjectIds = new([SHA256.HashData([])]);
|
||||
|
||||
Random rnd = new();
|
||||
var bytes = new byte[1024];
|
||||
|
@ -113,7 +114,7 @@ public class ObjectTest : ObjectTestsBase
|
|||
var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body;
|
||||
|
||||
Assert.NotNull(result);
|
||||
Assert.Equal(Mocker.ResultObjectId, result.ToHash());
|
||||
Assert.Equal(Mocker.ResultObjectIds.First(), result.ToHash());
|
||||
|
||||
Assert.True(Mocker.ClientStreamWriter.CompletedTask);
|
||||
|
||||
|
@ -137,57 +138,85 @@ public class ObjectTest : ObjectTestsBase
|
|||
{
|
||||
Header = Mocker.ObjectHeader,
|
||||
Payload = new MemoryStream(bytes),
|
||||
BufferMaxSize = 1024,
|
||||
ClientCut = true
|
||||
};
|
||||
|
||||
Random rnd = new();
|
||||
|
||||
List<byte[]> objIds = new([new byte[32], new byte[32], new byte[32]]);
|
||||
rnd.NextBytes(objIds.ElementAt(0));
|
||||
rnd.NextBytes(objIds.ElementAt(1));
|
||||
rnd.NextBytes(objIds.ElementAt(2));
|
||||
|
||||
Mocker.ResultObjectIds = objIds;
|
||||
|
||||
var result = await GetClient().PutObjectAsync(param);
|
||||
|
||||
var sentMessages = Mocker.PutSingleRequests.ToArray();
|
||||
var singleObjects = Mocker.PutSingleRequests.ToArray();
|
||||
|
||||
Assert.Equal(4, sentMessages.Length);
|
||||
var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
|
||||
|
||||
var object_0 = sentMessages[0].Body.Object;
|
||||
var object_1 = sentMessages[1].Body.Object;
|
||||
var object_2 = sentMessages[2].Body.Object;
|
||||
var object_3 = sentMessages[3].Body.Object;
|
||||
Assert.Single(singleObjects);
|
||||
|
||||
Assert.NotNull(object_0.Header.Split.SplitId);
|
||||
Assert.Null(object_0.Header.Split.Previous);
|
||||
Assert.Equal(blockSize, (int)object_0.Header.PayloadLength);
|
||||
Assert.Equal(bytes[..blockSize], object_0.Payload);
|
||||
Assert.True(object_0.Header.Attributes.Count == 0);
|
||||
Assert.Equal(11, streamObjects.Length);
|
||||
|
||||
Assert.Equal(object_0.Header.Split.SplitId, object_1.Header.Split.SplitId);
|
||||
Assert.Equal(object_0.ObjectId, object_1.Header.Split.Previous);
|
||||
Assert.Equal(blockSize, (int)object_1.Header.PayloadLength);
|
||||
Assert.Equal(bytes[blockSize..(blockSize * 2)], object_1.Payload);
|
||||
Assert.True(object_1.Header.Attributes.Count == 0);
|
||||
var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
|
||||
|
||||
Assert.Equal(3, bodies.Count(b => b.Init != null));
|
||||
Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
|
||||
|
||||
Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
|
||||
|
||||
var linkObject = singleObjects[0].Body.Object;
|
||||
var header1 = bodies[0].Init.Header;
|
||||
var header2 = bodies[4].Init.Header;
|
||||
var header3 = bodies[8].Init.Header;
|
||||
|
||||
var payload1 = bodies[1].Chunk
|
||||
.Concat(bodies[2].Chunk)
|
||||
.Concat(bodies[3].Chunk)
|
||||
.ToArray();
|
||||
|
||||
var payload2 = bodies[5].Chunk
|
||||
.Concat(bodies[6].Chunk)
|
||||
.Concat(bodies[7].Chunk)
|
||||
.ToArray();
|
||||
|
||||
var payload3 = bodies[9].Chunk
|
||||
.Concat(bodies[10].Chunk)
|
||||
.ToArray();
|
||||
|
||||
Assert.NotNull(header1.Split.SplitId);
|
||||
Assert.Null(header1.Split.Previous);
|
||||
Assert.Equal(bytes[..blockSize], payload1);
|
||||
Assert.True(header1.Attributes.Count == 0);
|
||||
|
||||
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
|
||||
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
|
||||
Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2);
|
||||
Assert.True(header2.Attributes.Count == 0);
|
||||
|
||||
// last part
|
||||
Assert.NotNull(object_2.Header.Split.Parent);
|
||||
Assert.NotNull(object_2.Header.Split.ParentHeader);
|
||||
Assert.NotNull(object_2.Header.Split.ParentSignature);
|
||||
Assert.Equal(object_1.Header.Split.SplitId, object_2.Header.Split.SplitId);
|
||||
Assert.Equal(object_1.ObjectId, object_2.Header.Split.Previous);
|
||||
Assert.Equal(fileLength%blockSize, (int)object_2.Header.PayloadLength);
|
||||
Assert.Equal(bytes[((fileLength/blockSize) * blockSize)..fileLength], object_2.Payload);
|
||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
||||
Assert.NotNull(header3.Split.Parent);
|
||||
Assert.NotNull(header3.Split.ParentHeader);
|
||||
Assert.NotNull(header3.Split.ParentSignature);
|
||||
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
|
||||
Assert.Equal(bytes[((fileLength / blockSize) * blockSize)..fileLength], payload3);
|
||||
Assert.True(header3.Attributes.Count == 0);
|
||||
|
||||
// link object
|
||||
Assert.Equal(object_2.Header.Split.Parent, object_3.Header.Split.Parent);
|
||||
Assert.Equal(object_2.Header.Split.ParentHeader, object_3.Header.Split.ParentHeader);
|
||||
Assert.Equal(object_2.Header.Split.SplitId, object_3.Header.Split.SplitId);
|
||||
Assert.Equal(0, (int)object_3.Header.PayloadLength);
|
||||
Assert.Contains(object_0.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
||||
//link object
|
||||
Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
|
||||
Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
|
||||
Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
|
||||
Assert.Equal(0, (int)linkObject.Header.PayloadLength);
|
||||
Assert.True(header3.Attributes.Count == 0);
|
||||
|
||||
Assert.Single(object_3.Header.Split.ParentHeader.Attributes);
|
||||
Assert.Equal("k", object_3.Header.Split.ParentHeader.Attributes[0].Key);
|
||||
Assert.Equal("v", object_3.Header.Split.ParentHeader.Attributes[0].Value);
|
||||
Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
|
||||
Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
|
||||
Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
|
||||
|
||||
var modelObjId = ObjectId.FromHash(object_3.Header.Split.Parent.Value.ToByteArray());
|
||||
var modelObjId = ObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
|
||||
|
||||
Assert.Equal(result.Value, modelObjId.ToString());
|
||||
}
|
||||
|
@ -201,8 +230,8 @@ public class ObjectTest : ObjectTestsBase
|
|||
|
||||
var request = Mocker.DeleteRequests.FirstOrDefault();
|
||||
Assert.NotNull(request);
|
||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
||||
Assert.Equal(ContainerId.ToMessage().Value, request.Body.Address.ContainerId.Value);
|
||||
Assert.Equal(Mocker.ObjectId.ToMessage().Value, request.Body.Address.ObjectId.Value);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
|
@ -214,13 +243,13 @@ public class ObjectTest : ObjectTestsBase
|
|||
|
||||
var request = Mocker.HeadRequests.FirstOrDefault();
|
||||
Assert.NotNull(request);
|
||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
||||
Assert.Equal(ContainerId.ToMessage(), request.Body.Address.ContainerId);
|
||||
Assert.Equal(Mocker.ObjectId.ToMessage(), request.Body.Address.ObjectId);
|
||||
|
||||
Assert.NotNull(response);
|
||||
Assert.Equal(ContainerId.Value, response.ContainerId.Value);
|
||||
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), response.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, response.OwnerId!.Value);
|
||||
Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString());
|
||||
|
||||
Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength);
|
||||
|
|
|
@ -72,13 +72,14 @@ public class SmokeTests
|
|||
[Fact]
|
||||
public async void GetSessionTest()
|
||||
{
|
||||
var ecdsaKey = this.key.LoadWif();
|
||||
|
||||
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
||||
|
||||
var token = await client.CreateSessionAsync(new PrmSessionCreate(100));
|
||||
|
||||
var session = new Session.SessionToken().Deserialize(token.Token);
|
||||
|
||||
var ecdsaKey = this.key.LoadWif();
|
||||
|
||||
var owner = OwnerId.FromKey(ecdsaKey);
|
||||
|
||||
var ownerHash = Base58.Decode(owner.Value);
|
||||
|
@ -217,9 +218,10 @@ public class SmokeTests
|
|||
await foreach (var objId in client.SearchObjectsAsync(searchParam))
|
||||
{
|
||||
resultObjectsCount++;
|
||||
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId));
|
||||
}
|
||||
|
||||
Assert.True(1 == resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
||||
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
|
@ -403,14 +405,14 @@ public class SmokeTests
|
|||
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
||||
[InlineData(200)]
|
||||
public async void ClientCutScenarioTest(int objectSize)
|
||||
{
|
||||
{
|
||||
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
||||
|
||||
await Cleanup(client);
|
||||
|
||||
var createContainerParam = new PrmContainerCreate(new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
|
||||
{
|
||||
WaitParams = lightWait
|
||||
WaitParams = lightWait
|
||||
};
|
||||
|
||||
var containerId = await client.CreateContainerAsync(createContainerParam);
|
||||
|
@ -449,8 +451,8 @@ public class SmokeTests
|
|||
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
|
||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
||||
Assert.Single(objHeader.Attributes);
|
||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
||||
Assert.Equal("fileName", objHeader.Attributes[0].Key);
|
||||
Assert.Equal("test", objHeader.Attributes[0].Value);
|
||||
}
|
||||
|
||||
Assert.True(hasObject);
|
||||
|
@ -542,7 +544,7 @@ public class MetricsInterceptor() : Interceptor
|
|||
|
||||
watch.Stop();
|
||||
|
||||
// Do somethins with call info
|
||||
// Do something with call info
|
||||
// var elapsed = watch.ElapsedTicks * 1_000_000/Stopwatch.Frequency;
|
||||
|
||||
return response;
|
||||
|
|
Loading…
Reference in a new issue