TrueCloudLab/frostfs-sdk-csharp
19
0
Fork 4
Code Issues 5 Pull requests 1 Projects Releases Packages Activity Actions

[#20] Client: Optimize memory usage #21

Merged
PavelGrossSpb merged 2 commits from PavelGrossSpb/frostfs-sdk-csharp:misc/memory_optimization into master 2024-09-04 19:51:24 +00:00
Conversation 0 Commits 2 Files changed 46 +596 -372
46 changed files with 596 additions and 372 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 0ddde467cd - Show all commits

25
src/FrostFS.SDK.ClientV2/Cache.cs Normal file
View file

@ -0,0 +1,25 @@

using Microsoft.Extensions.Caching.Memory;
namespace FrostFS.SDK.ClientV2
{
internal static class Cache
{
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
{
// TODO: get from options?
SizeLimit = 256
});
private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions
{
// TODO: get from options?
SizeLimit = 1024
});
internal static IMemoryCache Owners => _ownersCache;
internal static IMemoryCache Containers => _containersCache;
}
}

8
src/FrostFS.SDK.ClientV2/Client.cs
View file

@ -125,7 +125,7 @@ public class Client : IFrostFSClient
public IAsyncEnumerable<ContainerId> ListContainersAsync(PrmContainerGetAll? args = null)
{
args = args ?? new PrmContainerGetAll();
args ??= new PrmContainerGetAll();
var service = GetContainerService(args);
return service.ListContainersAsync(args);
}
@ -223,7 +223,10 @@ public class Client : IFrostFSClient
#region ToolsImplementation
public ObjectId CalculateObjectId(ObjectHeader header)
{
return new ObjectTools(ClientCtx).CalculateObjectId(header);
if (header == null)
throw new ArgumentNullException(nameof(header));
return ObjectTools.CalculateObjectId(header, ClientCtx);
}
#endregion
@ -236,7 +239,6 @@ public class Client : IFrostFSClient
if (!localNodeInfo.Version.IsSupported(ClientCtx.Version))
{
var msg = $"FrostFS {localNodeInfo.Version} is not supported.";
Console.WriteLine(msg);
throw new ApplicationException(msg);
}
}

13
src/FrostFS.SDK.ClientV2/CllientKey.cs Normal file
View file

@ -0,0 +1,13 @@
using FrostFS.SDK.Cryptography;
using Google.Protobuf;
using System.Security.Cryptography;
namespace FrostFS.SDK.ClientV2
{
public class ClientKey(ECDsa key)
{
internal ECDsa ECDsaKey { get; } = key;
internal ByteString PublicKeyProto { get; } = ByteString.CopyFrom(key.PublicKey());
}
}

13
src/FrostFS.SDK.ClientV2/Exceptions/ResponseException.cs
View file

@ -1,12 +1,9 @@
using System;
using FrostFS.SDK.ModelsV2;
using System;
public class ResponseException : Exception
namespace FrostFS.SDK.ClientV2;
public class ResponseException(ResponseStatus status) : Exception()
{
public Status Status { get; set; }
public ResponseException(Status status) : base()
{
Status = status;
}
public ResponseStatus Status { get; set; } = status;
}

3
src/FrostFS.SDK.ClientV2/FrostFS.SDK.ClientV2.csproj
View file

@ -12,8 +12,11 @@
<ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
<PackageReference Include="System.Runtime.Caching" Version="8.0.0" />
</ItemGroup>
<ItemGroup>

4
src/FrostFS.SDK.ClientV2/Mappers/Container.cs
View file

@ -10,12 +10,12 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ContainerMapper
{
public static Container.Container ToGrpcMessage(this ModelsV2.Container container)
public static Container.Container ToMessage(this ModelsV2.Container container)
{
return new Container.Container
{
BasicAcl = (uint)container.BasicAcl,
PlacementPolicy = container.PlacementPolicy.ToGrpcMessage(),
PlacementPolicy = container.PlacementPolicy.ToMessage(),
Nonce = ByteString.CopyFrom(container.Nonce.ToBytes())
};
}

19
src/FrostFS.SDK.ClientV2/Mappers/ContainerId.cs
View file

@ -2,16 +2,29 @@ using FrostFS.Refs;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using Google.Protobuf;
using Microsoft.Extensions.Caching.Memory;
using System;
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ContainerIdMapper
{
public static ContainerID ToGrpcMessage(this ContainerId containerId)
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromHours(1))
.SetSize(1);
public static ContainerID ToMessage(this ContainerId model)
{
return new ContainerID
if (!Cache.Owners.TryGetValue(model, out ContainerID? message))
dstepanov-yadro marked this conversation as resolved Outdated
dstepanov-yadro commented 2024-08-01 14:33:21 +00:00
Outdated
Review
Copy link

Why containerID from owners cache?

Why containerID from owners cache?
PavelGrossSpb commented 2024-08-01 15:04:20 +00:00
Outdated
Review
Copy link

This is a typo, sure. Fixed.

This is a typo, sure. Fixed.
{
Value = ByteString.CopyFrom(Base58.Decode(containerId.Value))
message = new ContainerID
{
Value = ByteString.CopyFrom(Base58.Decode(model.Value))
};
Cache.Owners.Set(model, message, _oneHourExpiration);
}
return message!;
}
}

10
src/FrostFS.SDK.ClientV2/Mappers/MetaHeader.cs
View file

@ -1,21 +1,15 @@
using FrostFS.SDK.ModelsV2;
using FrostFS.Session;
using Version = FrostFS.Refs.Version;
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class MetaHeaderMapper
{
public static RequestMetaHeader ToGrpcMessage(this MetaHeader metaHeader)
public static RequestMetaHeader ToMessage(this MetaHeader metaHeader)
{
return new RequestMetaHeader
{
Version = new Version
{
Major = (uint)metaHeader.Version.Major,
Minor = (uint)metaHeader.Version.Minor,
},
Version = metaHeader.Version.ToMessage(),
Epoch = (uint)metaHeader.Epoch,
Ttl = (uint)metaHeader.Ttl
};

4
src/FrostFS.SDK.ClientV2/Mappers/Netmap/Netmap.cs
View file

@ -10,6 +10,8 @@ public static class NetmapMapper
{
return new NetmapSnapshot(
netmap.Body.Netmap.Epoch,
netmap.Body.Netmap.Nodes.Select(n => n.ToModel(netmap.MetaHeader.Version)).ToArray());
netmap.Body.Netmap.Nodes
.Select(n => n.ToModel(netmap.MetaHeader.Version))
.ToArray());
}
}

5
src/FrostFS.SDK.ClientV2/Mappers/Netmap/PlacementPolicy.cs
View file

@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
public static class PlacementPolicyMapper
{
public static PlacementPolicy ToGrpcMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
public static PlacementPolicy ToMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
{
var pp = new PlacementPolicy
{
@ -14,9 +14,10 @@ public static class PlacementPolicyMapper
Replicas = { },
Unique = placementPolicy.Unique
};
foreach (var replica in placementPolicy.Replicas)
{
pp.Replicas.Add(replica.ToGrpcMessage());
pp.Replicas.Add(replica.ToMessage());
}
return pp;

2
src/FrostFS.SDK.ClientV2/Mappers/Netmap/Replica.cs
View file

@ -4,7 +4,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
public static class ReplicaMapper
{
public static Replica ToGrpcMessage(this ModelsV2.Netmap.Replica replica)
public static Replica ToMessage(this ModelsV2.Netmap.Replica replica)
{
return new Replica
{

2
src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectAttributeMapper.cs
View file

@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ObjectAttributeMapper
{
public static Header.Types.Attribute ToGrpcMessage(this ObjectAttribute attribute)
public static Header.Types.Attribute ToMessage(this ObjectAttribute attribute)
{
return new Header.Types.Attribute
{

2
src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectFilterMapper.cs
View file

@ -7,7 +7,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ObjectFilterMapper
{
public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this IObjectFilter filter)
public static SearchRequest.Types.Body.Types.Filter ToMessage(this IObjectFilter filter)
{
var objMatchTypeName = filter.MatchType switch
{

15
src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectHeaderMapper.cs
View file

@ -10,7 +10,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ObjectHeaderMapper
{
public static Header ToGrpcMessage(this ObjectHeader header)
public static Header ToMessage(this ObjectHeader header)
{
var objTypeName = header.ObjectType switch
{
@ -22,14 +22,16 @@ public static class ObjectHeaderMapper
var head = new Header
{
ContainerId = header.ContainerId.ToGrpcMessage(),
OwnerId = header!.OwnerId != null ? header.OwnerId.ToMessage() : null,
Version = header!.Version != null ? header.Version.ToMessage() : null,
ContainerId = header.ContainerId.ToMessage(),
ObjectType = objTypeName,
PayloadLength = header.PayloadLength
};
foreach (var attribute in header.Attributes)
{
head.Attributes.Add(attribute.ToGrpcMessage());
head.Attributes.Add(attribute.ToMessage());
}
var split = header.Split;
@ -37,15 +39,8 @@ public static class ObjectHeaderMapper
{
head.Split = new Header.Types.Split
{
Parent = split.Parent?.ToGrpcMessage(),
ParentSignature = split.ParentSignature?.ToGrpcMessage(),
ParentHeader = split.ParentHeader?.ToGrpcMessage(),
Previous = split.Previous?.ToGrpcMessage(),
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
};
if (split.Children != null && split.Children.Count != 0)
head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
}
return head;

2
src/FrostFS.SDK.ClientV2/Mappers/Object/ObjectId.cs
View file

@ -6,7 +6,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class ObjectIdMapper
{
public static ObjectID ToGrpcMessage(this ObjectId objectId)
public static ObjectID ToMessage(this ObjectId objectId)
{
return new ObjectID
{

31
src/FrostFS.SDK.ClientV2/Mappers/OwnerId.cs
View file

@ -1,21 +1,42 @@
using FrostFS.Refs;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using Google.Protobuf;
using Microsoft.Extensions.Caching.Memory;
using System;
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class OwnerIdMapper
{
public static OwnerID ToGrpcMessage(this OwnerId ownerId)
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromHours(1))
.SetSize(1);
public static OwnerID ToMessage(this OwnerId model)
{
return new OwnerID
if (!Cache.Owners.TryGetValue(model, out OwnerID? message))
{
Value = ByteString.CopyFrom(ownerId.ToHash())
message = new OwnerID
{
Value = ByteString.CopyFrom(model.ToHash())
};
Cache.Owners.Set(model, message, _oneHourExpiration);
}
public static OwnerId ToModel(this OwnerID ownerId)
return message!;
}
public static OwnerId ToModel(this OwnerID message)
{
return new OwnerId(ownerId.ToString());
if (!Cache.Owners.TryGetValue(message, out OwnerId? model))
{
model = new OwnerId(Base58.Encode(message.Value.ToByteArray()));
Cache.Owners.Set(message, model, _oneHourExpiration);
}
return model!;
}
}

5
src/FrostFS.SDK.ClientV2/Mappers/SignatureMapper.cs
View file

@ -6,14 +6,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class SignatureMapper
{
public static Refs.Signature ToGrpcMessage(this Signature signature)
public static Refs.Signature ToMessage(this Signature signature)
{
var scheme = signature.Scheme switch
{
SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256,
SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect,
SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512,
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme))
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}",
paramName: nameof(signature.Scheme))
};
return new Refs.Signature

8
src/FrostFS.SDK.ClientV2/Mappers/Status.cs
View file

@ -5,13 +5,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class StatusMapper
{
public static ModelsV2.Status ToModel(this Status.Status status)
public static ModelsV2.ResponseStatus ToModel(this Status.Status status)
{
if (status is null) return new ModelsV2.Status(StatusCode.Success);
if (status is null)
return new ModelsV2.ResponseStatus(StatusCode.Success);
var codeName = Enum.GetName(typeof(StatusCode), status.Code);
return codeName is null
? throw new ArgumentException($"Unknown StatusCode. Value: '{status.Code}'.")
: new ModelsV2.Status((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
: new ModelsV2.ResponseStatus((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
}
}

66
src/FrostFS.SDK.ClientV2/Mappers/Version.cs
View file

@ -1,20 +1,74 @@
using System.Collections;
using System.Threading;
using Version = FrostFS.Refs.Version;
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
public static class VersionMapper
{
public static Version ToGrpcMessage(this ModelsV2.Version version)
private static readonly Hashtable _cacheMessages = [];
private static readonly Hashtable _cacheModels = [];
private static SpinLock _spinlock = new();
public static Version ToMessage(this ModelsV2.Version model)
{
return new Version
var key = model.Major << 16 + model.Minor;
if (!_cacheMessages.ContainsKey(key))
{
Major = (uint)version.Major,
Minor = (uint)version.Minor
bool lockTaken = false;
try
{
_spinlock.Enter(ref lockTaken);
var message = new Version
{
Major = (uint)model.Major,
Minor = (uint)model.Minor
};
_cacheMessages.Add(key, message);
return message;
}
catch (System.ArgumentException)
{
// ignore attempt to add duplicate error.
}
finally
{
if (lockTaken)
_spinlock.Exit(false);
}
}
public static ModelsV2.Version ToModel(this Version version)
return (Version)_cacheMessages[key];
}
public static ModelsV2.Version ToModel(this Version message)
{
return new ModelsV2.Version((int)version.Major, (int)version.Minor);
var key = (int)message.Major << 16 + (int)message.Minor;
if (!_cacheModels.ContainsKey(key))
{
bool lockTaken = false;
try
{
_spinlock.Enter(ref lockTaken);
var model = new ModelsV2.Version((int)message.Major, (int)message.Minor);
_cacheModels.Add(key, model);
return model;
}
catch (System.ArgumentException)
{
// ignore attempt to add duplicate error.
}
finally
{
if (lockTaken)
_spinlock.Exit(false);
}
}
return (ModelsV2.Version)_cacheModels[key];
}
}

6
src/FrostFS.SDK.ClientV2/Parameters/PrmObjectPut.cs
View file

@ -41,4 +41,10 @@ public sealed class PrmObjectPut : IContext, ISessionToken
/// <inheritdoc />
public SessionToken? SessionToken { get; set; }
internal int MaxObjectSizeCache { get; set; }
internal ulong CurrentStreamPosition { get; set; } = 0;
internal ulong FullLength { get; set; } = 0;
}

24
src/FrostFS.SDK.ClientV2/Services/ContainerServiceProvider.cs
View file

@ -17,7 +17,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
{
internal async Task<ModelsV2.Container> GetContainerAsync(PrmContainerGet args)
{
GetRequest request = GetContainerRequest(args.ContainerId.ToGrpcMessage(), args.XHeaders);
GetRequest request = GetContainerRequest(args.ContainerId.ToMessage(), args.XHeaders);
var response = await service.GetAsync(request, null, args.Context!.Deadline, args.Context.CancellationToken);
@ -34,12 +34,12 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
{
Body = new ListRequest.Types.Body
{
OwnerId = Context.Owner.ToGrpcMessage()
OwnerId = Context.Owner.ToMessage()
}
};
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await service.ListAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -54,21 +54,21 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
internal async Task<ContainerId> CreateContainerAsync(PrmContainerCreate args)
{
var ctx = args.Context!;
var grpcContainer = args.Container.ToGrpcMessage();
grpcContainer.OwnerId = Context.Owner.ToGrpcMessage();
grpcContainer.Version = Context.Version.ToGrpcMessage();
var grpcContainer = args.Container.ToMessage();
grpcContainer.OwnerId = Context.Owner.ToMessage();
grpcContainer.Version = Context.Version.ToMessage();
var request = new PutRequest
{
Body = new PutRequest.Types.Body
{
Container = grpcContainer,
Signature = Context.Key.SignRFC6979(grpcContainer)
Signature = Context.Key.ECDsaKey.SignRFC6979(grpcContainer)
}
};
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await service.PutAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -86,14 +86,14 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
{
Body = new DeleteRequest.Types.Body
{
ContainerId = args.ContainerId.ToGrpcMessage(),
Signature = Context.Key.SignRFC6979(args.ContainerId.ToGrpcMessage().Value)
ContainerId = args.ContainerId.ToMessage(),
Signature = Context.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value)
}
};
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await service.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -113,7 +113,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
};
request.AddMetaHeader(xHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
return request;
}

6
src/FrostFS.SDK.ClientV2/Services/NetmapServiceProvider.cs
View file

@ -49,7 +49,7 @@ internal class NetmapServiceProvider : ContextAccessor
};
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -63,7 +63,7 @@ internal class NetmapServiceProvider : ContextAccessor
var request = new NetworkInfoRequest();
request.AddMetaHeader(null);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -79,7 +79,7 @@ internal class NetmapServiceProvider : ContextAccessor
var request = new NetmapSnapshotRequest();
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.Deadline, ctx.CancellationToken);

283
src/FrostFS.SDK.ClientV2/Services/ObjectServiceProvider.cs
View file

@ -13,13 +13,13 @@ using FrostFS.Session;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ClientV2.Extensions;
using FrostFS.SDK.ClientV2.Parameters;
using System.Buffers;
namespace FrostFS.SDK.ClientV2;
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment ctx) : ContextAccessor(ctx), ISessionProvider
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment env) : ContextAccessor(env), ISessionProvider
{
readonly ObjectTools tools = new(ctx);
readonly SessionProvider sessions = new (ctx);
readonly SessionProvider sessions = new (env);
public async ValueTask<Session.SessionToken> GetOrCreateSession(ISessionToken args, Context ctx)
{
@ -35,8 +35,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
{
Address = new Address
{
ContainerId = args.ContainerId.ToGrpcMessage(),
ObjectId = args.ObjectId.ToGrpcMessage()
ContainerId = args.ContainerId.ToMessage(),
ObjectId = args.ObjectId.ToMessage()
}
}
};
@ -46,11 +46,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
sessionToken.CreateObjectTokenContext(
request.Body.Address,
ObjectSessionContext.Types.Verb.Head,
Context.Key);
Context.Key.ECDsaKey);
request.AddMetaHeader(args.XHeaders, sessionToken);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await client!.HeadAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -69,8 +69,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
{
Address = new Address
{
ContainerId = args.ContainerId.ToGrpcMessage(),
ObjectId = args.ObjectId.ToGrpcMessage()
ContainerId = args.ContainerId.ToMessage(),
ObjectId = args.ObjectId.ToMessage()
}
}
};
@ -80,11 +80,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
sessionToken.CreateObjectTokenContext(
request.Body.Address,
ObjectSessionContext.Types.Verb.Get,
Context.Key);
Context.Key.ECDsaKey);
request.AddMetaHeader(args.XHeaders, sessionToken);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
return await GetObject(request, ctx);
}
@ -98,8 +98,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
{
Address = new Address
{
ContainerId = args.ContainerId.ToGrpcMessage(),
ObjectId = args.ObjectId.ToGrpcMessage()
ContainerId = args.ContainerId.ToMessage(),
ObjectId = args.ObjectId.ToMessage()
}
}
};
@ -109,10 +109,10 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
sessionToken.CreateObjectTokenContext(
request.Body.Address,
ObjectSessionContext.Types.Verb.Delete,
Context.Key);
Context.Key.ECDsaKey);
request.AddMetaHeader(args.XHeaders, sessionToken);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await client.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -126,24 +126,24 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
{
Body = new SearchRequest.Types.Body
{
ContainerId = args.ContainerId.ToGrpcMessage(),
ContainerId = args.ContainerId.ToMessage(),
Filters = { },
Version = 1 // TODO: clarify this param
}
};
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToGrpcMessage()));
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToMessage()));
var sessionToken = await GetOrCreateSession(args, ctx);
sessionToken.CreateObjectTokenContext(
new Address { ContainerId = request.Body.ContainerId },
ObjectSessionContext.Types.Verb.Search,
Context.Key);
Context.Key.ECDsaKey);
request.AddMetaHeader(args.XHeaders, sessionToken);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var objectsIds = SearchObjects(request, ctx);
@ -153,7 +153,7 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
}
}
internal Task<ObjectId> PutObjectAsync(PrmObjectPut args)
internal async Task<ObjectId> PutObjectAsync(PrmObjectPut args)
{
if (args.Header == null)
throw new ArgumentException("Value cannot be null", nameof(args.Header));
@ -162,15 +162,22 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
throw new ArgumentException("Value cannot be null", nameof(args.Payload));
if (args.ClientCut)
return PutClientCutObject(args);
return await PutClientCutObject(args);
else
return PutStreamObject(args);
{
if (args.Header.PayloadLength > 0)
args.FullLength = args.Header.PayloadLength;
else if (args.Payload.CanSeek)
args.FullLength = (ulong)args.Payload.Length;
return (await PutStreamObject(args)).ObjectId;
}
}
internal async Task<ObjectId> PutSingleObjectAsync(PrmSingleObjectPut args)
{
var ctx = args.Context!;
var grpcObject = tools.CreateObject(args.FrostFsObject);
var grpcObject = ObjectTools.CreateObject(args.FrostFsObject, env);
var request = new PutSingleRequest
{
@ -185,11 +192,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
sessionToken.CreateObjectTokenContext(
new Address { ContainerId = grpcObject.Header.ContainerId, ObjectId = grpcObject.ObjectId},
ObjectSessionContext.Types.Verb.Put,
Context.Key);
Context.Key.ECDsaKey);
request.AddMetaHeader(args.XHeaders, sessionToken);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
var response = await client.PutSingleAsync(request, null, ctx.Deadline, ctx.CancellationToken);
@ -201,156 +208,136 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
private async Task<ObjectId> PutClientCutObject(PrmObjectPut args)
{
var ctx = args.Context!;
var tokenRaw = await GetOrCreateSession(args, ctx);
var token = new ModelsV2.SessionToken(tokenRaw.Serialize());
args.SessionToken = token;
var payloadStream = args.Payload!;
var header = args.Header!;
ObjectId? objectId;
List<ObjectId> sentObjectIds = [];
FrostFsObject? currentObject;
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
var objectSize = (int)networkSettings.MaxObjectSize;
var fullLength = header.PayloadLength;
if (payloadStream.CanSeek)
{
objectSize = (int)Math.Min(objectSize, payloadStream.Length);
if (fullLength == 0)
if (payloadStream.CanSeek && fullLength == 0)
fullLength = (ulong)payloadStream.Length;
args.FullLength = fullLength;
if (args.MaxObjectSizeCache == 0)
{
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
args.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize;
}
if (fullLength == 0)
throw new ArgumentException("Payload stream must be able to seek or PayloadLength must be specified");
var restBytes = fullLength - args.CurrentStreamPosition;
var objectSize = restBytes > 0 ? (int)Math.Min((ulong)args.MaxObjectSizeCache, restBytes) : args.MaxObjectSizeCache;
var buffer = new byte[objectSize];
//define collection capacity
var restPart = (restBytes % (ulong)objectSize) > 0 ? 1 : 0;
var objectsCount = fullLength > 0 ? (int)(restBytes / (ulong)objectSize) + restPart : 0;
var largeObject = new LargeObject(header.ContainerId);
List<ObjectId> sentObjectIds = new(objectsCount);
var split = new Split();
Split? split = null;
while (true)
// keep attributes for the large object
var attributes = args.Header!.Attributes;
// send all parts except the last one as separate Objects
while (restBytes > (ulong)args.MaxObjectSizeCache)
{
var bytesCount = await payloadStream.ReadAsync(buffer, 0, objectSize);
split.Previous = sentObjectIds.LastOrDefault();
largeObject.Header.PayloadLength += (ulong)bytesCount;
currentObject = new FrostFsObject(header.ContainerId)
.SetPayload(bytesCount < objectSize ? buffer[..bytesCount] : buffer)
.SetSplit(split);
if (largeObject.PayloadLength == fullLength)
break;
objectId = await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token });
sentObjectIds.Add(objectId!);
if (split == null)
{
split = new Split();
args.Header!.Attributes = [];
}
if (sentObjectIds.Count != 0)
split!.Previous = sentObjectIds.LastOrDefault();
args.Header!.Split = split;
var result = await PutStreamObject(args);
sentObjectIds.Add(result.ObjectId);
restBytes -= (ulong)result.ObjectSize;
}
// send the last part and create linkObject
if (sentObjectIds.Count > 0)
{
largeObject.AddAttributes(args.Header!.Attributes);
var largeObjectHeader = new ObjectHeader(header.ContainerId) { PayloadLength = fullLength };
currentObject.SetParent(largeObject);
largeObjectHeader.Attributes.AddRange(attributes);
var putSingleObjectParams = new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token };
args.Header.Split!.ParentHeader = largeObjectHeader;
objectId = await PutSingleObjectAsync(putSingleObjectParams);
var result = await PutStreamObject(args);
sentObjectIds.Add(objectId);
sentObjectIds.Add(result.ObjectId);
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
var linkObject = new LinkObject(header.ContainerId, split!.SplitId, largeObjectHeader)
.AddChildren(sentObjectIds);
linkObject.Header.Attributes.Clear();
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject) { Context = args.Context});
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject, ctx){ SessionToken = token });
return tools.CalculateObjectId(largeObject.Header);
return split.Parent!;
}
currentObject
.SetSplit(null)
.AddAttributes(args.Header!.Attributes);
// We are here if the payload is placed to one Object. It means no cut action, just simple PUT.
var singlePartResult = await PutStreamObject(args);
return await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx));
return singlePartResult.ObjectId;
}
private async Task<ObjectId> PutStreamObject(PrmObjectPut args)
struct PutObjectResult(ObjectId objectId, int objectSize)
{
public ObjectId ObjectId = objectId;
public int ObjectSize = objectSize;
}
private async Task<PutObjectResult> PutStreamObject(PrmObjectPut args)
{
var ctx = args.Context!;
var payload = args.Payload!;
var header = args.Header!;
var hdr = header.ToGrpcMessage();
hdr.OwnerId = Context.Owner.ToGrpcMessage();
hdr.Version = Context.Version.ToGrpcMessage();
var chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
var oid = new ObjectID { Value = hdr.Sha256() };
var restBytes = args.FullLength - args.CurrentStreamPosition;
var initRequest = new PutRequest
chunkSize = (int)Math.Min(restBytes, (ulong)chunkSize);
var chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
var sentBytes = 0;
// 0 means no limit from client, so server side cut is performed
var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0;
var stream = await GetUploadStream(args, ctx);
while (objectLimitSize == 0 || sentBytes < objectLimitSize)
{
Body = new PutRequest.Types.Body
{
Init = new PutRequest.Types.Body.Types.Init
{
Header = hdr
}
}
};
// send chanks limited to default or user's settings
var bufferSize = objectLimitSize > 0 ?
(int)Math.Min(objectLimitSize - sentBytes, chunkSize)
: chunkSize;
var sessionToken = await GetOrCreateSession(args, ctx);
sessionToken.CreateObjectTokenContext(
new Address { ContainerId = hdr.ContainerId, ObjectId = oid },
ObjectSessionContext.Types.Verb.Put,
Context.Key
);
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
initRequest.Sign(Context.Key);
using var stream = await PutObjectInit(initRequest, ctx);
var bufferSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
if (payload.CanSeek)
{
bufferSize = (int)Math.Min(payload.Length, bufferSize);
}
else if (header.PayloadLength > 0)
{
bufferSize = (int)Math.Min((long)header.PayloadLength, bufferSize);
}
var buffer = new byte[bufferSize];
while (true)
{
var bytesCount = await payload.ReadAsync(buffer, 0, bufferSize, ctx.CancellationToken);
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken);
if (bytesCount == 0)
break;
var chunkRequest = new PutRequest(initRequest)
sentBytes += bytesCount;
var chunkRequest = new PutRequest
{
Body = new PutRequest.Types.Body
{
Chunk = ByteString.CopyFrom(buffer.AsSpan()[..bytesCount]),
},
VerifyHeader = null
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount)
}
};
chunkRequest.Sign(Context.Key);
chunkRequest.Sign(Context.Key.ECDsaKey);
await stream.Write(chunkRequest);
}
@ -358,7 +345,49 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
var response = await stream.Close();
Verifier.CheckResponse(response);
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes);
}
private async Task<ObjectStreamer> GetUploadStream(PrmObjectPut args, Context ctx)
{
var header = args.Header!;
header.OwnerId = Context.Owner;
header.Version = Context.Version;
var grpcHeader = header.ToMessage();
if (header.Split != null)
{
ObjectTools.SetSplitValues(grpcHeader, header.Split, env);
}
var oid = new ObjectID { Value = grpcHeader.Sha256() };
var initRequest = new PutRequest
{
Body = new PutRequest.Types.Body
{
Init = new PutRequest.Types.Body.Types.Init
{
Header = grpcHeader
}
}
};
var sessionToken = await GetOrCreateSession(args, ctx);
sessionToken.CreateObjectTokenContext(
new Address { ContainerId = grpcHeader.ContainerId, ObjectId = oid },
ObjectSessionContext.Types.Verb.Put,
Context.Key.ECDsaKey
);
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
initRequest.Sign(Context.Key.ECDsaKey);
return await PutObjectInit(initRequest, ctx);
}
private async Task<FrostFsObject> GetObject(GetRequest request, Context ctx)

4
src/FrostFS.SDK.ClientV2/Services/SessionServiceProvider.cs
View file

@ -21,13 +21,13 @@ internal class SessionServiceProvider : ContextAccessor
{
Body = new CreateRequest.Types.Body
{
OwnerId = Context.Owner.ToGrpcMessage(),
OwnerId = Context.Owner.ToMessage(),
Expiration = args.Expiration
}
};
request.AddMetaHeader(args.XHeaders);
request.Sign(Context.Key);
request.Sign(Context.Key.ECDsaKey);
return await CreateSession(request, args.Context!);
}

7
src/FrostFS.SDK.ClientV2/Tools/ClientEnvironment.cs
View file

@ -1,7 +1,9 @@
using FrostFS.SDK.ModelsV2;
using Google.Protobuf;
using Grpc.Net.Client;
using System;
using System.Security.Cryptography;
using FrostFS.SDK.Cryptography;
namespace FrostFS.SDK.ClientV2;
@ -9,11 +11,12 @@ public class ClientEnvironment(Client client, ECDsa key, OwnerId owner, GrpcChan
{
internal OwnerId Owner { get; } = owner;
internal GrpcChannel Channel { get; private set; } = channel;
internal ECDsa Key { get; } = key;
internal ModelsV2.Version Version { get; } = version;
internal NetworkSettings? NetworkSettings { get; set; }
internal Client Client { get; set; } = client;
internal Client Client { get; } = client;
internal ClientKey Key { get; } = new ClientKey(key);
public void Dispose()
{

86
src/FrostFS.SDK.ClientV2/Tools/ObjectTools.cs
View file

@ -7,54 +7,36 @@ using FrostFS.Refs;
using FrostFS.SDK.ClientV2.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using static FrostFS.Object.Header.Types;
namespace FrostFS.SDK.ClientV2;
internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
internal class ObjectTools
{
internal ObjectId CalculateObjectId(ObjectHeader header)
internal static ObjectId CalculateObjectId(ObjectHeader header, ClientEnvironment env)
{
var grpcHeader = CreateHeader(header, []);
var grpcHeader = CreateHeader(header, [], env);
if (header.Split != null)
SetSplitValues(grpcHeader, header.Split, env);
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
}
internal Object.Object CreateObject(FrostFsObject @object)
internal static Object.Object CreateObject(FrostFsObject @object, ClientEnvironment env)
{
var grpcHeader = @object.Header.ToGrpcMessage();
@object.Header.OwnerId = env.Owner;
@object.Header.Version = env.Version;
var grpcHeader = @object.Header.ToMessage();
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
grpcHeader.Version = Context.Version.ToGrpcMessage();
grpcHeader.PayloadLength = (ulong)@object.Payload.Length;
grpcHeader.PayloadHash = Sha256Checksum(@object.Payload);
var split = @object.Header.Split;
if (split != null)
{
grpcHeader.Split = new Header.Types.Split
{
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
};
if (split.Children != null && split.Children.Count != 0)
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
if (split.ParentHeader is not null)
{
var grpcParentHeader = CreateHeader(split.ParentHeader, []);
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
grpcHeader.Split.ParentHeader = grpcParentHeader;
grpcHeader.Split.ParentSignature = new Refs.Signature
{
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
Sign = ByteString.CopyFrom(Context.Key.SignData(grpcHeader.Split.Parent.ToByteArray())),
};
split.Parent = grpcHeader.Split.Parent.ToModel();
}
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
SetSplitValues(grpcHeader, split, env);
}
var obj = new Object.Object
@ -66,21 +48,49 @@ internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
obj.Signature = new Refs.Signature
{
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
Sign = ByteString.CopyFrom(Context.Key.SignData(obj.ObjectId.ToByteArray())),
Key = env.Key.PublicKeyProto,
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(obj.ObjectId.ToByteArray())),
};
return obj;
}
internal Header CreateHeader(ObjectHeader header, byte[]? payload)
internal static void SetSplitValues(Header grpcHeader, ModelsV2.Split split, ClientEnvironment env)
{
var grpcHeader = header.ToGrpcMessage();
grpcHeader.Split = new Header.Types.Split
{
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
};
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
grpcHeader.Version = Context.Version.ToGrpcMessage();
if (split.Children != null && split.Children.Count != 0)
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
if (payload != null)
if (split.ParentHeader is not null)
{
var grpcParentHeader = CreateHeader(split.ParentHeader, [], env);
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
grpcHeader.Split.ParentHeader = grpcParentHeader;
grpcHeader.Split.ParentSignature = new Refs.Signature
{
Key = env.Key.PublicKeyProto,
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray())),
};
split.Parent = grpcHeader.Split.Parent.ToModel();
}
grpcHeader.Split.Previous = split.Previous?.ToMessage();
}
internal static Header CreateHeader(ObjectHeader header, byte[]? payload, ClientEnvironment env)
{
var grpcHeader = header.ToMessage();
grpcHeader.OwnerId = env.Owner.ToMessage();
grpcHeader.Version = env.Version.ToMessage();
if (payload != null) // && payload.Length > 0
grpcHeader.PayloadHash = Sha256Checksum(payload);
return grpcHeader;

2
src/FrostFS.SDK.ClientV2/Tools/RequestConstructor.cs
View file

@ -17,7 +17,7 @@ public static class RequestConstructor
if (request.MetaHeader is not null)
return;
request.MetaHeader = MetaHeader.Default().ToGrpcMessage();
request.MetaHeader = MetaHeader.Default().ToMessage();
if (sessionToken != null)
request.MetaHeader.SessionToken = sessionToken;

2
src/FrostFS.SDK.ClientV2/Tools/RequestSigner.cs
View file

@ -69,7 +69,7 @@ public static class RequestSigner
var hash = new byte[65];
hash[0] = 0x04;
key.SignHash(SHA512.Create().ComputeHash(data)).CopyTo(hash, 1);
key.SignHash(data.Sha512()).CopyTo(hash, 1);
return hash;
}

5
src/FrostFS.SDK.ClientV2/Tools/Verifier.cs
View file

@ -60,7 +60,7 @@ public static class Verifier
public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
{
return key.VerifyHash(SHA512.Create().ComputeHash(data), sig[1..]);
return key.VerifyHash(data.Sha512(), sig[1..]);
}
public static bool VerifyMessagePart(this Signature sig, IMessage data)
@ -101,7 +101,8 @@ public static class Verifier
throw new FormatException($"invalid response, type={resp.GetType()}");
var status = resp.MetaHeader.Status.ToModel();
if (!status.IsSuccess)
if (status != null && !status.IsSuccess)
throw new ResponseException(status);
}

2
src/FrostFS.SDK.Cryptography/Base58.cs
View file

@ -65,7 +65,7 @@ public static class Base58
var firstNonZeroIndex = 0;
while(bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
while (bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
firstNonZeroIndex++;
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();

45
src/FrostFS.SDK.Cryptography/Extentions.cs
View file

@ -1,11 +1,18 @@
using Google.Protobuf;
using Org.BouncyCastle.Crypto.Digests;
using System.Security.Cryptography;
using System.Threading;
namespace FrostFS.SDK.Cryptography;
public static class Extentions
{
private static readonly SHA256 _sha256 = SHA256.Create();
private static SpinLock _spinlockSha256 = new();
private static readonly SHA512 _sha512 = SHA512.Create();
private static SpinLock _spinlockSha512 = new();
internal static byte[] RIPEMD160(this byte[] value)
{
var hash = new byte[20];
@ -16,14 +23,40 @@ public static class Extentions
return hash;
}
public static byte[] Sha256(this byte[] value)
{
var sha256 = SHA256.Create();
return sha256.ComputeHash(value);
}
public static ByteString Sha256(this IMessage data)
{
return ByteString.CopyFrom(data.ToByteArray().Sha256());
}
public static byte[] Sha256(this byte[] value)
{
bool lockTaken = false;
try
{
_spinlockSha256.Enter(ref lockTaken);
return _sha256.ComputeHash(value);
}
finally
{
if (lockTaken)
_spinlockSha256.Exit(false);
}
}
public static byte[] Sha512(this byte[] value)
{
bool lockTaken = false;
try
{
_spinlockSha512.Enter(ref lockTaken);
return _sha512.ComputeHash(value);
}
finally
{
if (lockTaken)
_spinlockSha512.Exit(false);
}
}
}

5
src/FrostFS.SDK.Cryptography/Key.cs
View file

@ -133,8 +133,9 @@ public static class KeyExtension
public static ECDsa LoadPrivateKey(this byte[] privateKey)
{
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
var publicKey =
secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)).GetEncoded(false)[1..];
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
.GetEncoded(false)[1..];
var key = ECDsa.Create(new ECParameters
{
Curve = ECCurve.NamedCurves.nistP256,

2
src/FrostFS.SDK.Cryptography/Murmur3_128.cs
View file

@ -112,7 +112,7 @@ internal class Murmur3_128 : HashAlgorithm
h2 = seed;
}
private ulong Fimix64(ulong k)
private static ulong Fimix64(ulong k)
{
k ^= k >> 33;
k *= 0xff51afd7ed558ccd;

14
src/FrostFS.SDK.Cryptography/Range.cs
View file

@ -77,14 +77,7 @@ namespace System
{
get
{
if (_value < 0)
{
return ~_value;
}
else
{
return _value;
}
return _value < 0 ? ~_value : _value;
}
}
@ -105,12 +98,9 @@ namespace System
var offset = _value;
if (IsFromEnd)
{
// offset = length - (~value)
// offset = length + (~(~value) + 1)
// offset = length + value + 1
offset += length + 1;
}
return offset;
}

1
src/FrostFS.SDK.ModelsV2/FrostFS.SDK.ModelsV2.csproj
View file

@ -16,6 +16,7 @@
<ItemGroup>
<ProjectReference Include="..\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj" />
<ProjectReference Include="..\FrostFS.SDK.ProtosV2\FrostFS.SDK.ProtosV2.csproj" />
</ItemGroup>
</Project>

9
src/FrostFS.SDK.ModelsV2/Misc/CheckSum.cs
View file

@ -1,3 +1,4 @@
using FrostFS.SDK.Cryptography;
using System;
using System.Security.Cryptography;
@ -8,15 +9,9 @@ public class CheckSum
// type is always Sha256
public byte[]? Hash { get; set; }
public static byte[] GetHash(byte[] content)
{
var sha256 = SHA256.Create();
return sha256.ComputeHash(content);
}
public static CheckSum CreateCheckSum(byte[] content)
{
return new CheckSum { Hash = GetHash(content) };
return new CheckSum { Hash = content.Sha256() };
}
public override string ToString()

10
src/FrostFS.SDK.ModelsV2/Object/FrostFsObject.cs
View file

@ -56,19 +56,17 @@ public class FrostFsObject
/// Applied only for the last Object in chain in case of manual multipart uploading
/// </summary>
/// <param name="largeObject">Parent for multipart object</param>
public void SetParent(LargeObject largeObject)
public void SetParent(ObjectHeader largeObjectHeader)
{
if (Header?.Split == null)
throw new Exception("The object is not initialized properly");
Header.Split.ParentHeader = largeObject.Header;
Header.Split.ParentHeader = largeObjectHeader;
}
}
public class LargeObject(ContainerId container) : FrostFsObject(container)
{
private readonly SHA256 payloadHash = SHA256.Create();
public ulong PayloadLength
{
get { return Header!.PayloadLength; }
@ -77,11 +75,11 @@ public class LargeObject(ContainerId container) : FrostFsObject(container)
public class LinkObject : FrostFsObject
{
public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId)
public LinkObject(ContainerId containerId, SplitId splitId, ObjectHeader largeObjectHeader) : base (containerId)
{
Header!.Split = new Split(splitId)
{
ParentHeader = largeObject.Header
ParentHeader = largeObjectHeader
};
}
}

3
src/FrostFS.SDK.ModelsV2/Object/Splitter.cs
View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
namespace FrostFS.SDK.ModelsV2;
public class Split(SplitId splitId)
@ -19,4 +20,6 @@ public class Split(SplitId splitId)
public ObjectHeader? ParentHeader { get; set; }
public List<ObjectId> Children { get; } = [];
public Refs.Signature ParentSignatureGrpc { get; set; }
}

2
src/FrostFS.SDK.ModelsV2/Response/Status.cs → src/FrostFS.SDK.ModelsV2/Response/ResponseStatus.cs
View file

@ -2,7 +2,7 @@ using FrostFS.SDK.ModelsV2.Enums;
namespace FrostFS.SDK.ModelsV2;
public class Status(StatusCode code, string? message = null)
public class ResponseStatus(StatusCode code, string? message = null)
{
public StatusCode Code { get; set; } = code;
public string Message { get; set; } = message ?? string.Empty;

2
src/FrostFS.SDK.Tests/ContainerTest.cs
View file

@ -110,6 +110,6 @@ public class ContainerTest : ContainerTestsBase
var request = Mocker.Requests.First();
Assert.Equal(cid.ToGrpcMessage(), request.Request.Body.ContainerId);
Assert.Equal(cid.ToMessage(), request.Request.Body.ContainerId);
}
}

8
src/FrostFS.SDK.Tests/Mocks/AsyncStreamReaderMock.cs
View file

@ -19,14 +19,14 @@ public class AsyncStreamReaderMock(string key, ObjectHeader objectHeader) : Serv
{
var header = new Header
{
ContainerId = objectHeader.ContainerId.ToGrpcMessage(),
ContainerId = objectHeader.ContainerId.ToMessage(),
PayloadLength = objectHeader.PayloadLength,
Version = objectHeader.Version!.ToGrpcMessage(),
OwnerId = objectHeader.OwnerId!.ToGrpcMessage()
Version = objectHeader.Version!.ToMessage(),
OwnerId = objectHeader.OwnerId!.ToMessage()
};
foreach (var attr in objectHeader.Attributes)
header.Attributes.Add(attr.ToGrpcMessage());
header.Attributes.Add(attr.ToMessage());
var response = new GetResponse
{

4
src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/ContainerServiceBase.cs
View file

@ -27,7 +27,7 @@ public abstract class ServiceBase(string key)
public static BasicAcl DefaultAcl { get; } = BasicAcl.PublicRW;
public static PlacementPolicy DefaultPlacementPolicy { get; } = new PlacementPolicy(true, new Replica(1));
public Metadata ResponseMetaData => [];
public static Metadata ResponseMetaData => [];
protected ResponseVerificationHeader GetResponseVerificationHeader(IResponse response)
{
@ -58,7 +58,7 @@ public abstract class ServiceBase(string key)
public ResponseMetaHeader ResponseMetaHeader => new()
{
Version = Version.ToGrpcMessage(),
Version = Version.ToMessage(),
Epoch = 100,
Ttl = 1
};

6
src/FrostFS.SDK.Tests/Mocks/ContainerServiceMocks/GetContainerMock.cs
View file

@ -19,7 +19,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
{
var mock = new Mock<ContainerService.ContainerServiceClient>();
var grpcVersion = Version.ToGrpcMessage();
var grpcVersion = Version.ToMessage();
PutResponse putResponse = new()
{
@ -32,7 +32,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
},
MetaHeader = new ResponseMetaHeader
{
Version = Version is null ? DefaultVersion.ToGrpcMessage() : Version.ToGrpcMessage(),
Version = (Version is null ? DefaultVersion : Version).ToMessage(),
Epoch = 100,
Ttl = 1
}
@ -69,7 +69,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
Version = grpcVersion,
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
BasicAcl = (uint)Acl,
PlacementPolicy = PlacementPolicy.ToGrpcMessage()
PlacementPolicy = PlacementPolicy.ToMessage()
}
},
MetaHeader = ResponseMetaHeader

16
src/FrostFS.SDK.Tests/Mocks/ObjectMock.cs
View file

@ -38,12 +38,12 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
HeadResponse ??= new Header
{
CreationEpoch = 99,
ContainerId = ObjectHeader.ContainerId.ToGrpcMessage(),
ContainerId = ObjectHeader.ContainerId.ToMessage(),
ObjectType = ObjectType.Regular,
OwnerId = ObjectHeader.OwnerId!.ToGrpcMessage(),
OwnerId = ObjectHeader.OwnerId!.ToMessage(),
PayloadLength = 1,
PayloadHash = new Refs.Checksum { Type = Refs.ChecksumType.Sha256, Sum = ByteString.CopyFrom(SHA256.HashData([0xff])) },
Version = ObjectHeader.Version!.ToGrpcMessage()
Version = ObjectHeader.Version!.ToMessage()
};
HeadResponse headResponse = new()
@ -89,7 +89,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
}
if (ResultObjectId != null)
if (ResultObjectIds != null)
{
PutResponse putResponse = new()
{
@ -97,7 +97,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
{
ObjectId = new Refs.ObjectID
{
Value = ByteString.CopyFrom(ResultObjectId)
Value = ByteString.CopyFrom(ResultObjectIds.ElementAt(0))
}
},
MetaHeader = ResponseMetaHeader,
@ -156,8 +156,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
{
Tombstone = new Refs.Address
{
ContainerId = ObjectHeader!.ContainerId.ToGrpcMessage(),
ObjectId = ObjectId.ToGrpcMessage()
ContainerId = ObjectHeader!.ContainerId.ToMessage(),
ObjectId = ObjectId.ToMessage()
}
},
MetaHeader = ResponseMetaHeader
@ -195,7 +195,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
public Header? HeadResponse { get; set; }
public byte[]? ResultObjectId { get; set; }
public List<byte[]>? ResultObjectIds { get; set; }
public ClientStreamWriter? ClientStreamWriter { get; private set; } = new ();

119
src/FrostFS.SDK.Tests/ObjectTest.cs
View file

@ -7,6 +7,7 @@ using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap;
using FrostFS.SDK.ProtosV2.Interfaces;
using Google.Protobuf;
using Microsoft.Extensions.Options;
using System.Security.Cryptography;
@ -82,7 +83,7 @@ public class ObjectTest : ObjectTestsBase
Assert.NotNull(result);
Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value);
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), result.Header.OwnerId!.Value);
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, result.Header.OwnerId!.Value);
Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength);
Assert.Single(result.Header.Attributes);
Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key);
@ -92,7 +93,7 @@ public class ObjectTest : ObjectTestsBase
[Fact]
public async void PutObjectTest()
{
Mocker.ResultObjectId = SHA256.HashData([]);
Mocker.ResultObjectIds = new([SHA256.HashData([])]);
Random rnd = new();
var bytes = new byte[1024];
@ -113,7 +114,7 @@ public class ObjectTest : ObjectTestsBase
var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body;
Assert.NotNull(result);
Assert.Equal(Mocker.ResultObjectId, result.ToHash());
Assert.Equal(Mocker.ResultObjectIds.First(), result.ToHash());
Assert.True(Mocker.ClientStreamWriter.CompletedTask);
@ -137,57 +138,85 @@ public class ObjectTest : ObjectTestsBase
{
Header = Mocker.ObjectHeader,
Payload = new MemoryStream(bytes),
BufferMaxSize = 1024,
ClientCut = true
};
Random rnd = new();
List<byte[]> objIds = new([new byte[32], new byte[32], new byte[32]]);
rnd.NextBytes(objIds.ElementAt(0));
rnd.NextBytes(objIds.ElementAt(1));
rnd.NextBytes(objIds.ElementAt(2));
Mocker.ResultObjectIds = objIds;
var result = await GetClient().PutObjectAsync(param);
var sentMessages = Mocker.PutSingleRequests.ToArray();
var singleObjects = Mocker.PutSingleRequests.ToArray();
Assert.Equal(4, sentMessages.Length);
var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
var object_0 = sentMessages[0].Body.Object;
var object_1 = sentMessages[1].Body.Object;
var object_2 = sentMessages[2].Body.Object;
var object_3 = sentMessages[3].Body.Object;
Assert.Single(singleObjects);
Assert.NotNull(object_0.Header.Split.SplitId);
Assert.Null(object_0.Header.Split.Previous);
Assert.Equal(blockSize, (int)object_0.Header.PayloadLength);
Assert.Equal(bytes[..blockSize], object_0.Payload);
Assert.True(object_0.Header.Attributes.Count == 0);
Assert.Equal(11, streamObjects.Length);
Assert.Equal(object_0.Header.Split.SplitId, object_1.Header.Split.SplitId);
Assert.Equal(object_0.ObjectId, object_1.Header.Split.Previous);
Assert.Equal(blockSize, (int)object_1.Header.PayloadLength);
Assert.Equal(bytes[blockSize..(blockSize * 2)], object_1.Payload);
Assert.True(object_1.Header.Attributes.Count == 0);
var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
Assert.Equal(3, bodies.Count(b => b.Init != null));
Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
var linkObject = singleObjects[0].Body.Object;
var header1 = bodies[0].Init.Header;
var header2 = bodies[4].Init.Header;
var header3 = bodies[8].Init.Header;
var payload1 = bodies[1].Chunk
.Concat(bodies[2].Chunk)
.Concat(bodies[3].Chunk)
.ToArray();
var payload2 = bodies[5].Chunk
.Concat(bodies[6].Chunk)
.Concat(bodies[7].Chunk)
.ToArray();
var payload3 = bodies[9].Chunk
.Concat(bodies[10].Chunk)
.ToArray();
Assert.NotNull(header1.Split.SplitId);
Assert.Null(header1.Split.Previous);
Assert.Equal(bytes[..blockSize], payload1);
Assert.True(header1.Attributes.Count == 0);
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2);
Assert.True(header2.Attributes.Count == 0);
// last part
Assert.NotNull(object_2.Header.Split.Parent);
Assert.NotNull(object_2.Header.Split.ParentHeader);
Assert.NotNull(object_2.Header.Split.ParentSignature);
Assert.Equal(object_1.Header.Split.SplitId, object_2.Header.Split.SplitId);
Assert.Equal(object_1.ObjectId, object_2.Header.Split.Previous);
Assert.Equal(fileLength%blockSize, (int)object_2.Header.PayloadLength);
Assert.Equal(bytes[((fileLength/blockSize) * blockSize)..fileLength], object_2.Payload);
Assert.True(object_2.Header.Attributes.Count == 0);
Assert.NotNull(header3.Split.Parent);
Assert.NotNull(header3.Split.ParentHeader);
Assert.NotNull(header3.Split.ParentSignature);
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
Assert.Equal(bytes[((fileLength / blockSize) * blockSize)..fileLength], payload3);
Assert.True(header3.Attributes.Count == 0);
// link object
Assert.Equal(object_2.Header.Split.Parent, object_3.Header.Split.Parent);
Assert.Equal(object_2.Header.Split.ParentHeader, object_3.Header.Split.ParentHeader);
Assert.Equal(object_2.Header.Split.SplitId, object_3.Header.Split.SplitId);
Assert.Equal(0, (int)object_3.Header.PayloadLength);
Assert.Contains(object_0.ObjectId, object_3.Header.Split.Children);
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
Assert.True(object_2.Header.Attributes.Count == 0);
//link object
Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
Assert.Equal(0, (int)linkObject.Header.PayloadLength);
Assert.True(header3.Attributes.Count == 0);
Assert.Single(object_3.Header.Split.ParentHeader.Attributes);
Assert.Equal("k", object_3.Header.Split.ParentHeader.Attributes[0].Key);
Assert.Equal("v", object_3.Header.Split.ParentHeader.Attributes[0].Value);
Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
var modelObjId = ObjectId.FromHash(object_3.Header.Split.Parent.Value.ToByteArray());
var modelObjId = ObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
Assert.Equal(result.Value, modelObjId.ToString());
}
@ -201,8 +230,8 @@ public class ObjectTest : ObjectTestsBase
var request = Mocker.DeleteRequests.FirstOrDefault();
Assert.NotNull(request);
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
Assert.Equal(ContainerId.ToMessage().Value, request.Body.Address.ContainerId.Value);
Assert.Equal(Mocker.ObjectId.ToMessage().Value, request.Body.Address.ObjectId.Value);
}
[Fact]
@ -214,13 +243,13 @@ public class ObjectTest : ObjectTestsBase
var request = Mocker.HeadRequests.FirstOrDefault();
Assert.NotNull(request);
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
Assert.Equal(ContainerId.ToMessage(), request.Body.Address.ContainerId);
Assert.Equal(Mocker.ObjectId.ToMessage(), request.Body.Address.ObjectId);
Assert.NotNull(response);
Assert.Equal(ContainerId.Value, response.ContainerId.Value);
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), response.OwnerId!.Value);
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, response.OwnerId!.Value);
Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString());
Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength);

12
src/FrostFS.SDK.Tests/SmokeTests.cs
View file

@ -72,13 +72,14 @@ public class SmokeTests
[Fact]
public async void GetSessionTest()
{
var ecdsaKey = this.key.LoadWif();
using var client = Client.GetInstance(GetOptions(this.key, this.url));
var token = await client.CreateSessionAsync(new PrmSessionCreate(100));
var session = new Session.SessionToken().Deserialize(token.Token);
var ecdsaKey = this.key.LoadWif();
var owner = OwnerId.FromKey(ecdsaKey);
var ownerHash = Base58.Decode(owner.Value);
@ -217,9 +218,10 @@ public class SmokeTests
await foreach (var objId in client.SearchObjectsAsync(searchParam))
{
resultObjectsCount++;
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId));
}
Assert.True(1 == resultObjectsCount, $"Filter for {filter.Key} doesn't work");
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
}
[Theory]
@ -449,8 +451,8 @@ public class SmokeTests
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
Assert.Single(objHeader.Attributes);
Assert.Equal("fileName", objHeader.Attributes.First().Key);
Assert.Equal("test", objHeader.Attributes.First().Value);
Assert.Equal("fileName", objHeader.Attributes[0].Key);
Assert.Equal("test", objHeader.Attributes[0].Value);
}
Assert.True(hasObject);
@ -542,7 +544,7 @@ public class MetricsInterceptor() : Interceptor
watch.Stop();
// Do somethins with call info
// Do something with call info
// var elapsed = watch.ElapsedTicks * 1_000_000/Stopwatch.Frequency;
return response;