[#20] Client: Optimize memory usage
All checks were successful
DCO / DCO (pull_request) Successful in 41s
All checks were successful
DCO / DCO (pull_request) Successful in 41s
Avoid memory allocation, use cache and static Signed-off-by: Pavel Gross <p.gross@yando.com>
This commit is contained in:
parent
35fe791406
commit
0ddde467cd
46 changed files with 596 additions and 372 deletions
25
src/FrostFS.SDK.ClientV2/Cache.cs
Normal file
25
src/FrostFS.SDK.ClientV2/Cache.cs
Normal file
|
@ -0,0 +1,25 @@
|
||||||
|
|
||||||
|
|
||||||
|
using Microsoft.Extensions.Caching.Memory;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.ClientV2
|
||||||
|
{
|
||||||
|
internal static class Cache
|
||||||
|
{
|
||||||
|
private static readonly IMemoryCache _ownersCache = new MemoryCache(new MemoryCacheOptions
|
||||||
|
{
|
||||||
|
// TODO: get from options?
|
||||||
|
SizeLimit = 256
|
||||||
|
});
|
||||||
|
|
||||||
|
private static readonly IMemoryCache _containersCache = new MemoryCache(new MemoryCacheOptions
|
||||||
|
{
|
||||||
|
// TODO: get from options?
|
||||||
|
SizeLimit = 1024
|
||||||
|
});
|
||||||
|
|
||||||
|
internal static IMemoryCache Owners => _ownersCache;
|
||||||
|
|
||||||
|
internal static IMemoryCache Containers => _containersCache;
|
||||||
|
}
|
||||||
|
}
|
|
@ -125,7 +125,7 @@ public class Client : IFrostFSClient
|
||||||
|
|
||||||
public IAsyncEnumerable<ContainerId> ListContainersAsync(PrmContainerGetAll? args = null)
|
public IAsyncEnumerable<ContainerId> ListContainersAsync(PrmContainerGetAll? args = null)
|
||||||
{
|
{
|
||||||
args = args ?? new PrmContainerGetAll();
|
args ??= new PrmContainerGetAll();
|
||||||
var service = GetContainerService(args);
|
var service = GetContainerService(args);
|
||||||
return service.ListContainersAsync(args);
|
return service.ListContainersAsync(args);
|
||||||
}
|
}
|
||||||
|
@ -223,7 +223,10 @@ public class Client : IFrostFSClient
|
||||||
#region ToolsImplementation
|
#region ToolsImplementation
|
||||||
public ObjectId CalculateObjectId(ObjectHeader header)
|
public ObjectId CalculateObjectId(ObjectHeader header)
|
||||||
{
|
{
|
||||||
return new ObjectTools(ClientCtx).CalculateObjectId(header);
|
if (header == null)
|
||||||
|
throw new ArgumentNullException(nameof(header));
|
||||||
|
|
||||||
|
return ObjectTools.CalculateObjectId(header, ClientCtx);
|
||||||
}
|
}
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
|
@ -236,7 +239,6 @@ public class Client : IFrostFSClient
|
||||||
if (!localNodeInfo.Version.IsSupported(ClientCtx.Version))
|
if (!localNodeInfo.Version.IsSupported(ClientCtx.Version))
|
||||||
{
|
{
|
||||||
var msg = $"FrostFS {localNodeInfo.Version} is not supported.";
|
var msg = $"FrostFS {localNodeInfo.Version} is not supported.";
|
||||||
Console.WriteLine(msg);
|
|
||||||
throw new ApplicationException(msg);
|
throw new ApplicationException(msg);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
13
src/FrostFS.SDK.ClientV2/CllientKey.cs
Normal file
13
src/FrostFS.SDK.ClientV2/CllientKey.cs
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
|
using Google.Protobuf;
|
||||||
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
|
namespace FrostFS.SDK.ClientV2
|
||||||
|
{
|
||||||
|
public class ClientKey(ECDsa key)
|
||||||
|
{
|
||||||
|
internal ECDsa ECDsaKey { get; } = key;
|
||||||
|
|
||||||
|
internal ByteString PublicKeyProto { get; } = ByteString.CopyFrom(key.PublicKey());
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,12 +1,9 @@
|
||||||
using System;
|
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
|
using System;
|
||||||
|
|
||||||
public class ResponseException : Exception
|
namespace FrostFS.SDK.ClientV2;
|
||||||
|
|
||||||
|
public class ResponseException(ResponseStatus status) : Exception()
|
||||||
{
|
{
|
||||||
public Status Status { get; set; }
|
public ResponseStatus Status { get; set; } = status;
|
||||||
|
|
||||||
public ResponseException(Status status) : base()
|
|
||||||
{
|
|
||||||
Status = status;
|
|
||||||
}
|
|
||||||
}
|
}
|
|
@ -12,8 +12,11 @@
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
|
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
|
||||||
|
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
|
||||||
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
|
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
|
||||||
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
|
<PackageReference Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
|
||||||
|
<PackageReference Include="System.Runtime.Caching" Version="8.0.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
|
|
|
@ -10,12 +10,12 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ContainerMapper
|
public static class ContainerMapper
|
||||||
{
|
{
|
||||||
public static Container.Container ToGrpcMessage(this ModelsV2.Container container)
|
public static Container.Container ToMessage(this ModelsV2.Container container)
|
||||||
{
|
{
|
||||||
return new Container.Container
|
return new Container.Container
|
||||||
{
|
{
|
||||||
BasicAcl = (uint)container.BasicAcl,
|
BasicAcl = (uint)container.BasicAcl,
|
||||||
PlacementPolicy = container.PlacementPolicy.ToGrpcMessage(),
|
PlacementPolicy = container.PlacementPolicy.ToMessage(),
|
||||||
Nonce = ByteString.CopyFrom(container.Nonce.ToBytes())
|
Nonce = ByteString.CopyFrom(container.Nonce.ToBytes())
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,16 +2,29 @@ using FrostFS.Refs;
|
||||||
using FrostFS.SDK.Cryptography;
|
using FrostFS.SDK.Cryptography;
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
using Microsoft.Extensions.Caching.Memory;
|
||||||
|
using System;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ContainerIdMapper
|
public static class ContainerIdMapper
|
||||||
{
|
{
|
||||||
public static ContainerID ToGrpcMessage(this ContainerId containerId)
|
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
|
||||||
|
.SetSlidingExpiration(TimeSpan.FromHours(1))
|
||||||
|
.SetSize(1);
|
||||||
|
|
||||||
|
public static ContainerID ToMessage(this ContainerId model)
|
||||||
{
|
{
|
||||||
return new ContainerID
|
if (!Cache.Owners.TryGetValue(model, out ContainerID? message))
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(Base58.Decode(containerId.Value))
|
message = new ContainerID
|
||||||
};
|
{
|
||||||
|
Value = ByteString.CopyFrom(Base58.Decode(model.Value))
|
||||||
|
};
|
||||||
|
|
||||||
|
Cache.Owners.Set(model, message, _oneHourExpiration);
|
||||||
|
}
|
||||||
|
|
||||||
|
return message!;
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,23 +1,17 @@
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
using FrostFS.Session;
|
using FrostFS.Session;
|
||||||
using Version = FrostFS.Refs.Version;
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class MetaHeaderMapper
|
public static class MetaHeaderMapper
|
||||||
{
|
{
|
||||||
public static RequestMetaHeader ToGrpcMessage(this MetaHeader metaHeader)
|
public static RequestMetaHeader ToMessage(this MetaHeader metaHeader)
|
||||||
{
|
{
|
||||||
return new RequestMetaHeader
|
return new RequestMetaHeader
|
||||||
{
|
{
|
||||||
Version = new Version
|
Version = metaHeader.Version.ToMessage(),
|
||||||
{
|
|
||||||
Major = (uint)metaHeader.Version.Major,
|
|
||||||
Minor = (uint)metaHeader.Version.Minor,
|
|
||||||
|
|
||||||
},
|
|
||||||
Epoch = (uint)metaHeader.Epoch,
|
Epoch = (uint)metaHeader.Epoch,
|
||||||
Ttl = (uint)metaHeader.Ttl
|
Ttl = (uint)metaHeader.Ttl
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -10,6 +10,8 @@ public static class NetmapMapper
|
||||||
{
|
{
|
||||||
return new NetmapSnapshot(
|
return new NetmapSnapshot(
|
||||||
netmap.Body.Netmap.Epoch,
|
netmap.Body.Netmap.Epoch,
|
||||||
netmap.Body.Netmap.Nodes.Select(n => n.ToModel(netmap.MetaHeader.Version)).ToArray());
|
netmap.Body.Netmap.Nodes
|
||||||
|
.Select(n => n.ToModel(netmap.MetaHeader.Version))
|
||||||
|
.ToArray());
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
|
||||||
|
|
||||||
public static class PlacementPolicyMapper
|
public static class PlacementPolicyMapper
|
||||||
{
|
{
|
||||||
public static PlacementPolicy ToGrpcMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
|
public static PlacementPolicy ToMessage(this ModelsV2.Netmap.PlacementPolicy placementPolicy)
|
||||||
{
|
{
|
||||||
var pp = new PlacementPolicy
|
var pp = new PlacementPolicy
|
||||||
{
|
{
|
||||||
|
@ -14,9 +14,10 @@ public static class PlacementPolicyMapper
|
||||||
Replicas = { },
|
Replicas = { },
|
||||||
Unique = placementPolicy.Unique
|
Unique = placementPolicy.Unique
|
||||||
};
|
};
|
||||||
|
|
||||||
foreach (var replica in placementPolicy.Replicas)
|
foreach (var replica in placementPolicy.Replicas)
|
||||||
{
|
{
|
||||||
pp.Replicas.Add(replica.ToGrpcMessage());
|
pp.Replicas.Add(replica.ToMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
return pp;
|
return pp;
|
||||||
|
|
|
@ -4,7 +4,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC.Netmap;
|
||||||
|
|
||||||
public static class ReplicaMapper
|
public static class ReplicaMapper
|
||||||
{
|
{
|
||||||
public static Replica ToGrpcMessage(this ModelsV2.Netmap.Replica replica)
|
public static Replica ToMessage(this ModelsV2.Netmap.Replica replica)
|
||||||
{
|
{
|
||||||
return new Replica
|
return new Replica
|
||||||
{
|
{
|
||||||
|
|
|
@ -5,7 +5,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ObjectAttributeMapper
|
public static class ObjectAttributeMapper
|
||||||
{
|
{
|
||||||
public static Header.Types.Attribute ToGrpcMessage(this ObjectAttribute attribute)
|
public static Header.Types.Attribute ToMessage(this ObjectAttribute attribute)
|
||||||
{
|
{
|
||||||
return new Header.Types.Attribute
|
return new Header.Types.Attribute
|
||||||
{
|
{
|
||||||
|
|
|
@ -7,7 +7,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ObjectFilterMapper
|
public static class ObjectFilterMapper
|
||||||
{
|
{
|
||||||
public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this IObjectFilter filter)
|
public static SearchRequest.Types.Body.Types.Filter ToMessage(this IObjectFilter filter)
|
||||||
{
|
{
|
||||||
var objMatchTypeName = filter.MatchType switch
|
var objMatchTypeName = filter.MatchType switch
|
||||||
{
|
{
|
||||||
|
|
|
@ -10,7 +10,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ObjectHeaderMapper
|
public static class ObjectHeaderMapper
|
||||||
{
|
{
|
||||||
public static Header ToGrpcMessage(this ObjectHeader header)
|
public static Header ToMessage(this ObjectHeader header)
|
||||||
{
|
{
|
||||||
var objTypeName = header.ObjectType switch
|
var objTypeName = header.ObjectType switch
|
||||||
{
|
{
|
||||||
|
@ -22,14 +22,16 @@ public static class ObjectHeaderMapper
|
||||||
|
|
||||||
var head = new Header
|
var head = new Header
|
||||||
{
|
{
|
||||||
ContainerId = header.ContainerId.ToGrpcMessage(),
|
OwnerId = header!.OwnerId != null ? header.OwnerId.ToMessage() : null,
|
||||||
|
Version = header!.Version != null ? header.Version.ToMessage() : null,
|
||||||
|
ContainerId = header.ContainerId.ToMessage(),
|
||||||
ObjectType = objTypeName,
|
ObjectType = objTypeName,
|
||||||
PayloadLength = header.PayloadLength
|
PayloadLength = header.PayloadLength
|
||||||
};
|
};
|
||||||
|
|
||||||
foreach (var attribute in header.Attributes)
|
foreach (var attribute in header.Attributes)
|
||||||
{
|
{
|
||||||
head.Attributes.Add(attribute.ToGrpcMessage());
|
head.Attributes.Add(attribute.ToMessage());
|
||||||
}
|
}
|
||||||
|
|
||||||
var split = header.Split;
|
var split = header.Split;
|
||||||
|
@ -37,17 +39,10 @@ public static class ObjectHeaderMapper
|
||||||
{
|
{
|
||||||
head.Split = new Header.Types.Split
|
head.Split = new Header.Types.Split
|
||||||
{
|
{
|
||||||
Parent = split.Parent?.ToGrpcMessage(),
|
|
||||||
ParentSignature = split.ParentSignature?.ToGrpcMessage(),
|
|
||||||
ParentHeader = split.ParentHeader?.ToGrpcMessage(),
|
|
||||||
Previous = split.Previous?.ToGrpcMessage(),
|
|
||||||
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
||||||
};
|
};
|
||||||
|
|
||||||
if (split.Children != null && split.Children.Count != 0)
|
|
||||||
head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return head;
|
return head;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class ObjectIdMapper
|
public static class ObjectIdMapper
|
||||||
{
|
{
|
||||||
public static ObjectID ToGrpcMessage(this ObjectId objectId)
|
public static ObjectID ToMessage(this ObjectId objectId)
|
||||||
{
|
{
|
||||||
return new ObjectID
|
return new ObjectID
|
||||||
{
|
{
|
||||||
|
|
|
@ -1,21 +1,42 @@
|
||||||
using FrostFS.Refs;
|
using FrostFS.Refs;
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
|
using Microsoft.Extensions.Caching.Memory;
|
||||||
|
using System;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class OwnerIdMapper
|
public static class OwnerIdMapper
|
||||||
{
|
{
|
||||||
public static OwnerID ToGrpcMessage(this OwnerId ownerId)
|
private static readonly MemoryCacheEntryOptions _oneHourExpiration = new MemoryCacheEntryOptions()
|
||||||
|
.SetSlidingExpiration(TimeSpan.FromHours(1))
|
||||||
|
.SetSize(1);
|
||||||
|
|
||||||
|
public static OwnerID ToMessage(this OwnerId model)
|
||||||
{
|
{
|
||||||
return new OwnerID
|
if (!Cache.Owners.TryGetValue(model, out OwnerID? message))
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(ownerId.ToHash())
|
message = new OwnerID
|
||||||
};
|
{
|
||||||
|
Value = ByteString.CopyFrom(model.ToHash())
|
||||||
|
};
|
||||||
|
|
||||||
|
Cache.Owners.Set(model, message, _oneHourExpiration);
|
||||||
|
}
|
||||||
|
|
||||||
|
return message!;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static OwnerId ToModel(this OwnerID ownerId)
|
public static OwnerId ToModel(this OwnerID message)
|
||||||
{
|
{
|
||||||
return new OwnerId(ownerId.ToString());
|
if (!Cache.Owners.TryGetValue(message, out OwnerId? model))
|
||||||
|
{
|
||||||
|
model = new OwnerId(Base58.Encode(message.Value.ToByteArray()));
|
||||||
|
|
||||||
|
Cache.Owners.Set(message, model, _oneHourExpiration);
|
||||||
|
}
|
||||||
|
|
||||||
|
return model!;
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -6,14 +6,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class SignatureMapper
|
public static class SignatureMapper
|
||||||
{
|
{
|
||||||
public static Refs.Signature ToGrpcMessage(this Signature signature)
|
public static Refs.Signature ToMessage(this Signature signature)
|
||||||
{
|
{
|
||||||
var scheme = signature.Scheme switch
|
var scheme = signature.Scheme switch
|
||||||
{
|
{
|
||||||
SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256,
|
SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256,
|
||||||
SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect,
|
SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect,
|
||||||
SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512,
|
SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512,
|
||||||
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme))
|
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}",
|
||||||
|
paramName: nameof(signature.Scheme))
|
||||||
};
|
};
|
||||||
|
|
||||||
return new Refs.Signature
|
return new Refs.Signature
|
||||||
|
|
|
@ -5,13 +5,15 @@ namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class StatusMapper
|
public static class StatusMapper
|
||||||
{
|
{
|
||||||
public static ModelsV2.Status ToModel(this Status.Status status)
|
public static ModelsV2.ResponseStatus ToModel(this Status.Status status)
|
||||||
{
|
{
|
||||||
if (status is null) return new ModelsV2.Status(StatusCode.Success);
|
if (status is null)
|
||||||
|
return new ModelsV2.ResponseStatus(StatusCode.Success);
|
||||||
|
|
||||||
var codeName = Enum.GetName(typeof(StatusCode), status.Code);
|
var codeName = Enum.GetName(typeof(StatusCode), status.Code);
|
||||||
|
|
||||||
return codeName is null
|
return codeName is null
|
||||||
? throw new ArgumentException($"Unknown StatusCode. Value: '{status.Code}'.")
|
? throw new ArgumentException($"Unknown StatusCode. Value: '{status.Code}'.")
|
||||||
: new ModelsV2.Status((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
|
: new ModelsV2.ResponseStatus((StatusCode)Enum.Parse(typeof(StatusCode), codeName), status.Message);
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,20 +1,74 @@
|
||||||
|
using System.Collections;
|
||||||
|
using System.Threading;
|
||||||
using Version = FrostFS.Refs.Version;
|
using Version = FrostFS.Refs.Version;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
namespace FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
|
|
||||||
public static class VersionMapper
|
public static class VersionMapper
|
||||||
{
|
{
|
||||||
public static Version ToGrpcMessage(this ModelsV2.Version version)
|
private static readonly Hashtable _cacheMessages = [];
|
||||||
|
private static readonly Hashtable _cacheModels = [];
|
||||||
|
private static SpinLock _spinlock = new();
|
||||||
|
|
||||||
|
public static Version ToMessage(this ModelsV2.Version model)
|
||||||
{
|
{
|
||||||
return new Version
|
var key = model.Major << 16 + model.Minor;
|
||||||
|
|
||||||
|
if (!_cacheMessages.ContainsKey(key))
|
||||||
{
|
{
|
||||||
Major = (uint)version.Major,
|
bool lockTaken = false;
|
||||||
Minor = (uint)version.Minor
|
try
|
||||||
};
|
{
|
||||||
|
_spinlock.Enter(ref lockTaken);
|
||||||
|
var message = new Version
|
||||||
|
{
|
||||||
|
Major = (uint)model.Major,
|
||||||
|
Minor = (uint)model.Minor
|
||||||
|
};
|
||||||
|
|
||||||
|
_cacheMessages.Add(key, message);
|
||||||
|
return message;
|
||||||
|
}
|
||||||
|
catch (System.ArgumentException)
|
||||||
|
{
|
||||||
|
// ignore attempt to add duplicate error.
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken)
|
||||||
|
_spinlock.Exit(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (Version)_cacheMessages[key];
|
||||||
}
|
}
|
||||||
|
|
||||||
public static ModelsV2.Version ToModel(this Version version)
|
public static ModelsV2.Version ToModel(this Version message)
|
||||||
{
|
{
|
||||||
return new ModelsV2.Version((int)version.Major, (int)version.Minor);
|
var key = (int)message.Major << 16 + (int)message.Minor;
|
||||||
|
|
||||||
|
if (!_cacheModels.ContainsKey(key))
|
||||||
|
{
|
||||||
|
bool lockTaken = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_spinlock.Enter(ref lockTaken);
|
||||||
|
var model = new ModelsV2.Version((int)message.Major, (int)message.Minor);
|
||||||
|
|
||||||
|
_cacheModels.Add(key, model);
|
||||||
|
return model;
|
||||||
|
}
|
||||||
|
catch (System.ArgumentException)
|
||||||
|
{
|
||||||
|
// ignore attempt to add duplicate error.
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken)
|
||||||
|
_spinlock.Exit(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (ModelsV2.Version)_cacheModels[key];
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -41,4 +41,10 @@ public sealed class PrmObjectPut : IContext, ISessionToken
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public SessionToken? SessionToken { get; set; }
|
public SessionToken? SessionToken { get; set; }
|
||||||
|
|
||||||
|
internal int MaxObjectSizeCache { get; set; }
|
||||||
|
|
||||||
|
internal ulong CurrentStreamPosition { get; set; } = 0;
|
||||||
|
|
||||||
|
internal ulong FullLength { get; set; } = 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -17,7 +17,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
||||||
{
|
{
|
||||||
internal async Task<ModelsV2.Container> GetContainerAsync(PrmContainerGet args)
|
internal async Task<ModelsV2.Container> GetContainerAsync(PrmContainerGet args)
|
||||||
{
|
{
|
||||||
GetRequest request = GetContainerRequest(args.ContainerId.ToGrpcMessage(), args.XHeaders);
|
GetRequest request = GetContainerRequest(args.ContainerId.ToMessage(), args.XHeaders);
|
||||||
|
|
||||||
var response = await service.GetAsync(request, null, args.Context!.Deadline, args.Context.CancellationToken);
|
var response = await service.GetAsync(request, null, args.Context!.Deadline, args.Context.CancellationToken);
|
||||||
|
|
||||||
|
@ -34,12 +34,12 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
||||||
{
|
{
|
||||||
Body = new ListRequest.Types.Body
|
Body = new ListRequest.Types.Body
|
||||||
{
|
{
|
||||||
OwnerId = Context.Owner.ToGrpcMessage()
|
OwnerId = Context.Owner.ToMessage()
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await service.ListAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await service.ListAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -54,21 +54,21 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
||||||
internal async Task<ContainerId> CreateContainerAsync(PrmContainerCreate args)
|
internal async Task<ContainerId> CreateContainerAsync(PrmContainerCreate args)
|
||||||
{
|
{
|
||||||
var ctx = args.Context!;
|
var ctx = args.Context!;
|
||||||
var grpcContainer = args.Container.ToGrpcMessage();
|
var grpcContainer = args.Container.ToMessage();
|
||||||
grpcContainer.OwnerId = Context.Owner.ToGrpcMessage();
|
grpcContainer.OwnerId = Context.Owner.ToMessage();
|
||||||
grpcContainer.Version = Context.Version.ToGrpcMessage();
|
grpcContainer.Version = Context.Version.ToMessage();
|
||||||
|
|
||||||
var request = new PutRequest
|
var request = new PutRequest
|
||||||
{
|
{
|
||||||
Body = new PutRequest.Types.Body
|
Body = new PutRequest.Types.Body
|
||||||
{
|
{
|
||||||
Container = grpcContainer,
|
Container = grpcContainer,
|
||||||
Signature = Context.Key.SignRFC6979(grpcContainer)
|
Signature = Context.Key.ECDsaKey.SignRFC6979(grpcContainer)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await service.PutAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await service.PutAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -86,14 +86,14 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
||||||
{
|
{
|
||||||
Body = new DeleteRequest.Types.Body
|
Body = new DeleteRequest.Types.Body
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
ContainerId = args.ContainerId.ToMessage(),
|
||||||
Signature = Context.Key.SignRFC6979(args.ContainerId.ToGrpcMessage().Value)
|
Signature = Context.Key.ECDsaKey.SignRFC6979(args.ContainerId.ToMessage().Value)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
|
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await service.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await service.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -113,7 +113,7 @@ internal class ContainerServiceProvider(ContainerService.ContainerServiceClient
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(xHeaders);
|
request.AddMetaHeader(xHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
return request;
|
return request;
|
||||||
}
|
}
|
||||||
|
|
|
@ -49,7 +49,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await netmapServiceClient.LocalNodeInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -63,7 +63,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
||||||
var request = new NetworkInfoRequest();
|
var request = new NetworkInfoRequest();
|
||||||
|
|
||||||
request.AddMetaHeader(null);
|
request.AddMetaHeader(null);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await netmapServiceClient.NetworkInfoAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -79,7 +79,7 @@ internal class NetmapServiceProvider : ContextAccessor
|
||||||
var request = new NetmapSnapshotRequest();
|
var request = new NetmapSnapshotRequest();
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await netmapServiceClient.NetmapSnapshotAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
|
|
@ -13,13 +13,13 @@ using FrostFS.Session;
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
using FrostFS.SDK.ClientV2.Extensions;
|
using FrostFS.SDK.ClientV2.Extensions;
|
||||||
using FrostFS.SDK.ClientV2.Parameters;
|
using FrostFS.SDK.ClientV2.Parameters;
|
||||||
|
using System.Buffers;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2;
|
namespace FrostFS.SDK.ClientV2;
|
||||||
|
|
||||||
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment ctx) : ContextAccessor(ctx), ISessionProvider
|
internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, ClientEnvironment env) : ContextAccessor(env), ISessionProvider
|
||||||
{
|
{
|
||||||
readonly ObjectTools tools = new(ctx);
|
readonly SessionProvider sessions = new (env);
|
||||||
readonly SessionProvider sessions = new (ctx);
|
|
||||||
|
|
||||||
public async ValueTask<Session.SessionToken> GetOrCreateSession(ISessionToken args, Context ctx)
|
public async ValueTask<Session.SessionToken> GetOrCreateSession(ISessionToken args, Context ctx)
|
||||||
{
|
{
|
||||||
|
@ -35,8 +35,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
ContainerId = args.ContainerId.ToMessage(),
|
||||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -46,11 +46,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
sessionToken.CreateObjectTokenContext(
|
sessionToken.CreateObjectTokenContext(
|
||||||
request.Body.Address,
|
request.Body.Address,
|
||||||
ObjectSessionContext.Types.Verb.Head,
|
ObjectSessionContext.Types.Verb.Head,
|
||||||
Context.Key);
|
Context.Key.ECDsaKey);
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
|
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await client!.HeadAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await client!.HeadAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -69,8 +69,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
ContainerId = args.ContainerId.ToMessage(),
|
||||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -80,11 +80,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
sessionToken.CreateObjectTokenContext(
|
sessionToken.CreateObjectTokenContext(
|
||||||
request.Body.Address,
|
request.Body.Address,
|
||||||
ObjectSessionContext.Types.Verb.Get,
|
ObjectSessionContext.Types.Verb.Get,
|
||||||
Context.Key);
|
Context.Key.ECDsaKey);
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
|
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
return await GetObject(request, ctx);
|
return await GetObject(request, ctx);
|
||||||
}
|
}
|
||||||
|
@ -98,8 +98,8 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
{
|
{
|
||||||
Address = new Address
|
Address = new Address
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
ContainerId = args.ContainerId.ToMessage(),
|
||||||
ObjectId = args.ObjectId.ToGrpcMessage()
|
ObjectId = args.ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -109,10 +109,10 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
sessionToken.CreateObjectTokenContext(
|
sessionToken.CreateObjectTokenContext(
|
||||||
request.Body.Address,
|
request.Body.Address,
|
||||||
ObjectSessionContext.Types.Verb.Delete,
|
ObjectSessionContext.Types.Verb.Delete,
|
||||||
Context.Key);
|
Context.Key.ECDsaKey);
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await client.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await client.DeleteAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -126,24 +126,24 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
{
|
{
|
||||||
Body = new SearchRequest.Types.Body
|
Body = new SearchRequest.Types.Body
|
||||||
{
|
{
|
||||||
ContainerId = args.ContainerId.ToGrpcMessage(),
|
ContainerId = args.ContainerId.ToMessage(),
|
||||||
Filters = { },
|
Filters = { },
|
||||||
Version = 1 // TODO: clarify this param
|
Version = 1 // TODO: clarify this param
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToGrpcMessage()));
|
request.Body.Filters.AddRange(args.Filters.Select(f => f.ToMessage()));
|
||||||
|
|
||||||
var sessionToken = await GetOrCreateSession(args, ctx);
|
var sessionToken = await GetOrCreateSession(args, ctx);
|
||||||
|
|
||||||
sessionToken.CreateObjectTokenContext(
|
sessionToken.CreateObjectTokenContext(
|
||||||
new Address { ContainerId = request.Body.ContainerId },
|
new Address { ContainerId = request.Body.ContainerId },
|
||||||
ObjectSessionContext.Types.Verb.Search,
|
ObjectSessionContext.Types.Verb.Search,
|
||||||
Context.Key);
|
Context.Key.ECDsaKey);
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
|
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var objectsIds = SearchObjects(request, ctx);
|
var objectsIds = SearchObjects(request, ctx);
|
||||||
|
|
||||||
|
@ -153,24 +153,31 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal Task<ObjectId> PutObjectAsync(PrmObjectPut args)
|
internal async Task<ObjectId> PutObjectAsync(PrmObjectPut args)
|
||||||
{
|
{
|
||||||
if (args.Header == null)
|
if (args.Header == null)
|
||||||
throw new ArgumentException("Value cannot be null", nameof(args.Header));
|
throw new ArgumentException("Value cannot be null", nameof(args.Header));
|
||||||
|
|
||||||
if (args.Payload == null)
|
if (args.Payload == null)
|
||||||
throw new ArgumentException("Value cannot be null", nameof(args.Payload));
|
throw new ArgumentException("Value cannot be null", nameof(args.Payload));
|
||||||
|
|
||||||
if (args.ClientCut)
|
if (args.ClientCut)
|
||||||
return PutClientCutObject(args);
|
return await PutClientCutObject(args);
|
||||||
else
|
else
|
||||||
return PutStreamObject(args);
|
{
|
||||||
|
if (args.Header.PayloadLength > 0)
|
||||||
|
args.FullLength = args.Header.PayloadLength;
|
||||||
|
else if (args.Payload.CanSeek)
|
||||||
|
args.FullLength = (ulong)args.Payload.Length;
|
||||||
|
|
||||||
|
return (await PutStreamObject(args)).ObjectId;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
internal async Task<ObjectId> PutSingleObjectAsync(PrmSingleObjectPut args)
|
internal async Task<ObjectId> PutSingleObjectAsync(PrmSingleObjectPut args)
|
||||||
{
|
{
|
||||||
var ctx = args.Context!;
|
var ctx = args.Context!;
|
||||||
var grpcObject = tools.CreateObject(args.FrostFsObject);
|
var grpcObject = ObjectTools.CreateObject(args.FrostFsObject, env);
|
||||||
|
|
||||||
var request = new PutSingleRequest
|
var request = new PutSingleRequest
|
||||||
{
|
{
|
||||||
|
@ -185,11 +192,11 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
sessionToken.CreateObjectTokenContext(
|
sessionToken.CreateObjectTokenContext(
|
||||||
new Address { ContainerId = grpcObject.Header.ContainerId, ObjectId = grpcObject.ObjectId},
|
new Address { ContainerId = grpcObject.Header.ContainerId, ObjectId = grpcObject.ObjectId},
|
||||||
ObjectSessionContext.Types.Verb.Put,
|
ObjectSessionContext.Types.Verb.Put,
|
||||||
Context.Key);
|
Context.Key.ECDsaKey);
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders, sessionToken);
|
request.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
|
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
var response = await client.PutSingleAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
var response = await client.PutSingleAsync(request, null, ctx.Deadline, ctx.CancellationToken);
|
||||||
|
|
||||||
|
@ -201,156 +208,136 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
private async Task<ObjectId> PutClientCutObject(PrmObjectPut args)
|
private async Task<ObjectId> PutClientCutObject(PrmObjectPut args)
|
||||||
{
|
{
|
||||||
var ctx = args.Context!;
|
var ctx = args.Context!;
|
||||||
|
|
||||||
var tokenRaw = await GetOrCreateSession(args, ctx);
|
var tokenRaw = await GetOrCreateSession(args, ctx);
|
||||||
var token = new ModelsV2.SessionToken(tokenRaw.Serialize());
|
var token = new ModelsV2.SessionToken(tokenRaw.Serialize());
|
||||||
|
|
||||||
|
args.SessionToken = token;
|
||||||
|
|
||||||
var payloadStream = args.Payload!;
|
var payloadStream = args.Payload!;
|
||||||
var header = args.Header!;
|
var header = args.Header!;
|
||||||
|
|
||||||
ObjectId? objectId;
|
|
||||||
List<ObjectId> sentObjectIds = [];
|
|
||||||
|
|
||||||
FrostFsObject? currentObject;
|
|
||||||
|
|
||||||
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
|
|
||||||
|
|
||||||
var objectSize = (int)networkSettings.MaxObjectSize;
|
|
||||||
|
|
||||||
var fullLength = header.PayloadLength;
|
var fullLength = header.PayloadLength;
|
||||||
|
|
||||||
if (payloadStream.CanSeek)
|
|
||||||
{
|
|
||||||
objectSize = (int)Math.Min(objectSize, payloadStream.Length);
|
|
||||||
|
|
||||||
if (fullLength == 0)
|
|
||||||
fullLength = (ulong)payloadStream.Length;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fullLength == 0)
|
if (payloadStream.CanSeek && fullLength == 0)
|
||||||
throw new ArgumentException("Payload stream must be able to seek or PayloadLength must be specified");
|
fullLength = (ulong)payloadStream.Length;
|
||||||
|
|
||||||
var buffer = new byte[objectSize];
|
|
||||||
|
|
||||||
var largeObject = new LargeObject(header.ContainerId);
|
args.FullLength = fullLength;
|
||||||
|
|
||||||
var split = new Split();
|
if (args.MaxObjectSizeCache == 0)
|
||||||
|
|
||||||
while (true)
|
|
||||||
{
|
{
|
||||||
var bytesCount = await payloadStream.ReadAsync(buffer, 0, objectSize);
|
var networkSettings = await Context.Client.GetNetworkSettingsAsync(new PrmNetworkSettings(ctx));
|
||||||
|
args.MaxObjectSizeCache = (int)networkSettings.MaxObjectSize;
|
||||||
split.Previous = sentObjectIds.LastOrDefault();
|
|
||||||
|
|
||||||
largeObject.Header.PayloadLength += (ulong)bytesCount;
|
|
||||||
|
|
||||||
currentObject = new FrostFsObject(header.ContainerId)
|
|
||||||
.SetPayload(bytesCount < objectSize ? buffer[..bytesCount] : buffer)
|
|
||||||
.SetSplit(split);
|
|
||||||
|
|
||||||
if (largeObject.PayloadLength == fullLength)
|
|
||||||
break;
|
|
||||||
|
|
||||||
objectId = await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token });
|
|
||||||
|
|
||||||
sentObjectIds.Add(objectId!);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (sentObjectIds.Count != 0)
|
var restBytes = fullLength - args.CurrentStreamPosition;
|
||||||
|
var objectSize = restBytes > 0 ? (int)Math.Min((ulong)args.MaxObjectSizeCache, restBytes) : args.MaxObjectSizeCache;
|
||||||
|
|
||||||
|
//define collection capacity
|
||||||
|
var restPart = (restBytes % (ulong)objectSize) > 0 ? 1 : 0;
|
||||||
|
var objectsCount = fullLength > 0 ? (int)(restBytes / (ulong)objectSize) + restPart : 0;
|
||||||
|
|
||||||
|
List<ObjectId> sentObjectIds = new(objectsCount);
|
||||||
|
|
||||||
|
Split? split = null;
|
||||||
|
|
||||||
|
// keep attributes for the large object
|
||||||
|
var attributes = args.Header!.Attributes;
|
||||||
|
|
||||||
|
// send all parts except the last one as separate Objects
|
||||||
|
while (restBytes > (ulong)args.MaxObjectSizeCache)
|
||||||
{
|
{
|
||||||
largeObject.AddAttributes(args.Header!.Attributes);
|
if (split == null)
|
||||||
|
{
|
||||||
currentObject.SetParent(largeObject);
|
split = new Split();
|
||||||
|
args.Header!.Attributes = [];
|
||||||
|
}
|
||||||
|
|
||||||
var putSingleObjectParams = new PrmSingleObjectPut(currentObject, ctx) { SessionToken = token };
|
split!.Previous = sentObjectIds.LastOrDefault();
|
||||||
|
args.Header!.Split = split;
|
||||||
|
|
||||||
objectId = await PutSingleObjectAsync(putSingleObjectParams);
|
var result = await PutStreamObject(args);
|
||||||
|
|
||||||
sentObjectIds.Add(objectId);
|
sentObjectIds.Add(result.ObjectId);
|
||||||
|
|
||||||
var linkObject = new LinkObject(header.ContainerId, split.SplitId, largeObject)
|
restBytes -= (ulong)result.ObjectSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
// send the last part and create linkObject
|
||||||
|
if (sentObjectIds.Count > 0)
|
||||||
|
{
|
||||||
|
var largeObjectHeader = new ObjectHeader(header.ContainerId) { PayloadLength = fullLength };
|
||||||
|
|
||||||
|
largeObjectHeader.Attributes.AddRange(attributes);
|
||||||
|
|
||||||
|
args.Header.Split!.ParentHeader = largeObjectHeader;
|
||||||
|
|
||||||
|
var result = await PutStreamObject(args);
|
||||||
|
|
||||||
|
sentObjectIds.Add(result.ObjectId);
|
||||||
|
|
||||||
|
var linkObject = new LinkObject(header.ContainerId, split!.SplitId, largeObjectHeader)
|
||||||
.AddChildren(sentObjectIds);
|
.AddChildren(sentObjectIds);
|
||||||
|
|
||||||
linkObject.Header.Attributes.Clear();
|
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject) { Context = args.Context});
|
||||||
|
|
||||||
_ = await PutSingleObjectAsync(new PrmSingleObjectPut(linkObject, ctx){ SessionToken = token });
|
return split.Parent!;
|
||||||
|
|
||||||
return tools.CalculateObjectId(largeObject.Header);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
currentObject
|
// We are here if the payload is placed to one Object. It means no cut action, just simple PUT.
|
||||||
.SetSplit(null)
|
var singlePartResult = await PutStreamObject(args);
|
||||||
.AddAttributes(args.Header!.Attributes);
|
|
||||||
|
return singlePartResult.ObjectId;
|
||||||
return await PutSingleObjectAsync(new PrmSingleObjectPut(currentObject, ctx));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<ObjectId> PutStreamObject(PrmObjectPut args)
|
struct PutObjectResult(ObjectId objectId, int objectSize)
|
||||||
|
{
|
||||||
|
public ObjectId ObjectId = objectId;
|
||||||
|
public int ObjectSize = objectSize;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<PutObjectResult> PutStreamObject(PrmObjectPut args)
|
||||||
{
|
{
|
||||||
var ctx = args.Context!;
|
var ctx = args.Context!;
|
||||||
var payload = args.Payload!;
|
var payload = args.Payload!;
|
||||||
var header = args.Header!;
|
|
||||||
|
|
||||||
var hdr = header.ToGrpcMessage();
|
var chunkSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
||||||
hdr.OwnerId = Context.Owner.ToGrpcMessage();
|
|
||||||
hdr.Version = Context.Version.ToGrpcMessage();
|
|
||||||
|
|
||||||
var oid = new ObjectID { Value = hdr.Sha256() };
|
var restBytes = args.FullLength - args.CurrentStreamPosition;
|
||||||
|
|
||||||
var initRequest = new PutRequest
|
chunkSize = (int)Math.Min(restBytes, (ulong)chunkSize);
|
||||||
|
|
||||||
|
var chunkBuffer = ArrayPool<byte>.Shared.Rent(chunkSize);
|
||||||
|
var sentBytes = 0;
|
||||||
|
|
||||||
|
// 0 means no limit from client, so server side cut is performed
|
||||||
|
var objectLimitSize = args.ClientCut ? args.MaxObjectSizeCache : 0;
|
||||||
|
|
||||||
|
var stream = await GetUploadStream(args, ctx);
|
||||||
|
|
||||||
|
while (objectLimitSize == 0 || sentBytes < objectLimitSize)
|
||||||
{
|
{
|
||||||
Body = new PutRequest.Types.Body
|
// send chanks limited to default or user's settings
|
||||||
{
|
var bufferSize = objectLimitSize > 0 ?
|
||||||
Init = new PutRequest.Types.Body.Types.Init
|
(int)Math.Min(objectLimitSize - sentBytes, chunkSize)
|
||||||
{
|
: chunkSize;
|
||||||
Header = hdr
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
var sessionToken = await GetOrCreateSession(args, ctx);
|
var bytesCount = await payload.ReadAsync(chunkBuffer, 0, bufferSize, ctx.CancellationToken);
|
||||||
|
|
||||||
sessionToken.CreateObjectTokenContext(
|
|
||||||
new Address { ContainerId = hdr.ContainerId, ObjectId = oid },
|
|
||||||
ObjectSessionContext.Types.Verb.Put,
|
|
||||||
Context.Key
|
|
||||||
);
|
|
||||||
|
|
||||||
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
|
|
||||||
|
|
||||||
initRequest.Sign(Context.Key);
|
|
||||||
|
|
||||||
using var stream = await PutObjectInit(initRequest, ctx);
|
|
||||||
|
|
||||||
var bufferSize = args.BufferMaxSize > 0 ? args.BufferMaxSize : Constants.ObjectChunkSize;
|
|
||||||
|
|
||||||
if (payload.CanSeek)
|
|
||||||
{
|
|
||||||
bufferSize = (int)Math.Min(payload.Length, bufferSize);
|
|
||||||
}
|
|
||||||
else if (header.PayloadLength > 0)
|
|
||||||
{
|
|
||||||
bufferSize = (int)Math.Min((long)header.PayloadLength, bufferSize);
|
|
||||||
}
|
|
||||||
|
|
||||||
var buffer = new byte[bufferSize];
|
|
||||||
|
|
||||||
while (true)
|
|
||||||
{
|
|
||||||
var bytesCount = await payload.ReadAsync(buffer, 0, bufferSize, ctx.CancellationToken);
|
|
||||||
|
|
||||||
if (bytesCount == 0)
|
if (bytesCount == 0)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
var chunkRequest = new PutRequest(initRequest)
|
sentBytes += bytesCount;
|
||||||
|
|
||||||
|
var chunkRequest = new PutRequest
|
||||||
{
|
{
|
||||||
Body = new PutRequest.Types.Body
|
Body = new PutRequest.Types.Body
|
||||||
{
|
{
|
||||||
Chunk = ByteString.CopyFrom(buffer.AsSpan()[..bytesCount]),
|
Chunk = ByteString.CopyFrom(chunkBuffer, 0, bytesCount)
|
||||||
},
|
}
|
||||||
VerifyHeader = null
|
|
||||||
};
|
};
|
||||||
|
|
||||||
chunkRequest.Sign(Context.Key);
|
chunkRequest.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
await stream.Write(chunkRequest);
|
await stream.Write(chunkRequest);
|
||||||
}
|
}
|
||||||
|
@ -358,7 +345,49 @@ internal class ObjectServiceProvider(ObjectService.ObjectServiceClient client, C
|
||||||
var response = await stream.Close();
|
var response = await stream.Close();
|
||||||
Verifier.CheckResponse(response);
|
Verifier.CheckResponse(response);
|
||||||
|
|
||||||
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
|
return new PutObjectResult(ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()), sentBytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<ObjectStreamer> GetUploadStream(PrmObjectPut args, Context ctx)
|
||||||
|
{
|
||||||
|
var header = args.Header!;
|
||||||
|
|
||||||
|
header.OwnerId = Context.Owner;
|
||||||
|
header.Version = Context.Version;
|
||||||
|
|
||||||
|
var grpcHeader = header.ToMessage();
|
||||||
|
|
||||||
|
if (header.Split != null)
|
||||||
|
{
|
||||||
|
ObjectTools.SetSplitValues(grpcHeader, header.Split, env);
|
||||||
|
}
|
||||||
|
|
||||||
|
var oid = new ObjectID { Value = grpcHeader.Sha256() };
|
||||||
|
|
||||||
|
var initRequest = new PutRequest
|
||||||
|
{
|
||||||
|
Body = new PutRequest.Types.Body
|
||||||
|
{
|
||||||
|
Init = new PutRequest.Types.Body.Types.Init
|
||||||
|
{
|
||||||
|
Header = grpcHeader
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
var sessionToken = await GetOrCreateSession(args, ctx);
|
||||||
|
|
||||||
|
sessionToken.CreateObjectTokenContext(
|
||||||
|
new Address { ContainerId = grpcHeader.ContainerId, ObjectId = oid },
|
||||||
|
ObjectSessionContext.Types.Verb.Put,
|
||||||
|
Context.Key.ECDsaKey
|
||||||
|
);
|
||||||
|
|
||||||
|
initRequest.AddMetaHeader(args.XHeaders, sessionToken);
|
||||||
|
|
||||||
|
initRequest.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
|
return await PutObjectInit(initRequest, ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<FrostFsObject> GetObject(GetRequest request, Context ctx)
|
private async Task<FrostFsObject> GetObject(GetRequest request, Context ctx)
|
||||||
|
|
|
@ -21,13 +21,13 @@ internal class SessionServiceProvider : ContextAccessor
|
||||||
{
|
{
|
||||||
Body = new CreateRequest.Types.Body
|
Body = new CreateRequest.Types.Body
|
||||||
{
|
{
|
||||||
OwnerId = Context.Owner.ToGrpcMessage(),
|
OwnerId = Context.Owner.ToMessage(),
|
||||||
Expiration = args.Expiration
|
Expiration = args.Expiration
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
request.AddMetaHeader(args.XHeaders);
|
request.AddMetaHeader(args.XHeaders);
|
||||||
request.Sign(Context.Key);
|
request.Sign(Context.Key.ECDsaKey);
|
||||||
|
|
||||||
return await CreateSession(request, args.Context!);
|
return await CreateSession(request, args.Context!);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,7 +1,9 @@
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
|
using Google.Protobuf;
|
||||||
using Grpc.Net.Client;
|
using Grpc.Net.Client;
|
||||||
using System;
|
using System;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2;
|
namespace FrostFS.SDK.ClientV2;
|
||||||
|
|
||||||
|
@ -9,11 +11,12 @@ public class ClientEnvironment(Client client, ECDsa key, OwnerId owner, GrpcChan
|
||||||
{
|
{
|
||||||
internal OwnerId Owner { get; } = owner;
|
internal OwnerId Owner { get; } = owner;
|
||||||
internal GrpcChannel Channel { get; private set; } = channel;
|
internal GrpcChannel Channel { get; private set; } = channel;
|
||||||
internal ECDsa Key { get; } = key;
|
|
||||||
internal ModelsV2.Version Version { get; } = version;
|
internal ModelsV2.Version Version { get; } = version;
|
||||||
internal NetworkSettings? NetworkSettings { get; set; }
|
internal NetworkSettings? NetworkSettings { get; set; }
|
||||||
|
|
||||||
internal Client Client { get; set; } = client;
|
internal Client Client { get; } = client;
|
||||||
|
|
||||||
|
internal ClientKey Key { get; } = new ClientKey(key);
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
{
|
{
|
||||||
|
|
|
@ -7,54 +7,36 @@ using FrostFS.Refs;
|
||||||
using FrostFS.SDK.ClientV2.Mappers.GRPC;
|
using FrostFS.SDK.ClientV2.Mappers.GRPC;
|
||||||
using FrostFS.SDK.Cryptography;
|
using FrostFS.SDK.Cryptography;
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
|
using static FrostFS.Object.Header.Types;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ClientV2;
|
namespace FrostFS.SDK.ClientV2;
|
||||||
|
|
||||||
internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
|
internal class ObjectTools
|
||||||
{
|
{
|
||||||
internal ObjectId CalculateObjectId(ObjectHeader header)
|
internal static ObjectId CalculateObjectId(ObjectHeader header, ClientEnvironment env)
|
||||||
{
|
{
|
||||||
var grpcHeader = CreateHeader(header, []);
|
var grpcHeader = CreateHeader(header, [], env);
|
||||||
|
|
||||||
|
if (header.Split != null)
|
||||||
|
SetSplitValues(grpcHeader, header.Split, env);
|
||||||
|
|
||||||
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
|
return new ObjectID { Value = grpcHeader.Sha256() }.ToModel();
|
||||||
}
|
}
|
||||||
|
|
||||||
internal Object.Object CreateObject(FrostFsObject @object)
|
internal static Object.Object CreateObject(FrostFsObject @object, ClientEnvironment env)
|
||||||
{
|
{
|
||||||
var grpcHeader = @object.Header.ToGrpcMessage();
|
@object.Header.OwnerId = env.Owner;
|
||||||
|
@object.Header.Version = env.Version;
|
||||||
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
|
|
||||||
grpcHeader.Version = Context.Version.ToGrpcMessage();
|
var grpcHeader = @object.Header.ToMessage();
|
||||||
|
|
||||||
grpcHeader.PayloadLength = (ulong)@object.Payload.Length;
|
grpcHeader.PayloadLength = (ulong)@object.Payload.Length;
|
||||||
grpcHeader.PayloadHash = Sha256Checksum(@object.Payload);
|
grpcHeader.PayloadHash = Sha256Checksum(@object.Payload);
|
||||||
|
|
||||||
var split = @object.Header.Split;
|
var split = @object.Header.Split;
|
||||||
if (split != null)
|
if (split != null)
|
||||||
{
|
{
|
||||||
grpcHeader.Split = new Header.Types.Split
|
SetSplitValues(grpcHeader, split, env);
|
||||||
{
|
|
||||||
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
|
||||||
};
|
|
||||||
|
|
||||||
if (split.Children != null && split.Children.Count != 0)
|
|
||||||
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
|
|
||||||
|
|
||||||
if (split.ParentHeader is not null)
|
|
||||||
{
|
|
||||||
var grpcParentHeader = CreateHeader(split.ParentHeader, []);
|
|
||||||
|
|
||||||
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
|
||||||
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
|
||||||
grpcHeader.Split.ParentSignature = new Refs.Signature
|
|
||||||
{
|
|
||||||
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
|
|
||||||
Sign = ByteString.CopyFrom(Context.Key.SignData(grpcHeader.Split.Parent.ToByteArray())),
|
|
||||||
};
|
|
||||||
|
|
||||||
split.Parent = grpcHeader.Split.Parent.ToModel();
|
|
||||||
}
|
|
||||||
|
|
||||||
grpcHeader.Split.Previous = split.Previous?.ToGrpcMessage();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var obj = new Object.Object
|
var obj = new Object.Object
|
||||||
|
@ -66,21 +48,49 @@ internal class ObjectTools(ClientEnvironment ctx) : ContextAccessor (ctx)
|
||||||
|
|
||||||
obj.Signature = new Refs.Signature
|
obj.Signature = new Refs.Signature
|
||||||
{
|
{
|
||||||
Key = ByteString.CopyFrom(Context.Key.PublicKey()),
|
Key = env.Key.PublicKeyProto,
|
||||||
Sign = ByteString.CopyFrom(Context.Key.SignData(obj.ObjectId.ToByteArray())),
|
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(obj.ObjectId.ToByteArray())),
|
||||||
};
|
};
|
||||||
|
|
||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal Header CreateHeader(ObjectHeader header, byte[]? payload)
|
internal static void SetSplitValues(Header grpcHeader, ModelsV2.Split split, ClientEnvironment env)
|
||||||
{
|
{
|
||||||
var grpcHeader = header.ToGrpcMessage();
|
grpcHeader.Split = new Header.Types.Split
|
||||||
|
{
|
||||||
grpcHeader.OwnerId = Context.Owner.ToGrpcMessage();
|
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
|
||||||
grpcHeader.Version = Context.Version.ToGrpcMessage();
|
};
|
||||||
|
|
||||||
if (payload != null)
|
if (split.Children != null && split.Children.Count != 0)
|
||||||
|
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToMessage()));
|
||||||
|
|
||||||
|
if (split.ParentHeader is not null)
|
||||||
|
{
|
||||||
|
var grpcParentHeader = CreateHeader(split.ParentHeader, [], env);
|
||||||
|
|
||||||
|
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
|
||||||
|
grpcHeader.Split.ParentHeader = grpcParentHeader;
|
||||||
|
grpcHeader.Split.ParentSignature = new Refs.Signature
|
||||||
|
{
|
||||||
|
Key = env.Key.PublicKeyProto,
|
||||||
|
Sign = ByteString.CopyFrom(env.Key.ECDsaKey.SignData(grpcHeader.Split.Parent.ToByteArray())),
|
||||||
|
};
|
||||||
|
|
||||||
|
split.Parent = grpcHeader.Split.Parent.ToModel();
|
||||||
|
}
|
||||||
|
|
||||||
|
grpcHeader.Split.Previous = split.Previous?.ToMessage();
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static Header CreateHeader(ObjectHeader header, byte[]? payload, ClientEnvironment env)
|
||||||
|
{
|
||||||
|
var grpcHeader = header.ToMessage();
|
||||||
|
|
||||||
|
grpcHeader.OwnerId = env.Owner.ToMessage();
|
||||||
|
grpcHeader.Version = env.Version.ToMessage();
|
||||||
|
|
||||||
|
if (payload != null) // && payload.Length > 0
|
||||||
grpcHeader.PayloadHash = Sha256Checksum(payload);
|
grpcHeader.PayloadHash = Sha256Checksum(payload);
|
||||||
|
|
||||||
return grpcHeader;
|
return grpcHeader;
|
||||||
|
|
|
@ -17,7 +17,7 @@ public static class RequestConstructor
|
||||||
if (request.MetaHeader is not null)
|
if (request.MetaHeader is not null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
request.MetaHeader = MetaHeader.Default().ToGrpcMessage();
|
request.MetaHeader = MetaHeader.Default().ToMessage();
|
||||||
|
|
||||||
if (sessionToken != null)
|
if (sessionToken != null)
|
||||||
request.MetaHeader.SessionToken = sessionToken;
|
request.MetaHeader.SessionToken = sessionToken;
|
||||||
|
|
|
@ -69,7 +69,7 @@ public static class RequestSigner
|
||||||
var hash = new byte[65];
|
var hash = new byte[65];
|
||||||
hash[0] = 0x04;
|
hash[0] = 0x04;
|
||||||
|
|
||||||
key.SignHash(SHA512.Create().ComputeHash(data)).CopyTo(hash, 1);
|
key.SignHash(data.Sha512()).CopyTo(hash, 1);
|
||||||
|
|
||||||
return hash;
|
return hash;
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,7 @@ public static class Verifier
|
||||||
|
|
||||||
public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
|
public static bool VerifyData(this ECDsa key, byte[] data, byte[] sig)
|
||||||
{
|
{
|
||||||
return key.VerifyHash(SHA512.Create().ComputeHash(data), sig[1..]);
|
return key.VerifyHash(data.Sha512(), sig[1..]);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static bool VerifyMessagePart(this Signature sig, IMessage data)
|
public static bool VerifyMessagePart(this Signature sig, IMessage data)
|
||||||
|
@ -101,7 +101,8 @@ public static class Verifier
|
||||||
throw new FormatException($"invalid response, type={resp.GetType()}");
|
throw new FormatException($"invalid response, type={resp.GetType()}");
|
||||||
|
|
||||||
var status = resp.MetaHeader.Status.ToModel();
|
var status = resp.MetaHeader.Status.ToModel();
|
||||||
if (!status.IsSuccess)
|
|
||||||
|
if (status != null && !status.IsSuccess)
|
||||||
throw new ResponseException(status);
|
throw new ResponseException(status);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -51,13 +51,13 @@ public static class Base58
|
||||||
int digit = Alphabet.IndexOf(input[i]);
|
int digit = Alphabet.IndexOf(input[i]);
|
||||||
if (digit < 0)
|
if (digit < 0)
|
||||||
throw new FormatException($"Invalid Base58 character '{input[i]}' at position {i}");
|
throw new FormatException($"Invalid Base58 character '{input[i]}' at position {i}");
|
||||||
|
|
||||||
bi = bi * Alphabet.Length + digit;
|
bi = bi * Alphabet.Length + digit;
|
||||||
}
|
}
|
||||||
|
|
||||||
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
|
int leadingZeroCount = input.TakeWhile(c => c == Alphabet[0]).Count();
|
||||||
var leadingZeros = new byte[leadingZeroCount];
|
var leadingZeros = new byte[leadingZeroCount];
|
||||||
|
|
||||||
if (bi.IsZero)
|
if (bi.IsZero)
|
||||||
return leadingZeros;
|
return leadingZeros;
|
||||||
|
|
||||||
|
@ -65,9 +65,9 @@ public static class Base58
|
||||||
|
|
||||||
var firstNonZeroIndex = 0;
|
var firstNonZeroIndex = 0;
|
||||||
|
|
||||||
while(bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
|
while (bytesBigEndian.ElementAt(firstNonZeroIndex) == 0x0)
|
||||||
firstNonZeroIndex++;
|
firstNonZeroIndex++;
|
||||||
|
|
||||||
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
|
var bytesWithoutLeadingZeros = bytesBigEndian.Skip(firstNonZeroIndex).ToArray();
|
||||||
|
|
||||||
return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros);
|
return ArrayHelper.Concat(leadingZeros, bytesWithoutLeadingZeros);
|
||||||
|
|
|
@ -1,11 +1,18 @@
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using Org.BouncyCastle.Crypto.Digests;
|
using Org.BouncyCastle.Crypto.Digests;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
using System.Threading;
|
||||||
|
|
||||||
namespace FrostFS.SDK.Cryptography;
|
namespace FrostFS.SDK.Cryptography;
|
||||||
|
|
||||||
public static class Extentions
|
public static class Extentions
|
||||||
{
|
{
|
||||||
|
private static readonly SHA256 _sha256 = SHA256.Create();
|
||||||
|
private static SpinLock _spinlockSha256 = new();
|
||||||
|
|
||||||
|
private static readonly SHA512 _sha512 = SHA512.Create();
|
||||||
|
private static SpinLock _spinlockSha512 = new();
|
||||||
|
|
||||||
internal static byte[] RIPEMD160(this byte[] value)
|
internal static byte[] RIPEMD160(this byte[] value)
|
||||||
{
|
{
|
||||||
var hash = new byte[20];
|
var hash = new byte[20];
|
||||||
|
@ -15,15 +22,41 @@ public static class Extentions
|
||||||
digest.DoFinal(hash, 0);
|
digest.DoFinal(hash, 0);
|
||||||
return hash;
|
return hash;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static byte[] Sha256(this byte[] value)
|
|
||||||
{
|
|
||||||
var sha256 = SHA256.Create();
|
|
||||||
return sha256.ComputeHash(value);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ByteString Sha256(this IMessage data)
|
public static ByteString Sha256(this IMessage data)
|
||||||
{
|
{
|
||||||
return ByteString.CopyFrom(data.ToByteArray().Sha256());
|
return ByteString.CopyFrom(data.ToByteArray().Sha256());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha256(this byte[] value)
|
||||||
|
{
|
||||||
|
bool lockTaken = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_spinlockSha256.Enter(ref lockTaken);
|
||||||
|
|
||||||
|
return _sha256.ComputeHash(value);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken)
|
||||||
|
_spinlockSha256.Exit(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] Sha512(this byte[] value)
|
||||||
|
{
|
||||||
|
bool lockTaken = false;
|
||||||
|
try
|
||||||
|
{
|
||||||
|
_spinlockSha512.Enter(ref lockTaken);
|
||||||
|
|
||||||
|
return _sha512.ComputeHash(value);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (lockTaken)
|
||||||
|
_spinlockSha512.Exit(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -133,8 +133,9 @@ public static class KeyExtension
|
||||||
public static ECDsa LoadPrivateKey(this byte[] privateKey)
|
public static ECDsa LoadPrivateKey(this byte[] privateKey)
|
||||||
{
|
{
|
||||||
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
var secp256R1 = SecNamedCurves.GetByName("secp256r1");
|
||||||
var publicKey =
|
var publicKey = secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey))
|
||||||
secp256R1.G.Multiply(new Org.BouncyCastle.Math.BigInteger(1, privateKey)).GetEncoded(false)[1..];
|
.GetEncoded(false)[1..];
|
||||||
|
|
||||||
var key = ECDsa.Create(new ECParameters
|
var key = ECDsa.Create(new ECParameters
|
||||||
{
|
{
|
||||||
Curve = ECCurve.NamedCurves.nistP256,
|
Curve = ECCurve.NamedCurves.nistP256,
|
||||||
|
|
|
@ -112,7 +112,7 @@ internal class Murmur3_128 : HashAlgorithm
|
||||||
h2 = seed;
|
h2 = seed;
|
||||||
}
|
}
|
||||||
|
|
||||||
private ulong Fimix64(ulong k)
|
private static ulong Fimix64(ulong k)
|
||||||
{
|
{
|
||||||
k ^= k >> 33;
|
k ^= k >> 33;
|
||||||
k *= 0xff51afd7ed558ccd;
|
k *= 0xff51afd7ed558ccd;
|
||||||
|
|
|
@ -77,14 +77,7 @@ namespace System
|
||||||
{
|
{
|
||||||
get
|
get
|
||||||
{
|
{
|
||||||
if (_value < 0)
|
return _value < 0 ? ~_value : _value;
|
||||||
{
|
|
||||||
return ~_value;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
return _value;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -105,12 +98,9 @@ namespace System
|
||||||
var offset = _value;
|
var offset = _value;
|
||||||
if (IsFromEnd)
|
if (IsFromEnd)
|
||||||
{
|
{
|
||||||
// offset = length - (~value)
|
|
||||||
// offset = length + (~(~value) + 1)
|
|
||||||
// offset = length + value + 1
|
|
||||||
|
|
||||||
offset += length + 1;
|
offset += length + 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return offset;
|
return offset;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,6 +16,7 @@
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<ProjectReference Include="..\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj" />
|
<ProjectReference Include="..\FrostFS.SDK.Cryptography\FrostFS.SDK.Cryptography.csproj" />
|
||||||
|
<ProjectReference Include="..\FrostFS.SDK.ProtosV2\FrostFS.SDK.ProtosV2.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
using FrostFS.SDK.Cryptography;
|
||||||
using System;
|
using System;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
|
||||||
|
@ -8,15 +9,9 @@ public class CheckSum
|
||||||
// type is always Sha256
|
// type is always Sha256
|
||||||
public byte[]? Hash { get; set; }
|
public byte[]? Hash { get; set; }
|
||||||
|
|
||||||
public static byte[] GetHash(byte[] content)
|
|
||||||
{
|
|
||||||
var sha256 = SHA256.Create();
|
|
||||||
return sha256.ComputeHash(content);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static CheckSum CreateCheckSum(byte[] content)
|
public static CheckSum CreateCheckSum(byte[] content)
|
||||||
{
|
{
|
||||||
return new CheckSum { Hash = GetHash(content) };
|
return new CheckSum { Hash = content.Sha256() };
|
||||||
}
|
}
|
||||||
|
|
||||||
public override string ToString()
|
public override string ToString()
|
||||||
|
|
|
@ -56,19 +56,17 @@ public class FrostFsObject
|
||||||
/// Applied only for the last Object in chain in case of manual multipart uploading
|
/// Applied only for the last Object in chain in case of manual multipart uploading
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="largeObject">Parent for multipart object</param>
|
/// <param name="largeObject">Parent for multipart object</param>
|
||||||
public void SetParent(LargeObject largeObject)
|
public void SetParent(ObjectHeader largeObjectHeader)
|
||||||
{
|
{
|
||||||
if (Header?.Split == null)
|
if (Header?.Split == null)
|
||||||
throw new Exception("The object is not initialized properly");
|
throw new Exception("The object is not initialized properly");
|
||||||
|
|
||||||
Header.Split.ParentHeader = largeObject.Header;
|
Header.Split.ParentHeader = largeObjectHeader;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class LargeObject(ContainerId container) : FrostFsObject(container)
|
public class LargeObject(ContainerId container) : FrostFsObject(container)
|
||||||
{
|
{
|
||||||
private readonly SHA256 payloadHash = SHA256.Create();
|
|
||||||
|
|
||||||
public ulong PayloadLength
|
public ulong PayloadLength
|
||||||
{
|
{
|
||||||
get { return Header!.PayloadLength; }
|
get { return Header!.PayloadLength; }
|
||||||
|
@ -77,11 +75,11 @@ public class LargeObject(ContainerId container) : FrostFsObject(container)
|
||||||
|
|
||||||
public class LinkObject : FrostFsObject
|
public class LinkObject : FrostFsObject
|
||||||
{
|
{
|
||||||
public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId)
|
public LinkObject(ContainerId containerId, SplitId splitId, ObjectHeader largeObjectHeader) : base (containerId)
|
||||||
{
|
{
|
||||||
Header!.Split = new Split(splitId)
|
Header!.Split = new Split(splitId)
|
||||||
{
|
{
|
||||||
ParentHeader = largeObject.Header
|
ParentHeader = largeObjectHeader
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
|
@ -1,5 +1,6 @@
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
|
|
||||||
namespace FrostFS.SDK.ModelsV2;
|
namespace FrostFS.SDK.ModelsV2;
|
||||||
|
|
||||||
public class Split(SplitId splitId)
|
public class Split(SplitId splitId)
|
||||||
|
@ -19,4 +20,6 @@ public class Split(SplitId splitId)
|
||||||
public ObjectHeader? ParentHeader { get; set; }
|
public ObjectHeader? ParentHeader { get; set; }
|
||||||
|
|
||||||
public List<ObjectId> Children { get; } = [];
|
public List<ObjectId> Children { get; } = [];
|
||||||
|
|
||||||
|
public Refs.Signature ParentSignatureGrpc { get; set; }
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@ using FrostFS.SDK.ModelsV2.Enums;
|
||||||
|
|
||||||
namespace FrostFS.SDK.ModelsV2;
|
namespace FrostFS.SDK.ModelsV2;
|
||||||
|
|
||||||
public class Status(StatusCode code, string? message = null)
|
public class ResponseStatus(StatusCode code, string? message = null)
|
||||||
{
|
{
|
||||||
public StatusCode Code { get; set; } = code;
|
public StatusCode Code { get; set; } = code;
|
||||||
public string Message { get; set; } = message ?? string.Empty;
|
public string Message { get; set; } = message ?? string.Empty;
|
|
@ -110,6 +110,6 @@ public class ContainerTest : ContainerTestsBase
|
||||||
|
|
||||||
var request = Mocker.Requests.First();
|
var request = Mocker.Requests.First();
|
||||||
|
|
||||||
Assert.Equal(cid.ToGrpcMessage(), request.Request.Body.ContainerId);
|
Assert.Equal(cid.ToMessage(), request.Request.Body.ContainerId);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,14 +19,14 @@ public class AsyncStreamReaderMock(string key, ObjectHeader objectHeader) : Serv
|
||||||
{
|
{
|
||||||
var header = new Header
|
var header = new Header
|
||||||
{
|
{
|
||||||
ContainerId = objectHeader.ContainerId.ToGrpcMessage(),
|
ContainerId = objectHeader.ContainerId.ToMessage(),
|
||||||
PayloadLength = objectHeader.PayloadLength,
|
PayloadLength = objectHeader.PayloadLength,
|
||||||
Version = objectHeader.Version!.ToGrpcMessage(),
|
Version = objectHeader.Version!.ToMessage(),
|
||||||
OwnerId = objectHeader.OwnerId!.ToGrpcMessage()
|
OwnerId = objectHeader.OwnerId!.ToMessage()
|
||||||
};
|
};
|
||||||
|
|
||||||
foreach (var attr in objectHeader.Attributes)
|
foreach (var attr in objectHeader.Attributes)
|
||||||
header.Attributes.Add(attr.ToGrpcMessage());
|
header.Attributes.Add(attr.ToMessage());
|
||||||
|
|
||||||
var response = new GetResponse
|
var response = new GetResponse
|
||||||
{
|
{
|
||||||
|
|
|
@ -27,7 +27,7 @@ public abstract class ServiceBase(string key)
|
||||||
public static BasicAcl DefaultAcl { get; } = BasicAcl.PublicRW;
|
public static BasicAcl DefaultAcl { get; } = BasicAcl.PublicRW;
|
||||||
public static PlacementPolicy DefaultPlacementPolicy { get; } = new PlacementPolicy(true, new Replica(1));
|
public static PlacementPolicy DefaultPlacementPolicy { get; } = new PlacementPolicy(true, new Replica(1));
|
||||||
|
|
||||||
public Metadata ResponseMetaData => [];
|
public static Metadata ResponseMetaData => [];
|
||||||
|
|
||||||
protected ResponseVerificationHeader GetResponseVerificationHeader(IResponse response)
|
protected ResponseVerificationHeader GetResponseVerificationHeader(IResponse response)
|
||||||
{
|
{
|
||||||
|
@ -58,7 +58,7 @@ public abstract class ServiceBase(string key)
|
||||||
|
|
||||||
public ResponseMetaHeader ResponseMetaHeader => new()
|
public ResponseMetaHeader ResponseMetaHeader => new()
|
||||||
{
|
{
|
||||||
Version = Version.ToGrpcMessage(),
|
Version = Version.ToMessage(),
|
||||||
Epoch = 100,
|
Epoch = 100,
|
||||||
Ttl = 1
|
Ttl = 1
|
||||||
};
|
};
|
||||||
|
|
|
@ -19,7 +19,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
||||||
{
|
{
|
||||||
var mock = new Mock<ContainerService.ContainerServiceClient>();
|
var mock = new Mock<ContainerService.ContainerServiceClient>();
|
||||||
|
|
||||||
var grpcVersion = Version.ToGrpcMessage();
|
var grpcVersion = Version.ToMessage();
|
||||||
|
|
||||||
PutResponse putResponse = new()
|
PutResponse putResponse = new()
|
||||||
{
|
{
|
||||||
|
@ -32,7 +32,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
||||||
},
|
},
|
||||||
MetaHeader = new ResponseMetaHeader
|
MetaHeader = new ResponseMetaHeader
|
||||||
{
|
{
|
||||||
Version = Version is null ? DefaultVersion.ToGrpcMessage() : Version.ToGrpcMessage(),
|
Version = (Version is null ? DefaultVersion : Version).ToMessage(),
|
||||||
Epoch = 100,
|
Epoch = 100,
|
||||||
Ttl = 1
|
Ttl = 1
|
||||||
}
|
}
|
||||||
|
@ -69,7 +69,7 @@ public class ContainerMocker(string key) : ContainerServiceBase(key)
|
||||||
Version = grpcVersion,
|
Version = grpcVersion,
|
||||||
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
|
Nonce = ByteString.CopyFrom(ContainerGuid.ToBytes()),
|
||||||
BasicAcl = (uint)Acl,
|
BasicAcl = (uint)Acl,
|
||||||
PlacementPolicy = PlacementPolicy.ToGrpcMessage()
|
PlacementPolicy = PlacementPolicy.ToMessage()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
MetaHeader = ResponseMetaHeader
|
MetaHeader = ResponseMetaHeader
|
||||||
|
|
|
@ -38,12 +38,12 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
HeadResponse ??= new Header
|
HeadResponse ??= new Header
|
||||||
{
|
{
|
||||||
CreationEpoch = 99,
|
CreationEpoch = 99,
|
||||||
ContainerId = ObjectHeader.ContainerId.ToGrpcMessage(),
|
ContainerId = ObjectHeader.ContainerId.ToMessage(),
|
||||||
ObjectType = ObjectType.Regular,
|
ObjectType = ObjectType.Regular,
|
||||||
OwnerId = ObjectHeader.OwnerId!.ToGrpcMessage(),
|
OwnerId = ObjectHeader.OwnerId!.ToMessage(),
|
||||||
PayloadLength = 1,
|
PayloadLength = 1,
|
||||||
PayloadHash = new Refs.Checksum { Type = Refs.ChecksumType.Sha256, Sum = ByteString.CopyFrom(SHA256.HashData([0xff])) },
|
PayloadHash = new Refs.Checksum { Type = Refs.ChecksumType.Sha256, Sum = ByteString.CopyFrom(SHA256.HashData([0xff])) },
|
||||||
Version = ObjectHeader.Version!.ToGrpcMessage()
|
Version = ObjectHeader.Version!.ToMessage()
|
||||||
};
|
};
|
||||||
|
|
||||||
HeadResponse headResponse = new()
|
HeadResponse headResponse = new()
|
||||||
|
@ -89,7 +89,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (ResultObjectId != null)
|
if (ResultObjectIds != null)
|
||||||
{
|
{
|
||||||
PutResponse putResponse = new()
|
PutResponse putResponse = new()
|
||||||
{
|
{
|
||||||
|
@ -97,7 +97,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
{
|
{
|
||||||
ObjectId = new Refs.ObjectID
|
ObjectId = new Refs.ObjectID
|
||||||
{
|
{
|
||||||
Value = ByteString.CopyFrom(ResultObjectId)
|
Value = ByteString.CopyFrom(ResultObjectIds.ElementAt(0))
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
MetaHeader = ResponseMetaHeader,
|
MetaHeader = ResponseMetaHeader,
|
||||||
|
@ -156,8 +156,8 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
{
|
{
|
||||||
Tombstone = new Refs.Address
|
Tombstone = new Refs.Address
|
||||||
{
|
{
|
||||||
ContainerId = ObjectHeader!.ContainerId.ToGrpcMessage(),
|
ContainerId = ObjectHeader!.ContainerId.ToMessage(),
|
||||||
ObjectId = ObjectId.ToGrpcMessage()
|
ObjectId = ObjectId.ToMessage()
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
MetaHeader = ResponseMetaHeader
|
MetaHeader = ResponseMetaHeader
|
||||||
|
@ -195,7 +195,7 @@ public class ObjectMocker(string key) : ObjectServiceBase(key)
|
||||||
|
|
||||||
public Header? HeadResponse { get; set; }
|
public Header? HeadResponse { get; set; }
|
||||||
|
|
||||||
public byte[]? ResultObjectId { get; set; }
|
public List<byte[]>? ResultObjectIds { get; set; }
|
||||||
|
|
||||||
public ClientStreamWriter? ClientStreamWriter { get; private set; } = new ();
|
public ClientStreamWriter? ClientStreamWriter { get; private set; } = new ();
|
||||||
|
|
||||||
|
|
|
@ -7,6 +7,7 @@ using FrostFS.SDK.Cryptography;
|
||||||
using FrostFS.SDK.ModelsV2;
|
using FrostFS.SDK.ModelsV2;
|
||||||
using FrostFS.SDK.ModelsV2.Enums;
|
using FrostFS.SDK.ModelsV2.Enums;
|
||||||
using FrostFS.SDK.ModelsV2.Netmap;
|
using FrostFS.SDK.ModelsV2.Netmap;
|
||||||
|
using FrostFS.SDK.ProtosV2.Interfaces;
|
||||||
using Google.Protobuf;
|
using Google.Protobuf;
|
||||||
using Microsoft.Extensions.Options;
|
using Microsoft.Extensions.Options;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
@ -82,7 +83,7 @@ public class ObjectTest : ObjectTestsBase
|
||||||
Assert.NotNull(result);
|
Assert.NotNull(result);
|
||||||
|
|
||||||
Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value);
|
Assert.Equal(Mocker.ObjectHeader!.ContainerId.Value, result.Header.ContainerId.Value);
|
||||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), result.Header.OwnerId!.Value);
|
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, result.Header.OwnerId!.Value);
|
||||||
Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength);
|
Assert.Equal(Mocker.ObjectHeader.PayloadLength, result.Header.PayloadLength);
|
||||||
Assert.Single(result.Header.Attributes);
|
Assert.Single(result.Header.Attributes);
|
||||||
Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key);
|
Assert.Equal(Mocker.ObjectHeader.Attributes[0].Key, result.Header.Attributes[0].Key);
|
||||||
|
@ -92,7 +93,7 @@ public class ObjectTest : ObjectTestsBase
|
||||||
[Fact]
|
[Fact]
|
||||||
public async void PutObjectTest()
|
public async void PutObjectTest()
|
||||||
{
|
{
|
||||||
Mocker.ResultObjectId = SHA256.HashData([]);
|
Mocker.ResultObjectIds = new([SHA256.HashData([])]);
|
||||||
|
|
||||||
Random rnd = new();
|
Random rnd = new();
|
||||||
var bytes = new byte[1024];
|
var bytes = new byte[1024];
|
||||||
|
@ -113,7 +114,7 @@ public class ObjectTest : ObjectTestsBase
|
||||||
var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body;
|
var body2 = sentMessages.ElementAt(1).GetBody() as Object.PutRequest.Types.Body;
|
||||||
|
|
||||||
Assert.NotNull(result);
|
Assert.NotNull(result);
|
||||||
Assert.Equal(Mocker.ResultObjectId, result.ToHash());
|
Assert.Equal(Mocker.ResultObjectIds.First(), result.ToHash());
|
||||||
|
|
||||||
Assert.True(Mocker.ClientStreamWriter.CompletedTask);
|
Assert.True(Mocker.ClientStreamWriter.CompletedTask);
|
||||||
|
|
||||||
|
@ -137,57 +138,85 @@ public class ObjectTest : ObjectTestsBase
|
||||||
{
|
{
|
||||||
Header = Mocker.ObjectHeader,
|
Header = Mocker.ObjectHeader,
|
||||||
Payload = new MemoryStream(bytes),
|
Payload = new MemoryStream(bytes),
|
||||||
|
BufferMaxSize = 1024,
|
||||||
ClientCut = true
|
ClientCut = true
|
||||||
};
|
};
|
||||||
|
|
||||||
|
Random rnd = new();
|
||||||
|
|
||||||
|
List<byte[]> objIds = new([new byte[32], new byte[32], new byte[32]]);
|
||||||
|
rnd.NextBytes(objIds.ElementAt(0));
|
||||||
|
rnd.NextBytes(objIds.ElementAt(1));
|
||||||
|
rnd.NextBytes(objIds.ElementAt(2));
|
||||||
|
|
||||||
|
Mocker.ResultObjectIds = objIds;
|
||||||
|
|
||||||
var result = await GetClient().PutObjectAsync(param);
|
var result = await GetClient().PutObjectAsync(param);
|
||||||
|
|
||||||
var sentMessages = Mocker.PutSingleRequests.ToArray();
|
var singleObjects = Mocker.PutSingleRequests.ToArray();
|
||||||
|
|
||||||
Assert.Equal(4, sentMessages.Length);
|
var streamObjects = Mocker.ClientStreamWriter.Messages.ToArray();
|
||||||
|
|
||||||
var object_0 = sentMessages[0].Body.Object;
|
Assert.Single(singleObjects);
|
||||||
var object_1 = sentMessages[1].Body.Object;
|
|
||||||
var object_2 = sentMessages[2].Body.Object;
|
|
||||||
var object_3 = sentMessages[3].Body.Object;
|
|
||||||
|
|
||||||
Assert.NotNull(object_0.Header.Split.SplitId);
|
Assert.Equal(11, streamObjects.Length);
|
||||||
Assert.Null(object_0.Header.Split.Previous);
|
|
||||||
Assert.Equal(blockSize, (int)object_0.Header.PayloadLength);
|
|
||||||
Assert.Equal(bytes[..blockSize], object_0.Payload);
|
|
||||||
Assert.True(object_0.Header.Attributes.Count == 0);
|
|
||||||
|
|
||||||
Assert.Equal(object_0.Header.Split.SplitId, object_1.Header.Split.SplitId);
|
var bodies = streamObjects.Select(o => ((Object.PutRequest)o).Body).ToArray();
|
||||||
Assert.Equal(object_0.ObjectId, object_1.Header.Split.Previous);
|
|
||||||
Assert.Equal(blockSize, (int)object_1.Header.PayloadLength);
|
Assert.Equal(3, bodies.Count(b => b.Init != null));
|
||||||
Assert.Equal(bytes[blockSize..(blockSize * 2)], object_1.Payload);
|
Assert.Equal(5, bodies.Count(b => b.Chunk.Length == 1024));
|
||||||
Assert.True(object_1.Header.Attributes.Count == 0);
|
|
||||||
|
Assert.Equal(596, ((Object.PutRequest)streamObjects[10]).Body.Chunk.Length);
|
||||||
|
|
||||||
|
var linkObject = singleObjects[0].Body.Object;
|
||||||
|
var header1 = bodies[0].Init.Header;
|
||||||
|
var header2 = bodies[4].Init.Header;
|
||||||
|
var header3 = bodies[8].Init.Header;
|
||||||
|
|
||||||
|
var payload1 = bodies[1].Chunk
|
||||||
|
.Concat(bodies[2].Chunk)
|
||||||
|
.Concat(bodies[3].Chunk)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
var payload2 = bodies[5].Chunk
|
||||||
|
.Concat(bodies[6].Chunk)
|
||||||
|
.Concat(bodies[7].Chunk)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
var payload3 = bodies[9].Chunk
|
||||||
|
.Concat(bodies[10].Chunk)
|
||||||
|
.ToArray();
|
||||||
|
|
||||||
|
Assert.NotNull(header1.Split.SplitId);
|
||||||
|
Assert.Null(header1.Split.Previous);
|
||||||
|
Assert.Equal(bytes[..blockSize], payload1);
|
||||||
|
Assert.True(header1.Attributes.Count == 0);
|
||||||
|
|
||||||
|
Assert.Equal(header1.Split.SplitId, header2.Split.SplitId);
|
||||||
|
Assert.Equal(objIds.ElementAt(0), header2.Split.Previous.Value);
|
||||||
|
Assert.Equal(bytes[blockSize..(blockSize * 2)], payload2);
|
||||||
|
Assert.True(header2.Attributes.Count == 0);
|
||||||
|
|
||||||
// last part
|
// last part
|
||||||
Assert.NotNull(object_2.Header.Split.Parent);
|
Assert.NotNull(header3.Split.Parent);
|
||||||
Assert.NotNull(object_2.Header.Split.ParentHeader);
|
Assert.NotNull(header3.Split.ParentHeader);
|
||||||
Assert.NotNull(object_2.Header.Split.ParentSignature);
|
Assert.NotNull(header3.Split.ParentSignature);
|
||||||
Assert.Equal(object_1.Header.Split.SplitId, object_2.Header.Split.SplitId);
|
Assert.Equal(header2.Split.SplitId, header3.Split.SplitId);
|
||||||
Assert.Equal(object_1.ObjectId, object_2.Header.Split.Previous);
|
Assert.Equal(bytes[((fileLength / blockSize) * blockSize)..fileLength], payload3);
|
||||||
Assert.Equal(fileLength%blockSize, (int)object_2.Header.PayloadLength);
|
Assert.True(header3.Attributes.Count == 0);
|
||||||
Assert.Equal(bytes[((fileLength/blockSize) * blockSize)..fileLength], object_2.Payload);
|
|
||||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
|
||||||
|
|
||||||
// link object
|
//link object
|
||||||
Assert.Equal(object_2.Header.Split.Parent, object_3.Header.Split.Parent);
|
Assert.Equal(header3.Split.Parent, linkObject.Header.Split.Parent);
|
||||||
Assert.Equal(object_2.Header.Split.ParentHeader, object_3.Header.Split.ParentHeader);
|
Assert.Equal(header3.Split.ParentHeader, linkObject.Header.Split.ParentHeader);
|
||||||
Assert.Equal(object_2.Header.Split.SplitId, object_3.Header.Split.SplitId);
|
Assert.Equal(header3.Split.SplitId, linkObject.Header.Split.SplitId);
|
||||||
Assert.Equal(0, (int)object_3.Header.PayloadLength);
|
Assert.Equal(0, (int)linkObject.Header.PayloadLength);
|
||||||
Assert.Contains(object_0.ObjectId, object_3.Header.Split.Children);
|
Assert.True(header3.Attributes.Count == 0);
|
||||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
|
||||||
Assert.Contains(object_2.ObjectId, object_3.Header.Split.Children);
|
|
||||||
Assert.True(object_2.Header.Attributes.Count == 0);
|
|
||||||
|
|
||||||
Assert.Single(object_3.Header.Split.ParentHeader.Attributes);
|
Assert.Single(linkObject.Header.Split.ParentHeader.Attributes);
|
||||||
Assert.Equal("k", object_3.Header.Split.ParentHeader.Attributes[0].Key);
|
Assert.Equal("k", linkObject.Header.Split.ParentHeader.Attributes[0].Key);
|
||||||
Assert.Equal("v", object_3.Header.Split.ParentHeader.Attributes[0].Value);
|
Assert.Equal("v", linkObject.Header.Split.ParentHeader.Attributes[0].Value);
|
||||||
|
|
||||||
var modelObjId = ObjectId.FromHash(object_3.Header.Split.Parent.Value.ToByteArray());
|
var modelObjId = ObjectId.FromHash(linkObject.Header.Split.Parent.Value.ToByteArray());
|
||||||
|
|
||||||
Assert.Equal(result.Value, modelObjId.ToString());
|
Assert.Equal(result.Value, modelObjId.ToString());
|
||||||
}
|
}
|
||||||
|
@ -201,8 +230,8 @@ public class ObjectTest : ObjectTestsBase
|
||||||
|
|
||||||
var request = Mocker.DeleteRequests.FirstOrDefault();
|
var request = Mocker.DeleteRequests.FirstOrDefault();
|
||||||
Assert.NotNull(request);
|
Assert.NotNull(request);
|
||||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
Assert.Equal(ContainerId.ToMessage().Value, request.Body.Address.ContainerId.Value);
|
||||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
Assert.Equal(Mocker.ObjectId.ToMessage().Value, request.Body.Address.ObjectId.Value);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
@ -214,13 +243,13 @@ public class ObjectTest : ObjectTestsBase
|
||||||
|
|
||||||
var request = Mocker.HeadRequests.FirstOrDefault();
|
var request = Mocker.HeadRequests.FirstOrDefault();
|
||||||
Assert.NotNull(request);
|
Assert.NotNull(request);
|
||||||
Assert.Equal(ContainerId.ToGrpcMessage(), request.Body.Address.ContainerId);
|
Assert.Equal(ContainerId.ToMessage(), request.Body.Address.ContainerId);
|
||||||
Assert.Equal(Mocker.ObjectId.ToGrpcMessage(), request.Body.Address.ObjectId);
|
Assert.Equal(Mocker.ObjectId.ToMessage(), request.Body.Address.ObjectId);
|
||||||
|
|
||||||
Assert.NotNull(response);
|
Assert.NotNull(response);
|
||||||
Assert.Equal(ContainerId.Value, response.ContainerId.Value);
|
Assert.Equal(ContainerId.Value, response.ContainerId.Value);
|
||||||
|
|
||||||
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.ToGrpcMessage().ToString(), response.OwnerId!.Value);
|
Assert.Equal(Mocker.ObjectHeader!.OwnerId!.Value, response.OwnerId!.Value);
|
||||||
Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString());
|
Assert.Equal(Mocker.ObjectHeader!.Version!.ToString(), response.Version!.ToString());
|
||||||
|
|
||||||
Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength);
|
Assert.Equal(Mocker.HeadResponse!.PayloadLength, response.PayloadLength);
|
||||||
|
|
|
@ -72,13 +72,14 @@ public class SmokeTests
|
||||||
[Fact]
|
[Fact]
|
||||||
public async void GetSessionTest()
|
public async void GetSessionTest()
|
||||||
{
|
{
|
||||||
|
var ecdsaKey = this.key.LoadWif();
|
||||||
|
|
||||||
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
||||||
|
|
||||||
var token = await client.CreateSessionAsync(new PrmSessionCreate(100));
|
var token = await client.CreateSessionAsync(new PrmSessionCreate(100));
|
||||||
|
|
||||||
var session = new Session.SessionToken().Deserialize(token.Token);
|
var session = new Session.SessionToken().Deserialize(token.Token);
|
||||||
|
|
||||||
var ecdsaKey = this.key.LoadWif();
|
|
||||||
var owner = OwnerId.FromKey(ecdsaKey);
|
var owner = OwnerId.FromKey(ecdsaKey);
|
||||||
|
|
||||||
var ownerHash = Base58.Decode(owner.Value);
|
var ownerHash = Base58.Decode(owner.Value);
|
||||||
|
@ -217,9 +218,10 @@ public class SmokeTests
|
||||||
await foreach (var objId in client.SearchObjectsAsync(searchParam))
|
await foreach (var objId in client.SearchObjectsAsync(searchParam))
|
||||||
{
|
{
|
||||||
resultObjectsCount++;
|
resultObjectsCount++;
|
||||||
|
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objId));
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.True(1 == resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
Assert.True(0 < resultObjectsCount, $"Filter for {filter.Key} doesn't work");
|
||||||
}
|
}
|
||||||
|
|
||||||
[Theory]
|
[Theory]
|
||||||
|
@ -403,14 +405,14 @@ public class SmokeTests
|
||||||
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
[InlineData(2 * 64 * 1024 * 1024 + 256)]
|
||||||
[InlineData(200)]
|
[InlineData(200)]
|
||||||
public async void ClientCutScenarioTest(int objectSize)
|
public async void ClientCutScenarioTest(int objectSize)
|
||||||
{
|
{
|
||||||
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
using var client = Client.GetInstance(GetOptions(this.key, this.url));
|
||||||
|
|
||||||
await Cleanup(client);
|
await Cleanup(client);
|
||||||
|
|
||||||
var createContainerParam = new PrmContainerCreate(new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
|
var createContainerParam = new PrmContainerCreate(new ModelsV2.Container(BasicAcl.PublicRW, new PlacementPolicy(true, new Replica(1))))
|
||||||
{
|
{
|
||||||
WaitParams = lightWait
|
WaitParams = lightWait
|
||||||
};
|
};
|
||||||
|
|
||||||
var containerId = await client.CreateContainerAsync(createContainerParam);
|
var containerId = await client.CreateContainerAsync(createContainerParam);
|
||||||
|
@ -449,8 +451,8 @@ public class SmokeTests
|
||||||
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
|
var objHeader = await client.GetObjectHeadAsync(new PrmObjectHeadGet(containerId, objectId));
|
||||||
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
Assert.Equal((ulong)bytes.Length, objHeader.PayloadLength);
|
||||||
Assert.Single(objHeader.Attributes);
|
Assert.Single(objHeader.Attributes);
|
||||||
Assert.Equal("fileName", objHeader.Attributes.First().Key);
|
Assert.Equal("fileName", objHeader.Attributes[0].Key);
|
||||||
Assert.Equal("test", objHeader.Attributes.First().Value);
|
Assert.Equal("test", objHeader.Attributes[0].Value);
|
||||||
}
|
}
|
||||||
|
|
||||||
Assert.True(hasObject);
|
Assert.True(hasObject);
|
||||||
|
@ -542,7 +544,7 @@ public class MetricsInterceptor() : Interceptor
|
||||||
|
|
||||||
watch.Stop();
|
watch.Stop();
|
||||||
|
|
||||||
// Do somethins with call info
|
// Do something with call info
|
||||||
// var elapsed = watch.ElapsedTicks * 1_000_000/Stopwatch.Frequency;
|
// var elapsed = watch.ElapsedTicks * 1_000_000/Stopwatch.Frequency;
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
|
|
Loading…
Reference in a new issue