[#4] infrastructure and sample Client Cut
All checks were successful
DCO / DCO (pull_request) Successful in 47s

Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2024-06-10 11:31:36 +03:00
parent 0c4723c705
commit 545e647d7b
22 changed files with 717 additions and 193 deletions

View file

@ -29,9 +29,8 @@ Global
{5012EF96-9C9E-4E77-BC78-B4111EE54107}.Debug|Any CPU.Build.0 = Debug|Any CPU {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.ActiveCfg = Release|Any CPU {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.Build.0 = Release|Any CPU {5012EF96-9C9E-4E77-BC78-B4111EE54107}.Release|Any CPU.Build.0 = Release|Any CPU
{B738F3E1-654D-41A3-B068-58ED122BB688}.Debug|Any CPU.ActiveCfg = Debug|Any CPU EndGlobalSection
{B738F3E1-654D-41A3-B068-58ED122BB688}.Debug|Any CPU.Build.0 = Debug|Any CPU GlobalSection(SolutionProperties) = preSolution
{B738F3E1-654D-41A3-B068-58ED122BB688}.Release|Any CPU.ActiveCfg = Release|Any CPU HideSolutionNode = FALSE
{B738F3E1-654D-41A3-B068-58ED122BB688}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection EndGlobalSection
EndGlobal EndGlobal

View file

@ -26,7 +26,7 @@ using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums; using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap; using FrostFS.SDK.ModelsV2.Netmap;
var fsClient = new Client(<your_key>, <your_host>); var fsClient = Client.GetInstance(<your_key>, <your_host>);
// List containers // List containers
var containersIds = await fsClient.ListContainersAsync(); var containersIds = await fsClient.ListContainersAsync();
@ -55,7 +55,7 @@ using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums; using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap; using FrostFS.SDK.ModelsV2.Netmap;
var fsClient = new Client(<your_key>, <your_host>); var fsClient = Client.GetInstance(<your_key>, <your_host>);
// Search regular objects // Search regular objects
var objectsIds = await fsClient.SearchObjectAsync( var objectsIds = await fsClient.SearchObjectAsync(
@ -77,4 +77,77 @@ var objHeader = await fsClient.GetObjectHeadAsync(cId, oId);
// Get object // Get object
var obj = await fsClient.GetObjectAsync(cId, oId); var obj = await fsClient.GetObjectAsync(cId, oId);
```
### Custom client cut
```csharp
using FrostFS.SDK.ClientV2;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap;
using FrostFS.SDK.ClientV2.Extensions;
using FrostFS.SDK.ClientV2.Interfaces;
var fsClient = Client.GetInstance(<your_key>, <your_host>);
ContainerId containerId = <containerId>;
string fileName = <fileName>;
await PutObjectClientCut(fsClient, containerId, fileName);
static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, ContainerId containerId, string fileName)
{
List<ObjectId> sentObjectIds = [];
FrostFS.SDK.ModelsV2.Object? currentObject;
var partSize = 1024 * 1024;
var buffer = new byte[partSize];
var largeObject = new LargeObject(containerId);
var split = new Split();
var fileInfo = new FileInfo(fileName);
var fullLength = (ulong)fileInfo.Length;
var fileNameAttribute = new ObjectAttribute("fileName", fileInfo.Name);
using var stream = File.OpenRead(fileName);
while (true)
{
var bytesCount = await stream.ReadAsync(buffer.AsMemory(0, partSize));
split.Previous = sentObjectIds.LastOrDefault();
largeObject.AppendBlock(buffer, bytesCount);
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, buffer)
.AddAttribute(fileNameAttribute)
.SetSplit(split);
if (largeObject.PayloadLength == fullLength)
break;
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
sentObjectIds.Add(objectId);
}
if (sentObjectIds.Any())
{
largeObject.CalculateHash();
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds);
_ = await fsClient.PutSingleObjectAsync(linkObject);
currentObject.SetParent(largeObject);
_ = await fsClient.PutSingleObjectAsync(currentObject);
return currentObject.GetParentId();
}
return await fsClient.PutSingleObjectAsync(currentObject);
}
``` ```

View file

@ -1,5 +1,6 @@
using System; using System;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Threading.Tasks;
using FrostFS.Container; using FrostFS.Container;
using FrostFS.Netmap; using FrostFS.Netmap;
using FrostFS.Object; using FrostFS.Object;
@ -25,7 +26,12 @@ public partial class Client: IFrostFSClient
private ObjectService.ObjectServiceClient? _objectServiceClient; private ObjectService.ObjectServiceClient? _objectServiceClient;
private SessionService.SessionServiceClient? _sessionServiceClient; private SessionService.SessionServiceClient? _sessionServiceClient;
public Client(string key, string host) public static IFrostFSClient GetInstance(string key, string host)
{
return new Client(key, host);
}
private Client(string key, string host)
{ {
// TODO: Развязать клиент и реализацию GRPC // TODO: Развязать клиент и реализацию GRPC
_key = key.LoadWif(); _key = key.LoadWif();
@ -37,7 +43,7 @@ public partial class Client: IFrostFSClient
InitSessionClient(); InitSessionClient();
CheckFrostFsVersionSupport(); CheckFrostFsVersionSupport();
} }
private async void CheckFrostFsVersionSupport() private async void CheckFrostFsVersionSupport()
{ {
var localNodeInfo = await GetLocalNodeInfoAsync(); var localNodeInfo = await GetLocalNodeInfoAsync();

View file

@ -0,0 +1,44 @@
using System.Collections.Generic;
using FrostFS.SDK.ModelsV2;
namespace FrostFS.SDK.ClientV2.Extensions;
public static class Extensions
{
public static ModelsV2.Object SetPayloadLength(this ModelsV2.Object obj, ulong length)
{
obj.Header.PayloadLength = length;
return obj;
}
public static ModelsV2.Object AddAttribute(this ModelsV2.Object obj, string key, string value)
{
obj.AddAttribute(new ObjectAttribute(key, value));
return obj;
}
public static ModelsV2.Object AddAttribute(this ModelsV2.Object obj, ObjectAttribute attribute)
{
obj.Header.Attributes.Add(attribute);
return obj;
}
public static ModelsV2.Object AddAttributes(this ModelsV2.Object obj, IEnumerable<ObjectAttribute> attributes)
{
obj.Header.Attributes.AddRange(attributes);
return obj;
}
public static ModelsV2.Object SetSplit(this ModelsV2.Object obj, Split split)
{
obj.Header.Split = split;
return obj;
}
public static LinkObject AddChildren(this LinkObject linkObject, IEnumerable<ObjectId> objectIds)
{
linkObject.Header.Split.Children.AddRange(objectIds);
return linkObject;
}
}

View file

@ -8,6 +8,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" /> <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="Moq.AutoMock" Version="3.5.0" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

View file

@ -9,13 +9,24 @@ namespace FrostFS.SDK.ClientV2.Interfaces;
public interface IFrostFSClient public interface IFrostFSClient
{ {
Task<ModelsV2.Container> GetContainerAsync(ContainerId containerId); Task<ModelsV2.Container> GetContainerAsync(ContainerId containerId);
IAsyncEnumerable<ContainerId> ListContainersAsync(); IAsyncEnumerable<ContainerId> ListContainersAsync();
Task<ContainerId> CreateContainerAsync(ModelsV2.Container container); Task<ContainerId> CreateContainerAsync(ModelsV2.Container container);
Task DeleteContainerAsync(ContainerId containerId); Task DeleteContainerAsync(ContainerId containerId);
Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId); Task<ObjectHeader> GetObjectHeadAsync(ContainerId containerId, ObjectId objectId);
Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId); Task<ModelsV2.Object> GetObjectAsync(ContainerId containerId, ObjectId objectId);
Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload); Task<ObjectId> PutObjectAsync(ObjectHeader header, Stream payload);
Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload); Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload);
Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object obj);
Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId); Task DeleteObjectAsync(ContainerId containerId, ObjectId objectId);
IAsyncEnumerable<ObjectId> SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters);
IAsyncEnumerable<ObjectId> SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters);
} }

View file

@ -1,4 +1,5 @@
using FrostFS.Refs; using FrostFS.Refs;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2;
using Google.Protobuf; using Google.Protobuf;
@ -10,7 +11,7 @@ public static class ContainerIdMapper
{ {
return new ContainerID return new ContainerID
{ {
Value = ByteString.CopyFrom(containerId.ToHash()) Value = ByteString.CopyFrom(Base58.Decode(containerId.Value))
}; };
} }
} }

View file

@ -1,8 +1,9 @@
using System; using System;
using System.Linq; using System.Linq;
using FrostFS.Object; using FrostFS.Object;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2;
using Google.Protobuf;
using MatchType = FrostFS.Object.MatchType; using MatchType = FrostFS.Object.MatchType;
using ObjectType = FrostFS.Object.ObjectType; using ObjectType = FrostFS.Object.ObjectType;
@ -29,11 +30,20 @@ public static class ObjectFilterMapper
{ {
public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this ObjectFilter filter) public static SearchRequest.Types.Body.Types.Filter ToGrpcMessage(this ObjectFilter filter)
{ {
var objMatchTypeName = Enum.GetName(typeof(MatchType), filter.MatchType) var objMatchTypeName = filter.MatchType switch
?? throw new ArgumentException($"Unknown MatchType. Value: '{filter.MatchType}'."); {
ModelsV2.Enums.ObjectMatchType.Unspecified => MatchType.Unspecified,
ModelsV2.Enums.ObjectMatchType.Equals => MatchType.StringEqual,
ModelsV2.Enums.ObjectMatchType.NotEquals => MatchType.StringNotEqual,
ModelsV2.Enums.ObjectMatchType.KeyAbsent => MatchType.NotPresent,
ModelsV2.Enums.ObjectMatchType.StartsWith => MatchType.CommonPrefix,
_ => throw new ArgumentException($"Unknown MatchType. Value: '{filter.MatchType}'.")
};
return new SearchRequest.Types.Body.Types.Filter return new SearchRequest.Types.Body.Types.Filter
{ {
MatchType = (MatchType)Enum.Parse(typeof(MatchType), objMatchTypeName), MatchType = objMatchTypeName,
Key = filter.Key, Key = filter.Key,
Value = filter.Value Value = filter.Value
}; };
@ -44,13 +54,19 @@ public static class ObjectHeaderMapper
{ {
public static Header ToGrpcMessage(this ObjectHeader header) public static Header ToGrpcMessage(this ObjectHeader header)
{ {
var objTypeName = Enum.GetName(typeof(ObjectType), header.ObjectType) var objTypeName = header.ObjectType switch
?? throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.");
var head = new Header
{ {
Attributes = { }, ModelsV2.Enums.ObjectType.Regular => ObjectType.Regular,
ModelsV2.Enums.ObjectType.Lock => ObjectType.Lock,
ModelsV2.Enums.ObjectType.Tombstone => ObjectType.Tombstone,
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
};
var head = new Header
{
ContainerId = header.ContainerId.ToGrpcMessage(), ContainerId = header.ContainerId.ToGrpcMessage(),
ObjectType = (ObjectType)Enum.Parse(typeof(ObjectType), objTypeName) ObjectType = objTypeName,
PayloadLength = header.PayloadLength
}; };
foreach (var attribute in header.Attributes) foreach (var attribute in header.Attributes)
@ -58,20 +74,42 @@ public static class ObjectHeaderMapper
head.Attributes.Add(attribute.ToGrpcMessage()); head.Attributes.Add(attribute.ToGrpcMessage());
} }
var split = header.Split;
if (split != null)
{
head.Split = new Header.Types.Split
{
Parent = split.Parent?.ToGrpcMessage(),
ParentSignature = split.ParentSignature?.ToGrpcMessage(),
ParentHeader = split.ParentHeader?.ToGrpcMessage(),
Previous = split.Previous?.ToGrpcMessage(),
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
};
if (split.Children != null && split.Children.Any())
head.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
}
return head; return head;
} }
public static ObjectHeader ToModel(this Header header) public static ObjectHeader ToModel(this Header header)
{ {
var objTypeName = Enum.GetName(typeof(ModelsV2.Enums.ObjectType), header.ObjectType) var objTypeName = header.ObjectType switch
?? throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'."); {
ObjectType.Regular => ModelsV2.Enums.ObjectType.Regular,
ObjectType.Lock => ModelsV2.Enums.ObjectType.Lock,
ObjectType.Tombstone => ModelsV2.Enums.ObjectType.Tombstone,
_ => throw new ArgumentException($"Unknown ObjectType. Value: '{header.ObjectType}'.")
};
return new ObjectHeader( return new ObjectHeader(
ContainerId.FromHash(header.ContainerId.Value.ToByteArray()), new ContainerId(Base58.Encode(header.ContainerId.Value.ToByteArray())),
(ModelsV2.Enums.ObjectType)Enum.Parse(typeof(ModelsV2.Enums.ObjectType), objTypeName), objTypeName,
header.Attributes.Select(attribute => attribute.ToModel()).ToArray() header.Attributes.Select(attribute => attribute.ToModel()).ToArray()
) )
{ {
Size = (long)header.PayloadLength, PayloadLength = header.PayloadLength,
Version = header.Version.ToModel() Version = header.Version.ToModel()
}; };
} }
@ -81,11 +119,33 @@ public static class ObjectMapper
{ {
public static ModelsV2.Object ToModel(this Object.Object obj) public static ModelsV2.Object ToModel(this Object.Object obj)
{ {
return new ModelsV2.Object return new ModelsV2.Object()
{ {
Header = obj.Header.ToModel(), Header = obj.Header.ToModel(),
ObjectId = ObjectId.FromHash(obj.ObjectId.Value.ToByteArray()), ObjectId = ObjectId.FromHash(obj.ObjectId.Value.ToByteArray()),
Payload = obj.Payload.ToByteArray() Payload = obj.Payload.ToByteArray()
}; };
}
}
public static class SignatureMapper
{
public static Refs.Signature ToGrpcMessage(this Signature signature)
{
var scheme = signature.Scheme switch
{
SignatureScheme.EcdsaRfc6979Sha256 => Refs.SignatureScheme.EcdsaRfc6979Sha256,
SignatureScheme.EcdsaRfc6979Sha256WalletConnect => Refs.SignatureScheme.EcdsaRfc6979Sha256WalletConnect,
SignatureScheme.EcdsaSha512 => Refs.SignatureScheme.EcdsaSha512,
_ => throw new ArgumentException(message: $"Unexpected enum value: {signature.Scheme}", paramName: nameof(signature.Scheme))
};
return new Refs.Signature
{
Key = ByteString.CopyFrom(signature.Key),
Scheme = scheme,
Sign = ByteString.CopyFrom(signature.Sign)
};
} }
} }

View file

@ -13,4 +13,9 @@ public static class ObjectIdMapper
Value = ByteString.CopyFrom(objectId.ToHash()) Value = ByteString.CopyFrom(objectId.ToHash())
}; };
} }
public static ObjectId ToModel(this ObjectID objectId)
{
return ObjectId.FromHash(objectId.Value.ToByteArray());
}
} }

View file

@ -91,7 +91,7 @@ public static class RequestSigner
{ {
IRequest => new RequestVerificationHeader(), IRequest => new RequestVerificationHeader(),
IResponse => new ResponseVerificationHeader(), IResponse => new ResponseVerificationHeader(),
_ => throw new InvalidOperationException("Unsopported message type") _ => throw new InvalidOperationException("Unsupported message type")
}; };
var verifyOrigin = message.GetVerificationHeader(); var verifyOrigin = message.GetVerificationHeader();

View file

@ -1,5 +1,6 @@
using FrostFS.Container; using FrostFS.Container;
using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.ClientV2.Mappers.GRPC;
using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2; using FrostFS.SDK.ModelsV2;
using System.Collections.Generic; using System.Collections.Generic;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -17,6 +18,7 @@ public partial class Client
ContainerId = cid.ToGrpcMessage() ContainerId = cid.ToGrpcMessage()
}, },
}; };
request.AddMetaHeader(); request.AddMetaHeader();
request.Sign(_key); request.Sign(_key);
var response = await _containerServiceClient.GetAsync(request); var response = await _containerServiceClient.GetAsync(request);
@ -33,13 +35,14 @@ public partial class Client
OwnerId = OwnerId.ToGrpcMessage() OwnerId = OwnerId.ToGrpcMessage()
} }
}; };
request.AddMetaHeader(); request.AddMetaHeader();
request.Sign(_key); request.Sign(_key);
var response = await _containerServiceClient.ListAsync(request); var response = await _containerServiceClient.ListAsync(request);
Verifier.CheckResponse(response); Verifier.CheckResponse(response);
foreach (var cid in response.Body.ContainerIds) foreach (var cid in response.Body.ContainerIds)
{ {
yield return ContainerId.FromHash(cid.Value.ToByteArray()); yield return new ContainerId(Base58.Encode(cid.Value.ToByteArray()));
} }
} }
@ -48,6 +51,7 @@ public partial class Client
var cntnr = container.ToGrpcMessage(); var cntnr = container.ToGrpcMessage();
cntnr.OwnerId = OwnerId.ToGrpcMessage(); cntnr.OwnerId = OwnerId.ToGrpcMessage();
cntnr.Version = Version.ToGrpcMessage(); cntnr.Version = Version.ToGrpcMessage();
var request = new PutRequest var request = new PutRequest
{ {
Body = new PutRequest.Types.Body Body = new PutRequest.Types.Body
@ -60,7 +64,7 @@ public partial class Client
request.Sign(_key); request.Sign(_key);
var response = await _containerServiceClient.PutAsync(request); var response = await _containerServiceClient.PutAsync(request);
Verifier.CheckResponse(response); Verifier.CheckResponse(response);
return ContainerId.FromHash(response.Body.ContainerId.Value.ToByteArray()); return new ContainerId(Base58.Encode(response.Body.ContainerId.Value.ToByteArray()));
} }
public async Task DeleteContainerAsync(ContainerId cid) public async Task DeleteContainerAsync(ContainerId cid)

View file

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Google.Protobuf; using Google.Protobuf;
@ -9,9 +10,10 @@ using FrostFS.Object;
using FrostFS.Refs; using FrostFS.Refs;
using FrostFS.SDK.ClientV2.Mappers.GRPC; using FrostFS.SDK.ClientV2.Mappers.GRPC;
using FrostFS.SDK.Cryptography; using FrostFS.SDK.Cryptography;
using FrostFS.SDK.ModelsV2;
using FrostFS.Session; using FrostFS.Session;
using FrostFS.SDK.ModelsV2;
namespace FrostFS.SDK.ClientV2; namespace FrostFS.SDK.ClientV2;
public partial class Client public partial class Client
@ -20,7 +22,7 @@ public partial class Client
{ {
var request = new HeadRequest var request = new HeadRequest
{ {
Body = new HeadRequest.Types.Body Body = new HeadRequest.Types.Body
{ {
Address = new Address Address = new Address
{ {
@ -29,12 +31,12 @@ public partial class Client
} }
} }
}; };
request.AddMetaHeader(); request.AddMetaHeader();
request.Sign(_key); request.Sign(_key);
var response = await _objectServiceClient.HeadAsync(request); var response = await _objectServiceClient!.HeadAsync(request);
Verifier.CheckResponse(response); Verifier.CheckResponse(response);
return response.Body.Header.Header.ToModel(); return response.Body.Header.Header.ToModel();
} }
@ -45,7 +47,6 @@ public partial class Client
{ {
Body = new GetRequest.Types.Body Body = new GetRequest.Types.Body
{ {
Raw = false,
Address = new Address Address = new Address
{ {
ContainerId = cid.ToGrpcMessage(), ContainerId = cid.ToGrpcMessage(),
@ -76,85 +77,8 @@ public partial class Client
public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload) public async Task<ObjectId> PutObjectAsync(ObjectHeader header, byte[] payload)
{ {
return await PutObject(header, new MemoryStream(payload)); using var stream = new MemoryStream(payload);
} return await PutObject(header, stream);
public async Task DeleteObjectAsync(ContainerId cid, ObjectId oid)
{
var request = new DeleteRequest
{
Body = new DeleteRequest.Types.Body
{
Address = new Address
{
ContainerId = cid.ToGrpcMessage(),
ObjectId = oid.ToGrpcMessage()
}
}
};
request.AddMetaHeader();
request.Sign(_key);
var response = await _objectServiceClient.DeleteAsync(request);
Verifier.CheckResponse(response);
}
public async IAsyncEnumerable<ObjectId> SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters)
{
var request = new SearchRequest
{
Body = new SearchRequest.Types.Body
{
ContainerId = cid.ToGrpcMessage(),
Filters = { },
Version = 1
}
};
foreach (var filter in filters)
{
request.Body.Filters.Add(filter.ToGrpcMessage());
}
request.AddMetaHeader();
request.Sign(_key);
var objectsIds = SearchObjects(request);
await foreach (var oid in objectsIds)
{
yield return ObjectId.FromHash(oid.Value.ToByteArray());
}
}
private async Task<Object.Object> GetObject(GetRequest request)
{
using var stream = GetObjectInit(request);
var obj = await stream.ReadHeader();
var payload = new byte[obj.Header.PayloadLength];
var offset = 0;
var chunk = await stream.ReadChunk();
while (chunk is not null)
{
chunk.CopyTo(payload, offset);
offset += chunk.Length;
chunk = await stream.ReadChunk();
}
obj.Payload = ByteString.CopyFrom(payload);
return obj;
}
private ObjectReader GetObjectInit(GetRequest initRequest)
{
if (initRequest is null)
throw new ArgumentNullException(nameof(initRequest));
return new ObjectReader
{
Call = _objectServiceClient.Get(initRequest)
};
} }
private async Task<ObjectId> PutObject(ObjectHeader header, Stream payload) private async Task<ObjectId> PutObject(ObjectHeader header, Stream payload)
@ -163,12 +87,12 @@ public partial class Client
var hdr = header.ToGrpcMessage(); var hdr = header.ToGrpcMessage();
hdr.OwnerId = OwnerId.ToGrpcMessage(); hdr.OwnerId = OwnerId.ToGrpcMessage();
hdr.Version = Version.ToGrpcMessage(); hdr.Version = Version.ToGrpcMessage();
var oid = new ObjectID var oid = new ObjectID
{ {
Value = hdr.Sha256() Value = hdr.Sha256()
}; };
var request = new PutRequest var request = new PutRequest
{ {
Body = new PutRequest.Types.Body Body = new PutRequest.Types.Body
@ -188,23 +112,27 @@ public partial class Client
ObjectSessionContext.Types.Verb.Put, ObjectSessionContext.Types.Verb.Put,
_key _key
); );
request.Sign(_key); request.Sign(_key);
using var stream = await PutObjectInit(request); using var stream = await PutObjectInit(request);
var buffer = new byte[Constants.ObjectChunkSize]; var buffer = new byte[Constants.ObjectChunkSize];
var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
while (true)
while (bufferLength > 0)
{ {
var bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
if (bufferLength == 0)
break;
request.Body = new PutRequest.Types.Body request.Body = new PutRequest.Types.Body
{ {
Chunk = ByteString.CopyFrom(buffer[..bufferLength]), Chunk = ByteString.CopyFrom(buffer[..bufferLength]),
}; };
request.VerifyHeader = null; request.VerifyHeader = null;
request.Sign(_key); request.Sign(_key);
await stream.Write(request); await stream.Write(request);
bufferLength = await payload.ReadAsync(buffer, 0, Constants.ObjectChunkSize);
} }
var response = await stream.Close(); var response = await stream.Close();
@ -213,6 +141,192 @@ public partial class Client
return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray()); return ObjectId.FromHash(response.Body.ObjectId.Value.ToByteArray());
} }
public async Task<ObjectId> PutSingleObjectAsync(ModelsV2.Object @object)
{
var sessionToken = await CreateSessionAsync(uint.MaxValue);
var obj = CreateObject(@object);
var request = new PutSingleRequest
{
Body = new () { Object = obj }
};
request.AddMetaHeader();
request.AddObjectSessionToken(
sessionToken,
obj.Header.ContainerId,
obj.ObjectId,
ObjectSessionContext.Types.Verb.Put,
_key
);
request.Sign(_key);
var response = await _objectServiceClient!.PutSingleAsync(request);
Verifier.CheckResponse(response);
return ObjectId.FromHash(obj.ObjectId.Value.ToByteArray());
}
public Object.Object CreateObject(ModelsV2.Object @object)
{
var grpcHeader = @object.Header.ToGrpcMessage();
grpcHeader.OwnerId = OwnerId.ToGrpcMessage();
grpcHeader.Version = Version.ToGrpcMessage();
if (@object.Payload != null)
{
grpcHeader.PayloadLength = (ulong)@object.Payload.Length;
grpcHeader.PayloadHash = Sha256Checksum(@object.Payload);
}
var split = @object.Header.Split;
if (split != null)
{
grpcHeader.Split = new Header.Types.Split
{
SplitId = split.SplitId != null ? ByteString.CopyFrom(split.SplitId.ToBinary()) : null
};
if (split.Children != null && split.Children.Any())
grpcHeader.Split.Children.AddRange(split.Children.Select(id => id.ToGrpcMessage()));
if (split.ParentHeader is not null)
{
var grpcParentHeader = CreateHeader(split.ParentHeader, []);
grpcHeader.Split.Parent = new ObjectID { Value = grpcParentHeader.Sha256() };
grpcHeader.Split.ParentHeader = grpcParentHeader;
grpcHeader.Split.ParentSignature = new Refs.Signature
{
Key = ByteString.CopyFrom(_key.PublicKey()),
Sign = ByteString.CopyFrom(_key.SignData(grpcHeader.Split.Parent.ToByteArray())),
};
split.Parent = grpcHeader.Split.Parent.ToModel();
}
}
var obj = new Object.Object
{
Header = grpcHeader,
ObjectId = new ObjectID { Value = grpcHeader.Sha256() },
Payload = ByteString.CopyFrom(@object.Payload)
};
obj.Signature = new Refs.Signature
{
Key = ByteString.CopyFrom(_key.PublicKey()),
Sign = ByteString.CopyFrom(_key.SignData(obj.ObjectId.ToByteArray())),
};
return obj;
}
public Header CreateHeader(ObjectHeader header, byte[]? payload)
{
var grpcHeader = header.ToGrpcMessage();
grpcHeader.OwnerId = OwnerId.ToGrpcMessage();
grpcHeader.Version = Version.ToGrpcMessage();
if (header.PayloadCheckSum != null)
{
grpcHeader.PayloadHash = new Checksum
{
Type = ChecksumType.Sha256,
Sum = ByteString.CopyFrom(header.PayloadCheckSum)
};
}
else
{
if (payload != null)
grpcHeader.PayloadHash = Sha256Checksum(payload);
}
return grpcHeader;
}
public async Task DeleteObjectAsync(ContainerId cid, ObjectId oid)
{
var request = new DeleteRequest
{
Body = new DeleteRequest.Types.Body
{
Address = new Address
{
ContainerId = cid.ToGrpcMessage(),
ObjectId = oid.ToGrpcMessage()
}
}
};
request.AddMetaHeader();
request.Sign(_key);
var response = await _objectServiceClient!.DeleteAsync(request);
Verifier.CheckResponse(response);
}
public async IAsyncEnumerable<ObjectId> SearchObjectsAsync(ContainerId cid, params ObjectFilter[] filters)
{
var request = new SearchRequest
{
Body = new SearchRequest.Types.Body
{
ContainerId = cid.ToGrpcMessage(),
Filters = { },
Version = 1
}
};
foreach (var filter in filters)
{
request.Body.Filters.Add(filter.ToGrpcMessage());
}
request.AddMetaHeader();
request.Sign(_key);
var objectsIds = SearchObjects(request);
await foreach (var oid in objectsIds)
{
yield return ObjectId.FromHash(oid.Value.ToByteArray());
}
}
private async Task<Object.Object> GetObject(GetRequest request)
{
using var stream = GetObjectInit(request);
var obj = await stream.ReadHeader();
var payload = new byte[obj.Header.PayloadLength];
var offset = 0;
var chunk = await stream.ReadChunk();
while (chunk is not null)
{
chunk.CopyTo(payload, offset);
offset += chunk.Length;
chunk = await stream.ReadChunk();
}
obj.Payload = ByteString.CopyFrom(payload);
return obj;
}
private ObjectReader GetObjectInit(GetRequest initRequest)
{
if (initRequest is null)
throw new ArgumentNullException(nameof(initRequest));
return new ObjectReader
{
Call = _objectServiceClient!.Get(initRequest)
};
}
private async Task<ObjectStreamer> PutObjectInit(PutRequest initRequest) private async Task<ObjectStreamer> PutObjectInit(PutRequest initRequest)
{ {
if (initRequest is null) if (initRequest is null)
@ -220,9 +334,9 @@ public partial class Client
throw new ArgumentNullException(nameof(initRequest)); throw new ArgumentNullException(nameof(initRequest));
} }
var call = _objectServiceClient.Put(); var call = _objectServiceClient!.Put();
await call.RequestStream.WriteAsync(initRequest); await call.RequestStream.WriteAsync(initRequest);
return new ObjectStreamer(call); return new ObjectStreamer(call);
} }
@ -236,7 +350,7 @@ public partial class Client
{ {
yield return oid; yield return oid;
} }
ids = await stream.Read(); ids = await stream.Read();
} }
} }
@ -248,6 +362,17 @@ public partial class Client
throw new ArgumentNullException(nameof(initRequest)); throw new ArgumentNullException(nameof(initRequest));
} }
return new SearchReader(_objectServiceClient.Search(initRequest)); return new SearchReader(_objectServiceClient!.Search(initRequest));
}
public Checksum Sha256Checksum(byte[] data)
{
return new Checksum
{
Type = ChecksumType.Sha256,
Sum = ByteString.CopyFrom(data.Sha256())
};
} }
} }

View file

@ -3,8 +3,6 @@ using System.Linq;
using System.Numerics; using System.Numerics;
using System.Text; using System.Text;
using Org.BouncyCastle.Security.Certificates;
namespace FrostFS.SDK.Cryptography; namespace FrostFS.SDK.Cryptography;
public static class Base58 public static class Base58

View file

@ -44,7 +44,6 @@ public static class Helper
{ {
return ByteString.CopyFrom(data.ToByteArray().Sha256()); return ByteString.CopyFrom(data.ToByteArray().Sha256());
} }
public static ulong Murmur64(this byte[] value, uint seed) public static ulong Murmur64(this byte[] value, uint seed)
{ {

View file

@ -1,30 +1,8 @@
using System; namespace FrostFS.SDK.ModelsV2;
using FrostFS.SDK.Cryptography; public class ContainerId(string id)
namespace FrostFS.SDK.ModelsV2;
public class ContainerId
{ {
public string Value { get; set; } public string Value { get; set; } = id;
public ContainerId(string id)
{
Value = id;
}
public static ContainerId FromHash(byte[] hash)
{
if (hash.Length != Constants.Sha256HashLength)
throw new FormatException("ContainerID must be a sha256 hash.");
return new ContainerId(Base58.Encode(hash));
}
public byte[] ToHash()
{
return Base58.Decode(Value);
}
public override string ToString() public override string ToString()
{ {

View file

@ -0,0 +1,65 @@
using System.Security.Cryptography;
using FrostFS.SDK.ModelsV2.Enums;
namespace FrostFS.SDK.ModelsV2;
public class Object
{
public Object()
{
}
public Object(ContainerId container, byte[] payload, ObjectType objectType = ObjectType.Regular)
{
Payload = payload;
Header = new ObjectHeader(containerId: container, type: objectType, attributes: []);
}
public ObjectHeader Header { get; set; }
public ObjectId ObjectId { get; set; }
public byte[] Payload { get; set; }
public Signature Signature { get; set; }
public void SetParent(LargeObject largeObject)
{
Header.Split!.ParentHeader = largeObject.Header;
}
public ObjectId? GetParentId()
{
return Header.Split?.Parent;
}
}
public class LargeObject(ContainerId container) : Object(container, [])
{
private SHA256 payloadHash = SHA256.Create();
public void AppendBlock(byte[] bytes, int count)
{
Header.PayloadLength += (ulong)count;
this.payloadHash.TransformBlock(bytes, 0, count, bytes, 0);
}
public void CalculateHash()
{
this.payloadHash.TransformFinalBlock([], 0, 0);
Header.PayloadCheckSum = this.payloadHash.Hash;
}
public ulong PayloadLength
{
get { return Header.PayloadLength; }
}
}
public class LinkObject : Object
{
public LinkObject(ContainerId containerId, SplitId splitId, LargeObject largeObject) : base (containerId, [])
{
Header.Split = new Split(splitId)
{
ParentHeader = largeObject.Header
};
}
}

View file

@ -0,0 +1,13 @@
namespace FrostFS.SDK.ModelsV2;
public class ObjectAttribute
{
public string Key { get; set; }
public string Value { get; set; }
public ObjectAttribute(string key, string value)
{
Key = key;
Value = value;
}
}

View file

@ -2,18 +2,6 @@ using FrostFS.SDK.ModelsV2.Enums;
namespace FrostFS.SDK.ModelsV2; namespace FrostFS.SDK.ModelsV2;
public class ObjectAttribute
{
public string Key { get; set; }
public string Value { get; set; }
public ObjectAttribute(string key, string value)
{
Key = key;
Value = value;
}
}
public class ObjectFilter public class ObjectFilter
{ {
private const string HeaderPrefix = "$Object:"; private const string HeaderPrefix = "$Object:";
@ -48,30 +36,3 @@ public class ObjectFilter
return new ObjectFilter(matchType, HeaderPrefix + "version", version.ToString()); return new ObjectFilter(matchType, HeaderPrefix + "version", version.ToString());
} }
} }
public class ObjectHeader
{
public ObjectAttribute[] Attributes { get; set; }
public ContainerId ContainerId { get; set; }
public long Size { get; set; }
public ObjectType ObjectType { get; set; }
public Version Version { get; set; }
public ObjectHeader(
ContainerId containerId,
ObjectType type = ObjectType.Regular,
params ObjectAttribute[] attributes
)
{
Attributes = attributes;
ContainerId = containerId;
ObjectType = type;
}
}
public class Object
{
public ObjectHeader Header { get; set; }
public ObjectId ObjectId { get; set; }
public byte[] Payload { get; set; }
}

View file

@ -0,0 +1,37 @@
using System.Collections.Generic;
using FrostFS.SDK.ModelsV2.Enums;
namespace FrostFS.SDK.ModelsV2;
public class ObjectHeader
{
public OwnerId OwnerId { get; set; }
public List<ObjectAttribute> Attributes { get; set; }
public ContainerId ContainerId { get; set; }
public ulong PayloadLength { get; set; }
public byte[]? PayloadCheckSum { get; set; }
public ObjectType ObjectType { get; set; }
public Version Version { get; set; }
public Split? Split { get; set; }
public bool ClientCut { get; set; }
public ObjectHeader(
ContainerId containerId,
ObjectType type = ObjectType.Regular,
params ObjectAttribute[] attributes
)
{
Attributes = [.. attributes];
ContainerId = containerId;
ObjectType = type;
}
}

View file

@ -0,0 +1,88 @@
using System;
using System.Collections.Generic;
namespace FrostFS.SDK.ModelsV2;
public class Split
{
public Split() : this(new SplitId())
{
}
public Split(SplitId splitId)
{
SplitId = splitId;
}
public SplitId SplitId { get; private set; }
public ObjectId? Parent { get; set; }
public ObjectId? Previous { get; set; }
public Signature? ParentSignature { get; set; }
public ObjectHeader? ParentHeader { get; set; }
public List<ObjectId> Children { get; } = [];
}
public enum SignatureScheme {
EcdsaSha512,
EcdsaRfc6979Sha256,
EcdsaRfc6979Sha256WalletConnect
}
public class Signature
{
public byte[] Key { get; set; }
public byte[] Sign { get; set; }
public SignatureScheme Scheme { get; set; }
}
public class SplitId
{
private Guid id;
public SplitId()
{
this.id = Guid.NewGuid();
}
public SplitId(Guid guid)
{
this.id = guid;
}
private SplitId(byte[] binary)
{
this.id = new Guid(binary);
}
private SplitId(string str)
{
this.id = new Guid(str);
}
public static SplitId CrateFromBinary(byte[] binaryData)
{
return new SplitId(binaryData);
}
public static SplitId CrateFromString(string stringData)
{
return new SplitId(stringData);
}
public override string ToString()
{
return this.id.ToString();
}
public byte[]? ToBinary()
{
if (this.id == Guid.Empty)
return null;
return this.id.ToByteArray();
}
}

View file

@ -115,6 +115,62 @@ namespace FrostFS.Object
} }
} }
public partial class PutSingleRequest : IRequest
{
IMetaHeader IVerificableMessage.GetMetaHeader()
{
return MetaHeader;
}
IVerificationHeader IVerificableMessage.GetVerificationHeader()
{
return VerifyHeader;
}
void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader)
{
MetaHeader = (RequestMetaHeader)metaHeader;
}
void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader)
{
VerifyHeader = (RequestVerificationHeader)verificationHeader;
}
public IMessage GetBody()
{
return Body;
}
}
public partial class PutSingleResponse : IResponse
{
IMetaHeader IVerificableMessage.GetMetaHeader()
{
return MetaHeader;
}
IVerificationHeader IVerificableMessage.GetVerificationHeader()
{
return VerifyHeader;
}
void IVerificableMessage.SetMetaHeader(IMetaHeader metaHeader)
{
MetaHeader = (ResponseMetaHeader)metaHeader;
}
void IVerificableMessage.SetVerificationHeader(IVerificationHeader verificationHeader)
{
VerifyHeader = (ResponseVerificationHeader)verificationHeader;
}
public IMessage GetBody()
{
return Body;
}
}
public partial class DeleteRequest : IRequest public partial class DeleteRequest : IRequest
{ {
IMetaHeader IVerificableMessage.GetMetaHeader() IMetaHeader IVerificableMessage.GetMetaHeader()

View file

@ -2,7 +2,7 @@
public partial class XHeader public partial class XHeader
{ {
public const string ReservedXHeaderPrefix = "__NEOFS__"; public const string ReservedXHeaderPrefix = "__SYSTEM__";
public const string XHeaderNetmapEpoch = ReservedXHeaderPrefix + "NETMAP_EPOCH"; public const string XHeaderNetmapEpoch = ReservedXHeaderPrefix + "NETMAP_EPOCH";
public const string XHeaderNetmapLookupDepth = ReservedXHeaderPrefix + "NETMAP_LOOKUP_DEPTH"; public const string XHeaderNetmapLookupDepth = ReservedXHeaderPrefix + "NETMAP_LOOKUP_DEPTH";
} }