[#27] Client: demo project

Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
Pavel Gross 2024-12-16 13:31:41 +03:00
parent 30e0daa022
commit 659bf3f19c
2 changed files with 314 additions and 194 deletions

View file

@ -1,220 +1,338 @@
using System.Diagnostics;
using System;
using System.Collections;
using System.Collections.ObjectModel;
using System.Drawing;
using System.IO;
using FrostFS.SDK.ClientV2;
using FrostFS.SDK.ModelsV2;
using FrostFS.SDK.ModelsV2.Enums;
using FrostFS.SDK.ModelsV2.Netmap;
using FrostFS.SDK.ClientV2.Extensions;
using FrostFS.SDK.ClientV2.Interfaces;
using Frostfs.V2.Ape;
try
using FrostFS.Refs;
using FrostFS.SDK;
using FrostFS.SDK.Client;
using FrostFS.SDK.Client.Interfaces;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Options;
using Org.BouncyCastle.Utilities;
using static FrostFS.Object.PatchRequest.Types.Body.Types;
class Demo
{
var fsClient = Client.GetInstance("KwHDAJ66o8FoLBjVbjP2sWBmgBMGjt7Vv4boA7xQrBoAYBE397Aq", "http://172.29.238.97:8080");
private const string Key = "KzPXA6669m2pf18XmUdoR8MnP1pi1PMmefiFujStVFnv7WR5SRmK";
private const string Host = "http://172.23.32.4:8080";
await FetchAndRemoveContainers(fsClient);
ContainerId containerId = await CreateContainer(fsClient);
ShowFileInExplorer();
var fileName = "cat.jpg";
//var fileName = "fat_cat.bmp";
var objectId = await PutObject(fsClient, containerId, fileName);
//var objectId = await PutSingleObject(fsClient, containerId, fileName);
//var objectId = await PutObjectClientCut(fsClient, containerId, fileName);
var filter = new ObjectFilter(ObjectMatchType.Equals, "fileName", fileName);
await foreach (var objId in fsClient.SearchObjectsAsync(containerId, filter))
static void Main(string[] args)
{
await GetObjectInfo(fsClient, containerId, objId);
}
var @object = await fsClient.GetObjectAsync(containerId, objectId!);
File.WriteAllBytes(GetBroFileName(fileName), @object.Payload);
await foreach (var cid in fsClient.ListContainersAsync())
#region Network
public static async Task<FrostFsNetmapSnapshot> GetNetmapSnapshotAsync()
{
Console.WriteLine($"ContainerId: {cid} (removing...)");
await fsClient.DeleteContainerAsync(cid);
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
return await client.GetNetmapSnapshotAsync(default);
}
await Task.Delay(500);
await foreach (var cid in fsClient.ListContainersAsync())
public static async Task<FrostFsNodeInfo> GetNodeInfoAsync()
{
Console.WriteLine($"Container {cid} still alive!");
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
return await client.GetNodeInfoAsync(default);
}
}
catch (Exception ex)
{
Console.WriteLine($"Error: {ex.Message}");
}
static async Task FetchAndRemoveContainers(IFrostFSClient fsClient)
{
Console.WriteLine("Existing containers:");
await foreach (var cid in fsClient.ListContainersAsync())
public static async Task<NetworkSettings> GetNetworkSettingsAsync()
{
await fsClient.DeleteContainerAsync(cid);
Console.WriteLine($"Removed container: {cid}");
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
return await client.GetNetworkSettingsAsync(default);
}
}
#endregion
static async Task<ContainerId> CreateContainer(IFrostFSClient fsClient)
{
var placementPolicy = new PlacementPolicy(true, new Replica(1));
var containerId = await fsClient.CreateContainerAsync(new Container(BasicAcl.PublicRW, placementPolicy));
Console.WriteLine($"Created container: {containerId}");
Container? container = null;
while (container is null)
#region Session
public static async Task<FrostFsSessionToken> GetSessionTokenAsync()
{
try
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
return await client.CreateSessionAsync(new PrmSessionCreate(int.MaxValue), default);
}
#endregion
#region Chain
public static async Task<ReadOnlyMemory<byte>> AddChainAsync(string targetString, string name, byte[] raw)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var target = new FrostFsChainTarget(Enum.Parse<FrostFsTargetType>(targetString), name);
var chain = new FrostFsChain(raw);
var args = new PrmApeChainAdd(target, chain);
return await client.AddChainAsync(args, default);
}
public static async Task RemoveChainAsync(string targetString, string name, byte[] raw)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var target = new FrostFsChainTarget(Enum.Parse<FrostFsTargetType>(targetString), name);
var chain = new FrostFsChain(raw);
var args = new PrmApeChainRemove(target, chain);
await client.RemoveChainAsync(args, default);
}
public static async Task RemoveChainAsync(string targetString, string name)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var target = new FrostFsChainTarget(Enum.Parse<FrostFsTargetType>(targetString), name);
var args = new PrmApeChainList(target);
await client.ListChainAsync(args, default);
}
#endregion
#region Container
public static async Task<FrostFsContainerId> CreateContainerAsync()
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
PrmWait.DefaultParams);
return await client.CreateContainerAsync(createContainerParam, default);
}
public static async Task<FrostFsContainerId> GetContainerAsync()
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var createContainerParam = new PrmContainerCreate(
new FrostFsContainerInfo(new FrostFsPlacementPolicy(true, new FrostFsReplica(1))),
PrmWait.DefaultParams);
return await client.CreateContainerAsync(createContainerParam, default);
}
public static async Task DeleteContainerAsync(FrostFsContainerId containerId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmContainerDelete(containerId, new PrmWait(60, 5));
await client.DeleteContainerAsync(args, default);
}
public static async Task<ICollection<FrostFsContainerId>> ListContainersAsync(FrostFsContainerId containerId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmContainerGetAll();
Collection<FrostFsContainerId> containers = [];
await foreach (var cid in client.ListContainersAsync(args, default))
{
await Task.Delay(1000);
container = await fsClient.GetContainerAsync(containerId);
containers.Add(cid);
}
catch
return containers;
}
#endregion
#region Object
public static async Task<FrostFsHeaderResult> GetObjectHeadAsync(FrostFsContainerId containerId, FrostFsObjectId objectId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmObjectHeadGet(containerId, objectId);
return await client.GetObjectHeadAsync(args, default);
}
public static async Task<FrostFsObject> GetObjectAsync(FrostFsContainerId containerId, FrostFsObjectId objectId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmObjectGet(containerId, objectId);
return await client.GetObjectAsync(args, default);
}
public static async Task<byte[]> GetRangeAsync(FrostFsContainerId containerId, FrostFsObjectId objectId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmRangeGet(containerId, objectId, new FrostFsRange(64, 128));
var rangeReader = await client.GetRangeAsync(args, default);
var downloadedBytes = new byte[128];
MemoryStream ms = new(downloadedBytes);
ReadOnlyMemory<byte>? chunk;
while ((chunk = await rangeReader!.ReadChunk()) != null)
{
Console.WriteLine("Waiting for the container is ready. Some time is required");
ms.Write(chunk.Value.Span);
}
return downloadedBytes;
}
return containerId;
}
static async Task<ObjectId> PutObject(IFrostFSClient fsClient, ContainerId containerId, string fileName)
{
var fileStream = File.OpenRead(fileName);
var header = new ObjectHeader(
containerId: containerId,
type: ObjectType.Regular,
new ObjectAttribute("fileName", fileName)
);
return await fsClient.PutObjectAsync(header, fileStream);
}
static async Task<ObjectId> PutSingleObject(IFrostFSClient fsClient, ContainerId containerId, string fileName)
{
var bytes = File.ReadAllBytes(fileName);
var obj = new FrostFS.SDK.ModelsV2.Object(containerId, bytes)
.AddAttribute("Filename", fileName);
var objectId = await fsClient.PutSingleObjectAsync(obj);
return objectId;
}
static async Task GetObjectInfo(IFrostFSClient fsClient, ContainerId containerId, ObjectId objectId)
{
Console.WriteLine("--------------------------------------------------");
Console.WriteLine($"Object ID: {objectId}");
var objHeader = await fsClient.GetObjectHeadAsync(containerId, objectId);
Console.WriteLine($"size: {objHeader.PayloadLength}");
if (objHeader.Split?.SplitId != null)
Console.WriteLine($"splitID: { objHeader.Split?.SplitId}");
if (objHeader.Split?.Parent != null)
Console.WriteLine($"parent: {objHeader.Split?.Parent}");
if (objHeader.Split?.Previous != null)
Console.WriteLine($"previous: {objHeader.Split?.Previous}");
Console.WriteLine($"Attributes:");
foreach (var attribute in objHeader.Attributes)
public static async Task<ReadOnlyMemory<byte>[]> GetRangeHashAsync(FrostFsContainerId containerId, FrostFsObjectId objectId)
{
Console.WriteLine($"\t{attribute.Key} = {attribute.Value}");
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
Random rnd = new();
var salt = new byte[32];
rnd.NextBytes(salt);
var args = new PrmRangeHashGet(containerId, objectId, [new FrostFsRange(100, 64)], salt);
return await client.GetRangeHashAsync(args, default);
}
if (objHeader.Split?.Children?.FirstOrDefault() != null)
public static async Task<FrostFsObjectId> PutObjectAsync(FrostFsContainerId containerId, byte[] bytes)
{
Console.WriteLine($"Children:");
objHeader.Split?.Children.ForEach(x => Console.WriteLine(x.Value));
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmObjectPut(
new FrostFsObjectHeader(
containerId: containerId,
type: FrostFsObjectType.Regular,
[new FrostFsAttributePair("fileName", "test")]));
var stream = await client.PutObjectAsync(args, default);
await stream.WriteAsync(bytes.AsMemory());
var objectId = await stream.CompleteAsync();
return objectId;
}
}
static async Task<ObjectId?> PutObjectClientCut(IFrostFSClient fsClient, ContainerId containerId, string fileName)
{
List<ObjectId> sentObjectIds = [];
FrostFS.SDK.ModelsV2.Object? currentObject;
var partSize = 1024 * 1024;
var buffer = new byte[partSize];
var largeObject = new LargeObject(containerId);
var split = new Split();
var fileInfo = new FileInfo(fileName);
var fullLength = (ulong)fileInfo.Length;
var fileNameAttribute = new ObjectAttribute("fileName", fileInfo.Name);
using var stream = File.OpenRead(fileName);
while (true)
public static async Task<FrostFsObjectId> PutSingleObjectAsync(FrostFsObject obj)
{
var bytesCount = await stream.ReadAsync(buffer.AsMemory(0, partSize));
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
split.Previous = sentObjectIds.LastOrDefault();
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
largeObject.AppendBlock(buffer, bytesCount);
var args = new PrmSingleObjectPut(obj);
currentObject = new FrostFS.SDK.ModelsV2.Object(containerId, bytesCount < partSize ? buffer.Take(bytesCount).ToArray() : buffer)
.AddAttribute(fileNameAttribute)
.AddAttribute("type", $"part_{sentObjectIds.Count}")
.SetSplit(split);
if (largeObject.PayloadLength == fullLength)
break;
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
sentObjectIds.Add(objectId);
return await client.PutSingleObjectAsync(args, default);
}
if (sentObjectIds.Any())
public static async Task<FrostFsObjectId> PatchObjectAsync(FrostFsContainerId containerId, FrostFsObjectId objectId, byte[] patch)
{
largeObject.CalculateHash()
.AddAttribute("type", $"ParentObject")
.AddAttribute(fileNameAttribute);
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
currentObject.SetParent(largeObject);
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var objectId = await fsClient.PutSingleObjectAsync(currentObject);
sentObjectIds.Add(objectId);
var range = new FrostFsRange(64, (ulong)patch.Length);
var linkObject = new LinkObject(containerId, split.SplitId, largeObject)
.AddChildren(sentObjectIds)
.AddAttribute(fileNameAttribute)
.AddAttribute("type", $"LinkedObject");
var args = new PrmObjectPatch(
new FrostFsAddress(containerId, objectId),
payload: new MemoryStream(patch),
maxChunkLength: 256,
range: range);
_ = await fsClient.PutSingleObjectAsync(linkObject);
//return fsClient.CalculateObjectId(largeObject.Header);
return currentObject.GetParentId();
return await client.PatchObjectAsync(args, default);
}
return await fsClient.PutSingleObjectAsync(currentObject);
public static async Task DeleteObjectAsync(FrostFsContainerId containerId, FrostFsObjectId objectId, byte[] patch)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var args = new PrmObjectDelete(containerId, objectId);
await client.DeleteObjectAsync(args, default);
}
public static async Task<ICollection<FrostFsObjectId>> SearchObjectsAsync(FrostFsContainerId containerId)
{
var clientOptions = Options.Create(new ClientSettings { Key = Key, Host = Host });
Func<string, ChannelBase> grpcChannel = (url) => GrpcChannel.ForAddress(new Uri(url));
var client = FrostFSClient.GetInstance(clientOptions, grpcChannel);
var filter = new FilterByAttributePair(FrostFsMatchType.Equals, "fileName", "test");
var args = new PrmObjectSearch(containerId, null, [], filter);
Collection<FrostFsObjectId> objects = [];
await foreach (var objId in client.SearchObjectsAsync(args, default))
{
objects.Add(objId);
}
return objects;
}
#endregion
}
static void ShowFileInExplorer()
{
Directory.SetCurrentDirectory(@"C:\Users\p.gross\Pictures\samples");
var currentDir = Directory.GetCurrentDirectory();
Process.Start("explorer.exe", currentDir);
}
static string GetBroFileName(string path)
{
var newName = Path.GetFileNameWithoutExtension(path) + "_bro" + Path.GetExtension(path);
return newName;
}

View file

@ -9,16 +9,9 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="FrostFS.SDK.ClientV2.dll">
<HintPath>netstandard2.0\FrostFS.SDK.ClientV2.dll</HintPath>
</Reference>
<Reference Include="FrostFS.SDK.ModelsV2.dll">
<HintPath>netstandard2.0\FrostFS.SDK.ModelsV2.dll</HintPath>
</Reference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageReference Include="System.Memory" Version="4.5.5" />
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageReference Include="BouncyCastle.Cryptography" Version="2.4.0" />
@ -26,4 +19,13 @@
<PackageReference Include="Grpc.Net.Client" Version="2.62.0" />
</ItemGroup>
<ItemGroup>
<Reference Include="FrostFS.SDK.Client">
<HintPath>..\frostfs-sdk-csharp\src\FrostFS.SDK.Client\bin\Debug\netstandard2.0\FrostFS.SDK.Client.dll</HintPath>
</Reference>
<Reference Include="FrostFS.SDK.Protos">
<HintPath>..\frostfs-sdk-csharp\src\FrostFS.SDK.Client\bin\Debug\netstandard2.0\FrostFS.SDK.Protos.dll</HintPath>
</Reference>
</ItemGroup>
</Project>