[#29] Client: Add PlacementVector unit tests
Signed-off-by: Pavel Gross <p.gross@yadro.com>
This commit is contained in:
parent
568bdc67e8
commit
43e300c773
33 changed files with 3054 additions and 234 deletions
|
|
@ -1,161 +1,275 @@
|
|||
using System.Collections.ObjectModel;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
using FrostFS.SDK.Client.Models.Netmap.Placement;
|
||||
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace FrostFS.SDK.Tests.Unit;
|
||||
|
||||
[SuppressMessage("Reliability", "CA2007:Consider calling ConfigureAwait on the awaited task", Justification = "Default Value is correct for tests")]
|
||||
public class PlacementVectorTests
|
||||
public class PlacementVectorTests(ITestOutputHelper testOutputHelper)
|
||||
{
|
||||
Dictionary<string, string>[] attribs;
|
||||
|
||||
public PlacementVectorTests()
|
||||
private static readonly JsonSerializerOptions serializeOptions = new()
|
||||
{
|
||||
var attribs1 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Germany" },
|
||||
{"Price", "2" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
PropertyNameCaseInsensitive = true
|
||||
};
|
||||
|
||||
var attribs2 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Germany" },
|
||||
{"Price", "4" },
|
||||
{"Capacity", "1"}
|
||||
};
|
||||
|
||||
var attribs3 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "France" },
|
||||
{"Price", "3" },
|
||||
{"Capacity", "10"}
|
||||
};
|
||||
|
||||
var attribs4 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Russia" },
|
||||
{"Price", "2" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
|
||||
var attribs5 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Russia" },
|
||||
{"Price", "1" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
|
||||
var attribs6 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Russia" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
var attribs7 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "France" },
|
||||
{"Price", "100" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
var attribs8 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "France" },
|
||||
{"Price", "7" },
|
||||
{"Capacity", "10000"}
|
||||
};
|
||||
var attribs9 = new Dictionary<string, string>
|
||||
{
|
||||
{"Country", "Russia" },
|
||||
{"Price", "2" },
|
||||
{"Capacity", "1"}
|
||||
};
|
||||
|
||||
attribs = [attribs1, attribs2, attribs3, attribs4, attribs5, attribs6, attribs7, attribs8, attribs9];
|
||||
}
|
||||
private readonly ITestOutputHelper _testOutputHelper = testOutputHelper;
|
||||
|
||||
[Fact]
|
||||
public void PlacementVectorTest()
|
||||
public void PlacementTest()
|
||||
{
|
||||
var path = ".\\..\\..\\..\\TestData\\PlacementTests";
|
||||
Assert.True(Directory.Exists(path));
|
||||
|
||||
var files = Directory.GetFiles(path);
|
||||
|
||||
FrostFsVersion v = new(2, 13);
|
||||
var addresses = new string[] { "localhost", "server1" };
|
||||
var key1 = new byte[] { 1 };
|
||||
var key2 = new byte[] { 2 };
|
||||
var key3 = new byte[] { 3 };
|
||||
|
||||
var nodes = new List<FrostFsNodeInfo>
|
||||
foreach (var file in files.Where(f => f.EndsWith(".json", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[5].AsReadOnly(), key1),
|
||||
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[0].AsReadOnly(), key2),
|
||||
new(v, NodeState.Online, addresses.AsReadOnly(), attribs[8].AsReadOnly(), key3)
|
||||
};
|
||||
//if (!file.EndsWith("selector_invalid.json"))
|
||||
// continue;
|
||||
|
||||
var netmap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly());
|
||||
var fileName = file[(file.LastIndexOf("..\\") + 3)..];
|
||||
_testOutputHelper.WriteLine($"Open file {fileName}");
|
||||
|
||||
var arg = new FrostFsNodeInfo[1][];
|
||||
var pivot = "objectID"u8.ToArray();
|
||||
var str = File.ReadAllText(file);
|
||||
Assert.False(string.IsNullOrEmpty(str));
|
||||
|
||||
arg[0] = [.. nodes];
|
||||
var result = netmap.PlacementVectors(arg, pivot);
|
||||
var testCase = JsonSerializer.Deserialize<TestCase>(str, serializeOptions);
|
||||
|
||||
Assert.Single(result);
|
||||
Assert.Equal(3, result[0].Length);
|
||||
Assert.Equal(key1, result[0][0].PublicKey);
|
||||
Assert.Equal(key2, result[0][1].PublicKey);
|
||||
Assert.Equal(key3, result[0][2].PublicKey);
|
||||
}
|
||||
Assert.NotNull(testCase);
|
||||
Assert.NotNull(testCase.Nodes);
|
||||
Assert.True(testCase.Nodes.Length > 0);
|
||||
|
||||
[Fact]
|
||||
public void TestPlacementPolicyUnique()
|
||||
{
|
||||
FrostFsVersion version = new(2, 13);
|
||||
var p = new FrostFsPlacementPolicy(true, [new FrostFsReplica(1, "S"), new FrostFsReplica(1, "S")])
|
||||
{
|
||||
BackupFactor = 2
|
||||
};
|
||||
p.Selectors.Add(new FrostFsSelector("S")
|
||||
{
|
||||
Attribute = "City",
|
||||
Count = 1,
|
||||
Filter = "*",
|
||||
Clause = (int)FrostFsClause.Same
|
||||
});
|
||||
_testOutputHelper.WriteLine($"Test case: \"{testCase.Name}\"");
|
||||
|
||||
List<FrostFsNodeInfo> nodes = [];
|
||||
var nodes = testCase.Nodes
|
||||
.Select(n => new FrostFsNodeInfo(v,
|
||||
n.State,
|
||||
addresses.AsReadOnly(),
|
||||
n.Attributes?.ToDictionary(x => x.Key, x => x.Value) ?? [],
|
||||
n.PublicKeyBytes
|
||||
)
|
||||
)
|
||||
.ToArray()
|
||||
.AsReadOnly();
|
||||
|
||||
var cities = new string[] { "Moscow", "Berlin", "Shenzhen" };
|
||||
for (int i = 0; i < 3; i++)
|
||||
{
|
||||
for (int j = 0; j < 3; j++)
|
||||
var netmap = new FrostFsNetmapSnapshot(100, nodes);
|
||||
|
||||
Assert.NotNull(testCase.Tests);
|
||||
|
||||
foreach (var test in testCase.Tests)
|
||||
{
|
||||
var attr = new Dictionary<string, string> { { "City", cities[i] } };
|
||||
var key = new byte[] { (byte)(i * 4 + j) };
|
||||
var node = new FrostFsNodeInfo(version, NodeState.Online, [], attr, key);
|
||||
_testOutputHelper.WriteLine($"Start test \"{test.Name}\"");
|
||||
|
||||
nodes.Add(node);
|
||||
}
|
||||
}
|
||||
var policy = new FrostFsPlacementPolicy(
|
||||
test.Policy!.Unique,
|
||||
test.Policy.ContainerBackupFactor,
|
||||
new Collection<FrostFsSelector>(test.Policy.Selectors?.Select(s => s.Selector).ToList() ?? []),
|
||||
new Collection<FrostFsFilter>(test.Policy.Filters?.Select(f => f.Filter).ToList() ?? []),
|
||||
test.Policy.Replicas?.Select(r => new FrostFsReplica(r.Count, r.Selector)).ToArray() ?? []
|
||||
);
|
||||
|
||||
var netMap = new FrostFsNetmapSnapshot(100, nodes.AsReadOnly());
|
||||
|
||||
var v = netMap.ContainerNodes(p, null);
|
||||
|
||||
Assert.Equal(2, v.Length);
|
||||
Assert.Equal(2, v[0].Length);
|
||||
Assert.Equal(2, v[1].Length);
|
||||
|
||||
|
||||
for (int i = 0; i < v.Length; i++)
|
||||
{
|
||||
foreach (var ni in v[i])
|
||||
{
|
||||
for (int j = 0; j < i; j++)
|
||||
try
|
||||
{
|
||||
foreach (var nj in v[j])
|
||||
var result = netmap.ContainerNodes(policy, test.PivotBytes);
|
||||
|
||||
if (test.Result == null)
|
||||
{
|
||||
Assert.NotEqual(ni.Hash, nj.Hash);
|
||||
if (!string.IsNullOrEmpty(test.Error))
|
||||
{
|
||||
Assert.Fail("Error is expected but has not been thrown");
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.NotNull(test.Policy?.Replicas);
|
||||
Assert.Equal(result.Length, test.Policy.Replicas.Length);
|
||||
|
||||
for (int i = 0; i < result.Length; i++)
|
||||
{
|
||||
Assert.Empty(result[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.Equal(test.Result.Length, result.Length);
|
||||
|
||||
for (var i = 0; i < test.Result.Length; i++)
|
||||
{
|
||||
Assert.Equal(test.Result[i].Length, result[i].Length);
|
||||
for (var j = 0; j < test.Result[i].Length; j++)
|
||||
{
|
||||
CompareNodes(nodes[test.Result[i][j]].Attributes, result[i][j]);
|
||||
}
|
||||
}
|
||||
|
||||
if (test.Placement?.Result != null && test.Placement.PivotBytes != null)
|
||||
{
|
||||
var placementResult = netmap.PlacementVectors(result, test.Placement.PivotBytes);
|
||||
|
||||
Assert.Equal(test.Placement.Result.Length, placementResult.Length);
|
||||
|
||||
for (int i = 0; i < placementResult.Length; i++)
|
||||
{
|
||||
Assert.Equal(test.Placement.Result[i].Length, placementResult[i].Length);
|
||||
for (int j = 0; j < placementResult[i].Length; j++)
|
||||
{
|
||||
CompareNodes(nodes[test.Placement.Result[i][j]].Attributes, placementResult[i][j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (!string.IsNullOrEmpty(test.Error))
|
||||
{
|
||||
Assert.Contains(test.Error, ex.Message, StringComparison.InvariantCulture);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
_testOutputHelper.WriteLine($"Done");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static void CompareNodes(IReadOnlyDictionary<string, string> attrs, FrostFsNodeInfo nodeInfo)
|
||||
{
|
||||
Assert.Equal(attrs.Count, nodeInfo.Attributes.Count);
|
||||
Assert.True(attrs.OrderBy(k => k.Key).SequenceEqual(nodeInfo.Attributes.OrderBy(x => x.Key)));
|
||||
}
|
||||
}
|
||||
|
||||
public class TestCase
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public Node[]? Nodes { get; set; }
|
||||
|
||||
public TestData[]? Tests { get; set; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
public class Node
|
||||
{
|
||||
[JsonPropertyName("attributes")]
|
||||
public KeyValuePair<string, string>[]? Attributes { get; set; }
|
||||
|
||||
public string? PublicKey { get; set; }
|
||||
|
||||
internal byte[]? PublicKeyBytes => string.IsNullOrEmpty(PublicKey) ? [] : Convert.FromBase64String(PublicKey);
|
||||
|
||||
public string[]? Addresses { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<NodeState>))]
|
||||
public NodeState State { get; set; } = NodeState.Online;
|
||||
}
|
||||
|
||||
public class TestData
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public PolicyDto? Policy { get; set; }
|
||||
|
||||
public string? Pivot { get; set; }
|
||||
|
||||
public int[][]? Result { get; set; }
|
||||
|
||||
public string? Error { get; set; }
|
||||
|
||||
internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null;
|
||||
|
||||
public ResultData? Placement { get; set; }
|
||||
}
|
||||
|
||||
public class PolicyDto
|
||||
{
|
||||
public bool Unique { get; set; }
|
||||
|
||||
public uint ContainerBackupFactor { get; set; }
|
||||
|
||||
public FilterDto[]? Filters { get; set; }
|
||||
|
||||
public ReplicaDto[]? Replicas { get; set; }
|
||||
|
||||
public SelectorDto[]? Selectors { get; set; }
|
||||
}
|
||||
|
||||
public class SelectorDto()
|
||||
{
|
||||
public uint Count { get; set; }
|
||||
|
||||
public string? Name { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<ClauseValues>))]
|
||||
public ClauseValues Clause { get; set; }
|
||||
|
||||
public string? Attribute { get; set; }
|
||||
|
||||
public string? Filter { get; set; }
|
||||
|
||||
public FrostFsSelector Selector => new(Name ?? string.Empty)
|
||||
{
|
||||
Count = Count,
|
||||
Clause = (int)Clause,
|
||||
Filter = Filter,
|
||||
Attribute = Attribute
|
||||
};
|
||||
}
|
||||
|
||||
public class FilterDto
|
||||
{
|
||||
public string? Name { get; set; }
|
||||
|
||||
public string? Key { get; set; }
|
||||
|
||||
[JsonConverter(typeof(JsonStringEnumConverter<Operation>))]
|
||||
public Operation Op { get; set; }
|
||||
|
||||
public string? Value { get; set; }
|
||||
|
||||
public FilterDto[]? Filters { get; set; }
|
||||
|
||||
public FrostFsFilter Filter => new(
|
||||
Name ?? string.Empty,
|
||||
Key ?? string.Empty,
|
||||
(int)Op,
|
||||
Value ?? string.Empty,
|
||||
Filters != null ? Filters.Select(f => f.Filter).ToArray() : []);
|
||||
}
|
||||
|
||||
public class ReplicaDto
|
||||
{
|
||||
public int Count { get; set; }
|
||||
|
||||
public string? Selector { get; set; }
|
||||
}
|
||||
|
||||
public class ResultData
|
||||
{
|
||||
public string? Pivot { get; set; }
|
||||
|
||||
public int[][]? Result { get; set; }
|
||||
|
||||
internal byte[]? PivotBytes => Pivot != null ? Convert.FromBase64String(Pivot) : null;
|
||||
}
|
||||
|
||||
public enum ClauseValues
|
||||
{
|
||||
UNSPECIFIED = 0,
|
||||
SAME,
|
||||
DISTINCT
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue