diff --git a/checkstyle.xml b/checkstyle.xml index ebe929e..1bb24e7 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -76,7 +76,7 @@ - + diff --git a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java index 8865173..fd8ef2c 100644 --- a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java +++ b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java @@ -5,6 +5,10 @@ public class CryptoConst { public static final int RFC6979_SIGNATURE_SIZE = 64; public static final int HASH_SIGNATURE_SIZE = 65; + public static final int MURMUR_MULTIPLIER = 33; + public static final long LANDAU_PRIME_DIVISOR_64BIT = 0xc4ceb9fe1a85ec53L; + public static final long LANDAU_PRIME_DIVISOR_65BIT = 0xff51afd7ed558ccdL; + private CryptoConst() { } } diff --git a/client/src/main/java/info/frostfs/sdk/placement/Context.java b/client/src/main/java/info/frostfs/sdk/placement/Context.java new file mode 100644 index 0000000..64fcb28 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Context.java @@ -0,0 +1,369 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.exceptions.FrostFSException; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; +import static info.frostfs.sdk.constants.ErrorConst.*; + +@Getter +@Setter +public final class Context { + public static final String MAIN_FILTER_NAME = "*"; + public static final String LIKE_WILDCARD = "*"; + + // network map to operate on + private final NetmapSnapshot netMap; + + // cache of processed filters + private final Map processedFilters = new HashMap<>(); + + // cache of processed selectors + private final Map processedSelectors = new HashMap<>(); + + // stores results of selector processing + private final Map>> selections = new HashMap<>(); + + // cache of parsed numeric values + private final Map numCache = new HashMap<>(); + private final Map usedNodes = new HashMap<>(); + private final Function weightFunc; + private byte[] hrwSeed; + private long hrwSeedHash; + private int cbf; + private boolean strict; + + public Context(NetmapSnapshot netMap) { + this.netMap = netMap; + this.weightFunc = Tools.defaultWeightFunc(netMap.getNodeInfoCollection()); + } + + private static Pair calcNodesCount(Selector selector) { + return selector.getClause() == SelectorClause.SAME + ? new ImmutablePair<>(1, selector.getCount()) + : new ImmutablePair<>(selector.getCount(), 1); + } + + private static double calcBucketWeight(List ns, MeanIQRAgg a, Function wf) { + for (NodeInfo node : ns) { + a.add(wf.apply(node)); + } + return a.compute(); + } + + public void processFilters(PlacementPolicy policy) { + for (Filter filter : policy.getFilters()) { + processFilter(filter, true); + } + } + + private void processFilter(Filter filter, boolean top) { + String filterName = filter.getName(); + if (MAIN_FILTER_NAME.equals(filterName)) { + throw new FrostFSException(String.format(INVALID_FILTER_NAME_TEMPLATE, MAIN_FILTER_NAME)); + } + + if (top && (filterName == null || filterName.isEmpty())) { + throw new FrostFSException(UNNAMED_TOP_FILTER); + } + + if (!top && filterName != null && !filterName.isEmpty() && !processedFilters.containsKey(filterName)) { + throw new FrostFSException(FILTER_NOT_FOUND); + } + + if (filter.getOperation() == FilterOperation.AND || + filter.getOperation() == FilterOperation.OR || + filter.getOperation() == FilterOperation.NOT) { + + for (Filter f : filter.getFilters()) { + processFilter(f, false); + } + } else { + if (filter.getFilters().length != 0) { + throw new FrostFSException(NON_EMPTY_FILTERS); + } else if (!top && filterName != null && !filterName.isEmpty()) { + // named reference + return; + } + + switch (filter.getOperation()) { + case EQ: + case NE: + case LIKE: + break; + case GT: + case GE: + case LT: + case LE: + long n = Long.parseLong(filter.getValue()); + numCache.put(filter.getValue(), n); + break; + default: + throw new FrostFSException(String.format(INVALID_FILTER_OPERATION_TEMPLATE, filter.getOperation())); + } + } + + if (top) { + processedFilters.put(filterName, filter); + } + } + + public void processSelectors(PlacementPolicy policy) { + for (Selector selector : policy.getSelectors()) { + String filterName = selector.getFilter(); + if (!MAIN_FILTER_NAME.equals(filterName)) { + if (selector.getFilter() == null || !processedFilters.containsKey(selector.getFilter())) { + throw new FrostFSException(String.format(FILTER_NOT_FOUND_TEMPLATE, filterName)); + } + } + + processedSelectors.put(selector.getName(), selector); + List> selection = getSelection(selector); + selections.put(selector.getName(), selection); + } + } + + private NodeAttributePair[] getSelectionBase(Selector selector) { + String fName = selector.getFilter(); + if (fName == null) { + throw new FrostFSException(FILTER_NAME_IS_EMPTY); + } + + Filter f = processedFilters.get(fName); + boolean isMain = MAIN_FILTER_NAME.equals(fName); + List result = new ArrayList<>(); + + Map> nodeMap = new HashMap<>(); + String attr = selector.getAttribute(); + + for (NodeInfo node : netMap.getNodeInfoCollection()) { + if (usedNodes.containsKey(node.getHash())) { + continue; + } + + if (isMain || match(f, node)) { + if (attr == null) { + result.add(new NodeAttributePair("", new NodeInfo[]{node})); + } else { + String v = node.getAttributes().get(attr); + List nodes = nodeMap.computeIfAbsent(v, k -> new ArrayList<>()); + nodes.add(node); + } + } + } + + if (attr != null && !attr.isEmpty()) { + for (Map.Entry> entry : nodeMap.entrySet()) { + result.add(new NodeAttributePair(entry.getKey(), entry.getValue().toArray(NodeInfo[]::new))); + } + } + + if (hrwSeed != null && hrwSeed.length != 0) { + NodeAttributePair[] sortedNodes = new NodeAttributePair[result.size()]; + + for (int i = 0; i < result.size(); i++) { + double[] ws = new double[result.get(i).getNodes().length]; + NodeAttributePair res = result.get(i); + Tools.appendWeightsTo(res.getNodes(), weightFunc, ws); + sortedNodes[i] = new NodeAttributePair( + res.getAttr(), + Tools.sortHasherSliceByWeightValue(Arrays.asList(res.getNodes()), ws, hrwSeedHash) + .toArray(NodeInfo[]::new) + ); + } + + return sortedNodes; + } + return result.toArray(new NodeAttributePair[0]); + } + + public List> getSelection(Selector s) { + Pair counts = calcNodesCount(s); + int bucketCount = counts.getKey(); + int nodesInBucket = counts.getValue(); + + NodeAttributePair[] buckets = getSelectionBase(s); + + if (strict && buckets.length < bucketCount) { + throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); + } + + if (hrwSeed == null || hrwSeed.length == 0) { + if (s.getAttribute() == null || s.getAttribute().isEmpty()) { + Arrays.sort(buckets, Comparator.comparing(b -> b.getNodes()[0].getHash())); + } else { + Arrays.sort(buckets, Comparator.comparing(NodeAttributePair::getAttr)); + } + } + + int maxNodesInBucket = nodesInBucket * cbf; + + List> res = new ArrayList<>(buckets.length); + List> fallback = new ArrayList<>(buckets.length); + + for (NodeAttributePair bucket : buckets) { + List ns = Arrays.asList(bucket.getNodes()); + if (ns.size() >= maxNodesInBucket) { + res.add(new ArrayList<>(ns.subList(0, maxNodesInBucket))); + } else if (ns.size() >= nodesInBucket) { + fallback.add(new ArrayList<>(ns)); + } + } + + if (res.size() < bucketCount) { + res.addAll(fallback); + + if (strict && res.size() < bucketCount) { + throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); + } + } + + if (hrwSeed != null && hrwSeed.length != 0) { + double[] weights = new double[res.size()]; + var a = new MeanIQRAgg(); + + for (int i = 0; i < res.size(); i++) { + a.clear(); + weights[i] = calcBucketWeight(res.get(i), a, weightFunc); + } + + List hashers = res.stream() + .map(HasherList::new) + .collect(Collectors.toList()); + + hashers = Tools.sortHasherSliceByWeightValue(hashers, weights, hrwSeedHash); + + for (int i = 0; i < res.size(); i++) { + res.set(i, hashers.get(i).getNodes()); + } + } + + if (res.size() < bucketCount) { + if (strict && res.isEmpty()) { + throw new FrostFSException(NOT_ENOUGH_NODES); + } + bucketCount = res.size(); + } + + if (s.getAttribute() == null || s.getAttribute().isEmpty()) { + fallback = res.subList(bucketCount, res.size()); + res = new ArrayList<>(res.subList(0, bucketCount)); + + for (int i = 0; i < fallback.size(); i++) { + int index = i % bucketCount; + if (res.get(index).size() >= maxNodesInBucket) { + break; + } + res.get(index).addAll(fallback.get(i)); + } + } + + return res.subList(0, bucketCount); + } + + private boolean matchKeyValue(Filter f, NodeInfo nodeInfo) { + switch (f.getOperation()) { + case EQ: + return nodeInfo.getAttributes().containsKey(f.getKey()) && + nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + case LIKE: + boolean hasPrefix = f.getValue().startsWith(LIKE_WILDCARD); + boolean hasSuffix = f.getValue().endsWith(LIKE_WILDCARD); + + int start = hasPrefix ? LIKE_WILDCARD.length() : 0; + int end = hasSuffix ? f.getValue().length() - LIKE_WILDCARD.length() : f.getValue().length(); + String str = f.getValue().substring(start, end); + + if (hasPrefix && hasSuffix) { + return nodeInfo.getAttributes().get(f.getKey()).contains(str); + } + if (hasPrefix) { + return nodeInfo.getAttributes().get(f.getKey()).endsWith(str); + } + if (hasSuffix) { + return nodeInfo.getAttributes().get(f.getKey()).startsWith(str); + } + return nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + case NE: + return !nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + default: + long attr; + switch (f.getKey()) { + case ATTRIBUTE_PRICE: + attr = nodeInfo.getPrice().longValue(); + break; + case ATTRIBUTE_CAPACITY: + attr = nodeInfo.getCapacity().longValue(); + break; + default: + try { + attr = Long.parseLong(nodeInfo.getAttributes().get(f.getKey())); + } catch (NumberFormatException e) { + return false; + } + break; + } + + switch (f.getOperation()) { + case GT: + return attr > numCache.get(f.getValue()); + case GE: + return attr >= numCache.get(f.getValue()); + case LT: + return attr < numCache.get(f.getValue()); + case LE: + return attr <= numCache.get(f.getValue()); + default: + break; + } + break; + } + return false; + } + + boolean match(Filter f, NodeInfo nodeInfo) { + if (f == null) { + return false; + } + + switch (f.getOperation()) { + case NOT: + Filter[] inner = f.getFilters(); + Filter fSub = inner[0]; + + if (inner[0].getName() != null && !inner[0].getName().isEmpty()) { + fSub = processedFilters.get(inner[0].getName()); + } + return !match(fSub, nodeInfo); + case AND: + case OR: + for (int i = 0; i < f.getFilters().length; i++) { + Filter currentFilter = f.getFilters()[i]; + + if (currentFilter.getName() != null && !currentFilter.getName().isEmpty()) { + currentFilter = processedFilters.get(currentFilter.getName()); + } + + boolean ok = match(currentFilter, nodeInfo); + + if (ok == (f.getOperation() == FilterOperation.OR)) { + return ok; + } + } + return f.getOperation() == FilterOperation.AND; + default: + return matchKeyValue(f, nodeInfo); + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/HasherList.java b/client/src/main/java/info/frostfs/sdk/placement/HasherList.java new file mode 100644 index 0000000..ee5a7ce --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/HasherList.java @@ -0,0 +1,20 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Hasher; +import info.frostfs.sdk.dto.netmap.NodeInfo; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +@Getter +@AllArgsConstructor +public final class HasherList implements Hasher { + private final List nodes; + + @Override + public long getHash() { + return CollectionUtils.isNotEmpty(nodes) ? nodes.get(0).getHash() : 0L; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java new file mode 100644 index 0000000..d3464d4 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java @@ -0,0 +1,18 @@ +package info.frostfs.sdk.placement; + +import java.math.BigInteger; + +public class MeanAgg { + private double mean; + private int count; + + public void add(BigInteger n) { + int c = count + 1; + mean = mean * count / c + n.doubleValue() / c; + count++; + } + + public double compute() { + return mean; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java new file mode 100644 index 0000000..b61b25e --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java @@ -0,0 +1,57 @@ +package info.frostfs.sdk.placement; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class MeanIQRAgg { + private static final int MIN_LN = 4; + + private final List arr = new ArrayList<>(); + + public MeanIQRAgg() { + } + + public void add(double d) { + arr.add(d); + } + + public double compute() { + int length = arr.size(); + if (length == 0) { + return 0; + } + + List sorted = new ArrayList<>(arr); + Collections.sort(sorted); + + double minV, maxV; + + if (length < MIN_LN) { + minV = sorted.get(0); + maxV = sorted.get(length - 1); + } else { + int start = length / MIN_LN; + int end = length * 3 / MIN_LN - 1; + + minV = sorted.get(start); + maxV = sorted.get(end); + } + + int count = 0; + double sum = 0; + + for (var e : sorted) { + if (e >= minV && e <= maxV) { + sum += e; + count++; + } + } + + return count == 0 ? 0 : sum / count; + } + + public void clear() { + arr.clear(); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java new file mode 100644 index 0000000..e2855b5 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java @@ -0,0 +1,24 @@ +package info.frostfs.sdk.placement; + +import java.math.BigInteger; + +public class MinAgg { + private double min; + private boolean minFound; + + public void add(BigInteger n) { + if (!minFound) { + min = n.doubleValue(); + minFound = true; + return; + } + + if (n.doubleValue() < min) { + min = n.doubleValue(); + } + } + + public double compute() { + return min; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java b/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java new file mode 100644 index 0000000..9fd1660 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.NodeInfo; +import lombok.Getter; + +@Getter +public class NodeAttributePair { + private final String attr; + private final NodeInfo[] nodes; + + NodeAttributePair(String attr, NodeInfo[] nodes) { + this.attr = attr; + this.nodes = nodes; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java b/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java new file mode 100644 index 0000000..587150d --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java @@ -0,0 +1,5 @@ +package info.frostfs.sdk.placement; + +public interface Normalizer { + double normalize(double w); +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java b/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java new file mode 100644 index 0000000..f20c75f --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java @@ -0,0 +1,197 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.exceptions.FrostFSException; +import lombok.AllArgsConstructor; +import org.apache.commons.codec.digest.MurmurHash3; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import static info.frostfs.sdk.constants.ErrorConst.SELECTOR_NOT_FOUND_TEMPLATE; +import static info.frostfs.sdk.constants.ErrorConst.VECTORS_IS_NULL; + +@AllArgsConstructor +public final class PlacementVector { + private final NetmapSnapshot netmapSnapshot; + + private static NodeInfo[] flattenNodes(List> nodes) { + int size = nodes.stream().mapToInt(List::size).sum(); + NodeInfo[] result = new NodeInfo[size]; + + int i = 0; + for (List ns : nodes) { + for (NodeInfo n : ns) { + result[i++] = n; + } + } + return result; + } + + /* + * PlacementVectors sorts container nodes returned by ContainerNodes method + * and returns placement vectors for the entity identified by the given pivot. + * For example, in order to build node list to store the object, + * binary-encoded object identifier can be used as pivot. + * Result is deterministic for the fixed NetMap and parameters. + * */ + public NodeInfo[][] placementVectors(NodeInfo[][] vectors, byte[] pivot) { + if (vectors == null) { + throw new FrostFSException(VECTORS_IS_NULL); + } + + long hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; + + Function wf = Tools.defaultWeightFunc(netmapSnapshot.getNodeInfoCollection()); + + NodeInfo[][] result = new NodeInfo[vectors.length][]; + int maxSize = Arrays.stream(vectors) + .mapToInt(v -> v.length) + .max() + .orElse(0); + + double[] spanWeights = new double[maxSize]; + + for (int i = 0; i < vectors.length; i++) { + result[i] = Arrays.copyOf(vectors[i], vectors[i].length); + + Tools.appendWeightsTo(result[i], wf, spanWeights); + + List sorted = Tools.sortHasherSliceByWeightValue( + Arrays.asList(result[i]), + spanWeights, + hash + ); + result[i] = sorted.toArray(new NodeInfo[0]); + } + + return result; + } + + /* + * SelectFilterNodes returns a two-dimensional list of nodes as a result of applying the given + * SelectFilterExpr to the NetMap. If the SelectFilterExpr contains only filters, the result contains + * a single row with the result of the last filter application. If the SelectFilterExpr contains only selectors, + * the result contains the selection rows of the last select application. + * */ + public List> selectFilterNodes(SelectFilterExpr expr) { + PlacementPolicy policy = new PlacementPolicy( + null, + false, + expr.getCbf(), + expr.getFilters().toArray(Filter[]::new), + new Selector[]{expr.getSelector()} + ); + + Context ctx = new Context(netmapSnapshot); + ctx.setCbf(expr.getCbf()); + + ctx.processFilters(policy); + ctx.processSelectors(policy); + + List> ret = new ArrayList<>(); + + if (expr.getSelector() == null) { + Filter lastFilter = expr.getFilters().get(expr.getFilters().size() - 1); + List subCollection = new ArrayList<>(); + ret.add(subCollection); + + for (NodeInfo nodeInfo : netmapSnapshot.getNodeInfoCollection()) { + if (ctx.match(ctx.getProcessedFilters().get(lastFilter.getName()), nodeInfo)) { + subCollection.add(nodeInfo); + } + } + } else if (expr.getSelector().getName() != null) { + List> sel = ctx.getSelection( + ctx.getProcessedSelectors().get(expr.getSelector().getName()) + ); + + for (List ns : sel) { + List subCollection = new ArrayList<>(ns); + ret.add(subCollection); + } + } + + return ret; + } + + /* + * ContainerNodes returns two-dimensional list of nodes as a result of applying given PlacementPolicy to the NetMap. + * Each line of the list corresponds to a replica descriptor. + * Line order corresponds to order of ReplicaDescriptor list in the policy. + * Nodes are pre-filtered according to the Filter list from the policy, and then selected by Selector list. + * Result is deterministic for the fixed NetMap and parameters. + * + * Result can be used in PlacementVectors. + * */ + public NodeInfo[][] containerNodes(PlacementPolicy p, byte[] pivot) { + Context c = new Context(netmapSnapshot); + c.setCbf(p.getBackupFactory() == 0 ? 3 : p.getBackupFactory()); + + if (pivot != null && pivot.length > 0) { + c.setHrwSeed(pivot); + + var hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; + c.setHrwSeedHash(hash); + } + + c.processFilters(p); + c.processSelectors(p); + + boolean unique = p.isUnique(); + + List> result = new ArrayList<>(p.getReplicas().length); + for (int i = 0; i < p.getReplicas().length; i++) { + result.add(new ArrayList<>()); + } + + for (int i = 0; i < p.getReplicas().length; i++) { + String sName = p.getReplicas()[i].getSelector(); + + if ((sName == null || sName.isEmpty()) && + !(p.getReplicas().length == 1 && p.getSelectors().length == 1)) { + + Selector s = new Selector( + "", p.getReplicas()[i].getCountNodes(), null, null, + Context.MAIN_FILTER_NAME + ); + + List> nodes = c.getSelection(s); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + + if (unique) { + for (NodeInfo n : result.get(i)) { + c.getUsedNodes().put(n.getHash(), true); + } + } + continue; + } + + if (unique) { + Selector s = c.getProcessedSelectors().get(sName); + if (s == null) { + throw new FrostFSException(String.format(SELECTOR_NOT_FOUND_TEMPLATE, sName)); + } + + List> nodes = c.getSelection(s); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + + for (NodeInfo n : result.get(i)) { + c.getUsedNodes().put(n.getHash(), true); + } + } else { + List> nodes = c.getSelections().get(sName); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + } + } + + NodeInfo[][] collection = new NodeInfo[result.size()][]; + for (int i = 0; i < result.size(); i++) { + collection[i] = result.get(i).toArray(new NodeInfo[0]); + } + + return collection; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java b/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java new file mode 100644 index 0000000..1bcc03a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java @@ -0,0 +1,14 @@ +package info.frostfs.sdk.placement; + +public class ReverseMinNorm implements Normalizer { + private final double min; + + public ReverseMinNorm(double min) { + this.min = min; + } + + @Override + public double normalize(double w) { + return (min + 1) / (w + 1); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java b/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java new file mode 100644 index 0000000..b7dd898 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java @@ -0,0 +1,16 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Filter; +import info.frostfs.sdk.dto.netmap.Selector; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.List; + +@Getter +@AllArgsConstructor +public class SelectFilterExpr { + private final int cbf; + private final Selector selector; + private final List filters; +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java b/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java new file mode 100644 index 0000000..c0e867a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java @@ -0,0 +1,19 @@ +package info.frostfs.sdk.placement; + +public class SigmoidNorm implements Normalizer { + private final double scale; + + public SigmoidNorm(double scale) { + this.scale = scale; + } + + @Override + public double normalize(double w) { + if (scale == 0) { + return 0; + } + + double x = w / scale; + return x / (1 + x); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Tools.java b/client/src/main/java/info/frostfs/sdk/placement/Tools.java new file mode 100644 index 0000000..0140078 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Tools.java @@ -0,0 +1,123 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Hasher; +import info.frostfs.sdk.dto.netmap.NodeInfo; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; +import static info.frostfs.sdk.constants.CryptoConst.*; + +public final class Tools { + private Tools() { + } + + public static long distance(long x, long y) { + long acc = x ^ y; + acc ^= acc >>> MURMUR_MULTIPLIER; + acc *= LANDAU_PRIME_DIVISOR_65BIT; + acc ^= acc >>> MURMUR_MULTIPLIER; + acc *= LANDAU_PRIME_DIVISOR_64BIT; + acc ^= acc >>> MURMUR_MULTIPLIER; + return acc; + } + + public static void appendWeightsTo(NodeInfo[] nodes, Function wf, double[] weights) { + if (weights.length < nodes.length) { + weights = new double[nodes.length]; + } + for (int i = 0; i < nodes.length; i++) { + weights[i] = wf.apply(nodes[i]); + } + } + + public static List sortHasherSliceByWeightValue(List nodes, double[] weights, long hash) { + if (nodes.isEmpty()) { + return nodes; + } + + boolean allEquals = true; + if (weights.length > 1) { + for (int i = 1; i < weights.length; i++) { + if (weights[i] != weights[0]) { + allEquals = false; + break; + } + } + } + + Double[] dist = new Double[nodes.size()]; + + if (allEquals) { + for (int i = 0; i < dist.length; i++) { + long x = nodes.get(i).getHash(); + dist[i] = toUnsignedBigInteger(distance(x, hash)).doubleValue(); + } + return sortHasherByDistance(nodes, dist, true); + } + + for (int i = 0; i < dist.length; i++) { + var reverse = UNSIGNED_LONG_MASK.subtract(toUnsignedBigInteger(distance(nodes.get(i).getHash(), hash))); + dist[i] = reverse.doubleValue() * weights[i]; + } + + return sortHasherByDistance(nodes, dist, false); + } + + public static > List sortHasherByDistance( + List nodes, N[] dist, boolean asc + ) { + IndexedValue[] indexes = new IndexedValue[nodes.size()]; + for (int i = 0; i < dist.length; i++) { + indexes[i] = new IndexedValue<>(nodes.get(i), dist[i]); + } + + if (asc) { + Arrays.sort(indexes, Comparator.comparing(iv -> iv.dist)); + } else { + Arrays.sort(indexes, (iv1, iv2) -> iv2.dist.compareTo(iv1.dist)); + } + + List result = new ArrayList<>(); + for (IndexedValue iv : indexes) { + result.add(iv.nodeInfo); + } + return result; + } + + public static Function defaultWeightFunc(List nodes) { + MeanAgg mean = new MeanAgg(); + MinAgg minV = new MinAgg(); + + for (NodeInfo node : nodes) { + mean.add(node.getCapacity()); + minV.add(node.getPrice()); + } + + return newWeightFunc(new SigmoidNorm(mean.compute()), new ReverseMinNorm(minV.compute())); + } + + private static BigInteger toUnsignedBigInteger(long i) { + return i >= 0 ? BigInteger.valueOf(i) : BigInteger.valueOf(i).and(UNSIGNED_LONG_MASK); + } + + private static Function newWeightFunc(Normalizer capNorm, Normalizer priceNorm) { + return nodeInfo -> capNorm.normalize(nodeInfo.getCapacity().doubleValue()) + * priceNorm.normalize(nodeInfo.getPrice().doubleValue()); + } + + private static class IndexedValue { + final T nodeInfo; + final N dist; + + IndexedValue(T nodeInfo, N dist) { + this.nodeInfo = nodeInfo; + this.dist = dist; + } + } +} diff --git a/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java b/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java new file mode 100644 index 0000000..8ae8d48 --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java @@ -0,0 +1,238 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.netmap.NodeState; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import lombok.Getter; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +public class PlacementVectorTest { + private static final Yaml YAML = new Yaml(); + + private static void compareNodes(Map attrs, NodeInfo nodeInfo) { + assertEquals(attrs.size(), nodeInfo.getAttributes().size()); + assertEquals( + attrs.entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()), + nodeInfo.getAttributes().entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()) + ); + } + + @SneakyThrows + @Test + public void placementTest() { + Path resourceDirYaml = Paths.get(Objects.requireNonNull(getClass().getClassLoader() + .getResource("placement")).toURI()); + + List yamlFiles; + try (Stream paths = Files.walk(resourceDirYaml)) { + yamlFiles = paths.filter(Files::isRegularFile).collect(Collectors.toList()); + } + + Version v = new Version(2, 13); + String[] addresses = {"localhost", "server1"}; + + for (Path file : yamlFiles) { + TestCase testCase = YAML.loadAs(Files.newInputStream(file), TestCase.class); + + assertNotNull(testCase); + assertNotNull(testCase.nodes); + assertTrue(testCase.nodes.length > 0); + + List nodes = Arrays.stream(testCase.nodes) + .map(n -> new NodeInfo( + n.state, + v, + List.of(addresses), + n.attributes != null ? + Arrays.stream(n.attributes) + .collect(Collectors.toMap(KeyValuePair::getKey, KeyValuePair::getValue)) : + Collections.emptyMap(), + n.getPublicKeyBytes() + )) + .collect(Collectors.toList()); + + NetmapSnapshot netmap = new NetmapSnapshot(100L, nodes); + + assertNotNull(testCase.tests); + + for (var entry : testCase.tests.entrySet()) { + var test = entry.getValue(); + PlacementPolicy policy = new PlacementPolicy( + test.policy.replicas != null ? + Arrays.stream(test.policy.replicas) + .map(r -> new Replica(r.count, r.selector)) + .toArray(Replica[]::new) : + new Replica[0], + test.policy.unique, + test.policy.containerBackupFactor, + test.policy.filters != null + ? Arrays.stream(test.policy.filters) + .map(FilterDto::getFilter) + .toArray(Filter[]::new) + : new Filter[]{}, + test.policy.selectors != null + ? Arrays.stream(test.policy.selectors) + .map(SelectorDto::getSelector) + .toArray(Selector[]::new) + : new Selector[]{} + ); + + try { + var vector = new PlacementVector(netmap); + NodeInfo[][] result = vector.containerNodes(policy, test.getPivotBytes()); + + if (test.result == null) { + if (test.error != null && !test.error.isEmpty()) { + fail("Error is expected but has not been thrown"); + } else { + assertNotNull(test.policy.replicas); + assertEquals(result.length, test.policy.replicas.length); + + for (NodeInfo[] nodesArr : result) { + assertEquals(0, nodesArr.length); + } + } + } else { + assertEquals(test.result.length, result.length); + + for (int i = 0; i < test.result.length; i++) { + assertEquals(test.result[i].length, result[i].length); + for (int j = 0; j < test.result[i].length; j++) { + compareNodes(nodes.get(test.result[i][j]).getAttributes(), result[i][j]); + } + } + + if (test.placement != null + && test.placement.result != null + && test.placement.getPivotBytes() != null) { + NodeInfo[][] placementResult = vector.placementVectors( + result, test.placement.getPivotBytes() + ); + + assertEquals(test.placement.result.length, placementResult.length); + + for (int i = 0; i < placementResult.length; i++) { + assertEquals(test.placement.result[i].length, placementResult[i].length); + for (int j = 0; j < placementResult[i].length; j++) { + compareNodes( + nodes.get(test.placement.result[i][j]).getAttributes(), + placementResult[i][j] + ); + } + } + } + } + } catch (Exception ex) { + if (test.error != null && !test.error.isEmpty()) { + assertTrue(ex.getMessage().contains(test.error)); + } else { + throw ex; + } + } + } + } + } + + + public static class TestCase { + public String name; + public String comment; + public Node[] nodes; + public Map tests; + } + + public static class Node { + public KeyValuePair[] attributes; + public String publicKey; + public String[] addresses; + public NodeState state = NodeState.ONLINE; + + public byte[] getPublicKeyBytes() { + return publicKey == null || publicKey.isEmpty() ? new byte[0] : Base64.getDecoder().decode(publicKey); + } + } + + @Getter + public static class KeyValuePair { + public String key; + public String value; + } + + public static class TestData { + public PolicyDto policy; + public String pivot; + public int[][] result; + public String error; + public ResultData placement; + + public byte[] getPivotBytes() { + return pivot == null ? null : Base64.getDecoder().decode(pivot); + } + } + + public static class PolicyDto { + public boolean unique; + public int containerBackupFactor; + public FilterDto[] filters; + public ReplicaDto[] replicas; + public SelectorDto[] selectors; + } + + public static class SelectorDto { + public int count; + public String name; + public SelectorClause clause; + public String attribute; + public String filter; + + public Selector getSelector() { + return new Selector(name != null ? name : "", count, clause, attribute, filter); + } + } + + public static class FilterDto { + public String name; + public String key; + public FilterOperation op; + public String value; + public FilterDto[] filters; + + public Filter getFilter() { + return new Filter( + name != null ? name : "", + key != null ? key : "", + op, + value != null ? value : "", + filters != null + ? Arrays.stream(filters).map(FilterDto::getFilter).toArray(Filter[]::new) + : new Filter[0] + ); + } + } + + public static class ReplicaDto { + public int count; + public String selector; + } + + public static class ResultData { + public String pivot; + public int[][] result; + + public byte[] getPivotBytes() { + return pivot == null ? null : Base64.getDecoder().decode(pivot); + } + } +} diff --git a/client/src/test/resources/placement/cbf_default.yml b/client/src/test/resources/placement/cbf_default.yml new file mode 100644 index 0000000..c43a703 --- /dev/null +++ b/client/src/test/resources/placement/cbf_default.yml @@ -0,0 +1,48 @@ +name: default CBF is 3 +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: St.Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: DE + - key: City + value: Berlin + - attributes: + - key: Location + value: Europe + - key: Country + value: FR + - key: City + value: Paris +tests: + set default CBF: + policy: + replicas: + - count: 1 + selector: EU + containerBackupFactor: 0 + selectors: + - name: EU + count: 1 + clause: SAME + attribute: Location + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/client/src/test/resources/placement/cbf_minimal.yml b/client/src/test/resources/placement/cbf_minimal.yml new file mode 100644 index 0000000..2fe2642 --- /dev/null +++ b/client/src/test/resources/placement/cbf_minimal.yml @@ -0,0 +1,52 @@ +name: Real node count multiplier is in range [1, specified CBF] +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: DE + - attributes: + - key: ID + value: '2' + - key: Country + value: DE + - attributes: + - key: ID + value: '3' + - key: Country + value: DE +tests: + select 2, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + select 3, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 3 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/client/src/test/resources/placement/cbf_requirements.yml b/client/src/test/resources/placement/cbf_requirements.yml new file mode 100644 index 0000000..ccd58d4 --- /dev/null +++ b/client/src/test/resources/placement/cbf_requirements.yml @@ -0,0 +1,82 @@ +name: CBF requirements +nodes: + - attributes: + - key: ID + value: '1' + - key: Attr + value: Same + - attributes: + - key: ID + value: '2' + - key: Attr + value: Same + - attributes: + - key: ID + value: '3' + - key: Attr + value: Same + - attributes: + - key: ID + value: '4' + - key: Attr + value: Same +tests: + default CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + explicit CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 3 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select distinct, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select same, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Attr + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/filter_complex.yml b/client/src/test/resources/placement/filter_complex.yml new file mode 100644 index 0000000..1abc46a --- /dev/null +++ b/client/src/test/resources/placement/filter_complex.yml @@ -0,0 +1,207 @@ +name: compound filter +nodes: + - attributes: + - key: Storage + value: SSD + - key: Rating + value: '10' + - key: IntField + value: '100' + - key: Param + value: Value1 +tests: + good: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - op: OR + filters: + - key: Param + op: EQ + value: Value1 + filters: [] + - key: Param + op: EQ + value: Value2 + filters: [] + result: + - - 0 + bad storage type: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: HDD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + bad rating: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '15' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + bad param: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value0 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] diff --git a/client/src/test/resources/placement/filter_invalid_integer.yml b/client/src/test/resources/placement/filter_invalid_integer.yml new file mode 100644 index 0000000..6674246 --- /dev/null +++ b/client/src/test/resources/placement/filter_invalid_integer.yml @@ -0,0 +1,43 @@ +name: invalid integer field +nodes: + - attributes: + - key: IntegerField + value: 'true' + - attributes: + - key: IntegerField + value: str +tests: + empty string is not casted to 0: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: LE + value: '8' + filters: [] + non-empty string is not casted to a number: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: GE + value: '0' + filters: [] diff --git a/client/src/test/resources/placement/filter_simple.yml b/client/src/test/resources/placement/filter_simple.yml new file mode 100644 index 0000000..7fdd84a --- /dev/null +++ b/client/src/test/resources/placement/filter_simple.yml @@ -0,0 +1,224 @@ +name: single-op filters +nodes: + - attributes: + - key: Rating + value: '4' + - key: Country + value: Germany +tests: + GE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '4' + filters: [] + result: + - - 0 + GE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '5' + filters: [] + GT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '3' + filters: [] + result: + - - 0 + GT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '4' + filters: [] + LE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '4' + filters: [] + result: + - - 0 + LE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '3' + filters: [] + LT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '5' + filters: [] + result: + - - 0 + LT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '4' + filters: [] + EQ true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: Germany + filters: [] + result: + - - 0 + EQ false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: China + filters: [] + NE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: France + filters: [] + result: + - - 0 + NE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: Germany + filters: [] diff --git a/client/src/test/resources/placement/hrw_sort.yml b/client/src/test/resources/placement/hrw_sort.yml new file mode 100644 index 0000000..c84f7c9 --- /dev/null +++ b/client/src/test/resources/placement/hrw_sort.yml @@ -0,0 +1,118 @@ +name: HRW ordering +nodes: + - attributes: + - key: Country + value: Germany + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Germany + - key: Price + value: '4' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '3' + - key: Capacity + value: '10' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '1' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: France + - key: Price + value: '100' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '7' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '1' +tests: + select 3 nodes in 3 distinct countries, same placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 1 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 0 + - 7 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 0 + - 7 + select 6 nodes in 3 distinct countries, different placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 4 + - 0 + - 1 + - 7 + - 2 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 4 + - 0 + - 7 + - 2 + - 1 diff --git a/client/src/test/resources/placement/issue213.yml b/client/src/test/resources/placement/issue213.yml new file mode 100644 index 0000000..8e8aea4 --- /dev/null +++ b/client/src/test/resources/placement/issue213.yml @@ -0,0 +1,52 @@ +name: unnamed selector (nspcc-dev/neofs-api-go#213) +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Saint-Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: Sweden + - key: City + value: Stockholm + - attributes: + - key: Location + value: Europe + - key: Country + value: Finalnd + - key: City + value: Helsinki +tests: + test: + policy: + replicas: + - count: 4 + containerBackupFactor: 1 + selectors: + - name: '' + count: 4 + clause: DISTINCT + filter: LOC_EU + filters: + - name: LOC_EU + key: Location + op: EQ + value: Europe + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/many_selects.yml b/client/src/test/resources/placement/many_selects.yml new file mode 100644 index 0000000..29efd43 --- /dev/null +++ b/client/src/test/resources/placement/many_selects.yml @@ -0,0 +1,141 @@ +name: single-op filters +nodes: + - attributes: + - key: Country + value: Russia + - key: Rating + value: '1' + - key: City + value: SPB + - attributes: + - key: Country + value: Germany + - key: Rating + value: '5' + - key: City + value: Berlin + - attributes: + - key: Country + value: Russia + - key: Rating + value: '6' + - key: City + value: Moscow + - attributes: + - key: Country + value: France + - key: Rating + value: '4' + - key: City + value: Paris + - attributes: + - key: Country + value: France + - key: Rating + value: '1' + - key: City + value: Lyon + - attributes: + - key: Country + value: Russia + - key: Rating + value: '5' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '7' + - key: City + value: Moscow + - attributes: + - key: Country + value: Germany + - key: Rating + value: '3' + - key: City + value: Darmstadt + - attributes: + - key: Country + value: Germany + - key: Rating + value: '7' + - key: City + value: Frankfurt + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB +tests: + Select: + policy: + replicas: + - count: 1 + selector: SameRU + - count: 1 + selector: DistinctRU + - count: 1 + selector: Good + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: SameRU + count: 2 + clause: SAME + attribute: City + filter: FromRU + - name: DistinctRU + count: 2 + clause: DISTINCT + attribute: City + filter: FromRU + - name: Good + count: 2 + clause: DISTINCT + attribute: Country + filter: Good + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + - name: Good + key: Rating + op: GE + value: '4' + result: + - - 0 + - 5 + - 9 + - 10 + - - 2 + - 6 + - 0 + - 5 + - - 1 + - 8 + - 2 + - 5 + - - 3 + - 4 + - 1 + - 7 + - 0 + - 2 diff --git a/client/src/test/resources/placement/multiple_rep.yml b/client/src/test/resources/placement/multiple_rep.yml new file mode 100644 index 0000000..448214f --- /dev/null +++ b/client/src/test/resources/placement/multiple_rep.yml @@ -0,0 +1,46 @@ +name: multiple replicas (#215) +nodes: + - attributes: + - key: City + value: Saint-Petersburg + - attributes: + - key: City + value: Moscow + - attributes: + - key: City + value: Berlin + - attributes: + - key: City + value: Paris +tests: + test: + policy: + replicas: + - count: 1 + selector: LOC_SPB_PLACE + - count: 1 + selector: LOC_MSK_PLACE + containerBackupFactor: 1 + selectors: + - name: LOC_SPB_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_SPB + - name: LOC_MSK_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_MSK + filters: + - name: LOC_SPB + key: City + op: EQ + value: Saint-Petersburg + filters: [] + - name: LOC_MSK + key: City + op: EQ + value: Moscow + filters: [] + result: + - - 0 + - - 1 diff --git a/client/src/test/resources/placement/multiple_rep_asymmetric.yml b/client/src/test/resources/placement/multiple_rep_asymmetric.yml new file mode 100644 index 0000000..61f8f76 --- /dev/null +++ b/client/src/test/resources/placement/multiple_rep_asymmetric.yml @@ -0,0 +1,162 @@ +name: multiple REP, asymmetric +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '5' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '6' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '7' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '8' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '9' + - key: Continent + value: SA + - key: City + value: Lima + - attributes: + - key: ID + value: '10' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '11' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '12' + - key: Continent + value: NA + - key: City + value: LosAngeles + - attributes: + - key: ID + value: '13' + - key: Continent + value: SA + - key: City + value: Lima +tests: + test: + policy: + replicas: + - count: 1 + selector: SPB + - count: 2 + selector: Americas + containerBackupFactor: 2 + selectors: + - name: SPB + count: 1 + clause: SAME + attribute: City + filter: SPBSSD + - name: Americas + count: 2 + clause: DISTINCT + attribute: City + filter: Americas + filters: + - name: SPBSSD + op: AND + filters: + - name: '' + key: Country + op: EQ + value: RU + filters: [] + - name: '' + key: City + op: EQ + value: St.Petersburg + filters: [] + - name: '' + key: SSD + op: EQ + value: '1' + filters: [] + - name: Americas + op: OR + filters: + - name: '' + key: Continent + op: EQ + value: NA + filters: [] + - name: '' + key: Continent + op: EQ + value: SA + filters: [] + result: + - - 1 + - 4 + - - 8 + - 12 + - 5 + - 10 diff --git a/client/src/test/resources/placement/non_strict.yml b/client/src/test/resources/placement/non_strict.yml new file mode 100644 index 0000000..a01986d --- /dev/null +++ b/client/src/test/resources/placement/non_strict.yml @@ -0,0 +1,52 @@ +name: non-strict selections +comment: These test specify loose selection behaviour, to allow fetching already PUT + objects even when there is not enough nodes to select from. +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + not enough nodes (backup factor): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 + not enough nodes (buckets): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 diff --git a/client/src/test/resources/placement/rep_only.yml b/client/src/test/resources/placement/rep_only.yml new file mode 100644 index 0000000..b354591 --- /dev/null +++ b/client/src/test/resources/placement/rep_only.yml @@ -0,0 +1,62 @@ +name: REP X +nodes: + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Saint-Petersburg + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Moscow + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Berlin + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Paris + state: UNSPECIFIED +tests: + REP 1: + policy: + replicas: + - count: 1 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + REP 3: + policy: + replicas: + - count: 3 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 3 + - 1 + - 2 + REP 5: + policy: + replicas: + - count: 5 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/select_no_attribute.yml b/client/src/test/resources/placement/select_no_attribute.yml new file mode 100644 index 0000000..02046f3 --- /dev/null +++ b/client/src/test/resources/placement/select_no_attribute.yml @@ -0,0 +1,56 @@ +name: select with unspecified attribute +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' +tests: + test: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 1 + selectors: + - name: X + count: 4 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/selector_invalid.yml b/client/src/test/resources/placement/selector_invalid.yml new file mode 100644 index 0000000..9b0a539 --- /dev/null +++ b/client/src/test/resources/placement/selector_invalid.yml @@ -0,0 +1,47 @@ +name: invalid selections +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + missing filter: + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 1 + clause: DISTINCT + attribute: Country + filter: FromNL + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + error: filter not found + not enough nodes (filter results in empty set): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromMoon + filters: + - name: FromMoon + key: Country + op: EQ + value: Moon + filters: [] diff --git a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java index 00c1711..8c248c9 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java +++ b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java @@ -59,6 +59,18 @@ public class ErrorConst { public static final String STRING_IS_TOO_BIG_TEMPLATE = "string size is too big=%s"; public static final String STRING_SIZE_IS_INVALID_TEMPLATE = "invalid string size=%s"; + public static final String FILTER_NAME_IS_EMPTY = "Filter name for selector is empty"; + public static final String INVALID_FILTER_NAME_TEMPLATE = "filter name is invalid: '%s' is reserved"; + public static final String INVALID_FILTER_OPERATION_TEMPLATE = "invalid filter operation: %s"; + public static final String FILTER_NOT_FOUND = "filter not found"; + public static final String FILTER_NOT_FOUND_TEMPLATE = "filter not found: SELECT FROM '%s'"; + public static final String NON_EMPTY_FILTERS = "simple filter contains sub-filters"; + public static final String NOT_ENOUGH_NODES = "not enough nodes"; + public static final String NOT_ENOUGH_NODES_TEMPLATE = "not enough nodes to SELECT from '%s'"; + public static final String UNNAMED_TOP_FILTER = "unnamed top-level filter"; + public static final String VECTORS_IS_NULL = "vectors cannot be null"; + public static final String SELECTOR_NOT_FOUND_TEMPLATE = "selector not found: %s"; + private ErrorConst() { } } diff --git a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java index feb1ca5..58b0041 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java @@ -1,5 +1,7 @@ package info.frostfs.sdk.constants; +import java.math.BigInteger; + public class AppConst { public static final String RESERVED_PREFIX = "__SYSTEM__"; @@ -15,6 +17,8 @@ public class AppConst { public static final int DEFAULT_GRPC_TIMEOUT = 5; public static final long DEFAULT_POLL_INTERVAL = 10; + public static final BigInteger UNSIGNED_LONG_MASK = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + private AppConst() { } } diff --git a/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java index 214a9e3..964b3d4 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java @@ -5,6 +5,17 @@ import static info.frostfs.sdk.constants.AppConst.RESERVED_PREFIX; public class AttributeConst { public static final String DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE = RESERVED_PREFIX + "DISABLE_HOMOMORPHIC_HASHING"; + /* + * ATTRIBUTE_PRICE is a key to the node attribute that indicates + * the price in GAS tokens for storing one GB of data during one Epoch. + * */ + public static final String ATTRIBUTE_PRICE = "Price"; + + /* + * ATTRIBUTE_CAPACITY is a key to the node attribute that indicates the total available disk space in Gigabytes. + * */ + public static final String ATTRIBUTE_CAPACITY = "Capacity"; + private AttributeConst() { } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java index fe809fe..a957dce 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.FilterOperation; +import info.frostfs.sdk.enums.netmap.FilterOperation; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java new file mode 100644 index 0000000..286f77f --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java @@ -0,0 +1,5 @@ +package info.frostfs.sdk.dto.netmap; + +public interface Hasher { + long getHash(); +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java index a3d28f8..a46cf2a 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java @@ -1,20 +1,31 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import lombok.Getter; +import org.apache.commons.codec.digest.MurmurHash3; +import java.math.BigInteger; import java.util.Collections; import java.util.List; import java.util.Map; +import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; +import static java.util.Objects.isNull; + @Getter -public class NodeInfo { +public class NodeInfo implements Hasher { private final NodeState state; private final Version version; private final List addresses; private final Map attributes; private final byte[] publicKey; + private long hash; + private BigInteger price = UNSIGNED_LONG_MASK; + + public NodeInfo(NodeState state, Version version, List addresses, Map attributes, byte[] publicKey) { this.state = state; @@ -23,4 +34,26 @@ public class NodeInfo { this.attributes = Collections.unmodifiableMap(attributes); this.publicKey = publicKey; } + + public long getHash() { + if (hash == 0) { + hash = MurmurHash3.hash128x64(publicKey, 0, publicKey.length, 0)[0]; + } + + return hash; + } + + public BigInteger getCapacity() { + var capacity = attributes.get(ATTRIBUTE_CAPACITY); + return isNull(capacity) ? BigInteger.valueOf(0) : new BigInteger(capacity); + } + + public BigInteger getPrice() { + if (price.equals(UNSIGNED_LONG_MASK)) { + var priceString = attributes.get(ATTRIBUTE_PRICE); + price = isNull(priceString) ? BigInteger.valueOf(0) : new BigInteger(priceString); + } + + return price; + } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java index 5c28462..09d059d 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java @@ -13,6 +13,14 @@ import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; public class Replica { private final int count; private final String selector; + private long ecDataCount; + private long ecParityCount; + + public Replica(int count, String selector, int ecDataCount, int ecParityCount) { + this(count, selector); + this.ecDataCount = Integer.toUnsignedLong(ecDataCount); + this.ecParityCount = Integer.toUnsignedLong(ecParityCount); + } public Replica(int count, String selector) { if (count <= 0) { @@ -32,8 +40,11 @@ public class Replica { ); } - this.count = count; this.selector = EMPTY_STRING; } + + public int getCountNodes() { + return count != 0 ? count : (int) (ecDataCount + ecParityCount); + } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java index 71197a1..4dee15c 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java @@ -1,15 +1,17 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.SelectorClause; +import info.frostfs.sdk.enums.netmap.SelectorClause; import lombok.AllArgsConstructor; import lombok.Getter; +import lombok.Setter; @Getter +@Setter @AllArgsConstructor public class Selector { private final String name; - private final int count; - private final SelectorClause clause; - private final String attribute; - private final String filter; + private int count; + private SelectorClause clause; + private String attribute; + private String filter; } diff --git a/models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java similarity index 95% rename from models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java rename to models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java index f49b0f2..69513c8 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums; +package info.frostfs.sdk.enums.netmap; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/enums/NodeState.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java similarity index 94% rename from models/src/main/java/info/frostfs/sdk/enums/NodeState.java rename to models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java index 3f833e3..ac6ae78 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/NodeState.java +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums; +package info.frostfs.sdk.enums.netmap; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java similarity index 94% rename from models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java rename to models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java index b10ff0c..f375f88 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums; +package info.frostfs.sdk.enums.netmap; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java index 3f63b3b..529b26d 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Filter; -import info.frostfs.sdk.enums.FilterOperation; +import info.frostfs.sdk.enums.netmap.FilterOperation; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.collections4.CollectionUtils; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java index f319583..c28cd35 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java @@ -4,7 +4,7 @@ import frostfs.netmap.Service; import frostfs.netmap.Types; import frostfs.netmap.Types.NodeInfo.Attribute; import info.frostfs.sdk.dto.netmap.NodeInfo; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import java.util.stream.Collectors; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java index 959b9d6..3ac0105 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java @@ -36,15 +36,17 @@ public class ReplicaMapper { return Types.Replica.newBuilder() .setCount(replica.getCount()) .setSelector(replica.getSelector()) + .setEcDataCount((int) replica.getEcDataCount()) + .setEcParityCount((int) replica.getEcParityCount()) .build(); } - public static Replica[] toModels(List filters) { - if (CollectionUtils.isEmpty(filters)) { + public static Replica[] toModels(List replicas) { + if (CollectionUtils.isEmpty(replicas)) { return null; } - return filters.stream().map(ReplicaMapper::toModel).toArray(Replica[]::new); + return replicas.stream().map(ReplicaMapper::toModel).toArray(Replica[]::new); } public static Replica toModel(Types.Replica replica) { @@ -52,6 +54,11 @@ public class ReplicaMapper { return null; } - return new Replica(replica.getCount(), replica.getSelector()); + return new Replica( + replica.getCount(), + replica.getSelector(), + replica.getEcDataCount(), + replica.getEcParityCount() + ); } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java index 8b9067d..462519a 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Selector; -import info.frostfs.sdk.enums.SelectorClause; +import info.frostfs.sdk.enums.netmap.SelectorClause; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.collections4.CollectionUtils; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java index 4b1649d..788d71e 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Filter; -import info.frostfs.sdk.enums.FilterOperation; +import info.frostfs.sdk.enums.netmap.FilterOperation; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.junit.jupiter.api.Test; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java index e2b6c29..dd2dc1f 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java @@ -3,7 +3,7 @@ package info.frostfs.sdk.mappers.netmap; import com.google.protobuf.ByteString; import frostfs.netmap.Service; import frostfs.netmap.Types; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java index 4e55ae0..6d46553 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Selector; -import info.frostfs.sdk.enums.SelectorClause; +import info.frostfs.sdk.enums.netmap.SelectorClause; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.junit.jupiter.api.Test; diff --git a/pom.xml b/pom.xml index 978fea2..40f8790 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ - 0.10.0 + 0.11.0 11 11 @@ -42,6 +42,11 @@ commons-lang3 3.14.0 + + commons-codec + commons-codec + 1.18.0 + org.projectlombok lombok @@ -78,6 +83,11 @@ ${mockito.version} test + + org.yaml + snakeyaml + 2.4 +