diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/publish.yml deleted file mode 100644 index 7c067ca..0000000 --- a/.forgejo/workflows/publish.yml +++ /dev/null @@ -1,22 +0,0 @@ -on: - push: - workflow_dispatch: - -jobs: - image: - name: Publish Maven packages - runs-on: docker - container: git.frostfs.info/truecloudlab/env:openjdk-11-maven-3.8.6 - steps: - - name: Clone git repo - uses: actions/checkout@v3 - - - name: Publish release packages - run: mvn clean --batch-mode --update-snapshots deploy - if: >- - startsWith(github.ref, 'refs/tags/v') && - (github.event_name == 'workflow_dispatch' || github.event_name == 'push') - env: - MAVEN_REGISTRY: TrueCloudLab - MAVEN_REGISTRY_USER: ${{secrets.MAVEN_REGISTRY_USER}} - MAVEN_REGISTRY_PASSWORD: ${{secrets.MAVEN_REGISTRY_PASSWORD}} diff --git a/CHANGELOG.md b/CHANGELOG.md index e859051..d9e6a86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,62 +1,21 @@ # Changelog -## [0.12.0] - 2025-04-24 - -### Fixed - -- Patch logic -- Patch payload requirements - -## [0.11.0] - 2025-04-23 - -### Added - -- Placement policy vectors - -## [0.10.0] - 2025-03-10 - -### Added - -- Auto deploy to forgejo - -## [0.9.0] - 2025-03-05 - -### Added - -- APE rule deserializer - -## [0.9.0] - 2025-03-05 - -### Added - -- APE rule deserializer - -## [0.8.0] - 2025-03-04 - -### Added - -- Creating client via wallet and password - ## [0.7.0] - 2025-02-20 ### Added - - Expanding the parameters for creating a container ### Fixed - -- Creating a session for working with objects +- Creating a session for working with objects ## [0.6.0] - 2025-02-13 ### Added - - APE rules serializer ## [0.5.0] - 2025-02-11 ### Fixed - - Loading large objects in chunks - .gitignore - pom revision \ No newline at end of file diff --git a/README.md b/README.md index 991c5fa..fd1efff 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ import info.frostfs.sdk.FrostFSClient; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.netmap.PlacementPolicy; import info.frostfs.sdk.dto.netmap.Replica; +import info.frostfs.sdk.enums.BasicAcl; import info.frostfs.sdk.jdo.ClientSettings; import info.frostfs.sdk.jdo.parameters.CallContext; import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; @@ -40,7 +41,7 @@ public class ContainerExample { FrostFSClient frostFSClient = new FrostFSClient(clientSettings); // Create container - var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(3)}, true, 1); + var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(1)}, true, 0); var prmContainerCreate = new PrmContainerCreate(new Container(placementPolicy)); var containerId = frostFSClient.createContainer(prmContainerCreate, callContext); diff --git a/checkstyle.xml b/checkstyle.xml index 1bb24e7..ebe929e 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -76,7 +76,7 @@ - + diff --git a/client/src/main/java/info/frostfs/sdk/FrostFSClient.java b/client/src/main/java/info/frostfs/sdk/FrostFSClient.java index 4aa685c..0026285 100644 --- a/client/src/main/java/info/frostfs/sdk/FrostFSClient.java +++ b/client/src/main/java/info/frostfs/sdk/FrostFSClient.java @@ -1,7 +1,6 @@ package info.frostfs.sdk; import frostfs.accounting.Types; -import info.frostfs.sdk.dto.ape.Chain; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.container.ContainerId; import info.frostfs.sdk.dto.netmap.NetmapSnapshot; @@ -14,7 +13,6 @@ import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; import info.frostfs.sdk.jdo.ClientSettings; -import info.frostfs.sdk.jdo.ECDsa; import info.frostfs.sdk.jdo.NetworkSettings; import info.frostfs.sdk.jdo.parameters.CallContext; import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; @@ -42,7 +40,6 @@ import info.frostfs.sdk.utils.Validator; import io.grpc.Channel; import io.grpc.ClientInterceptors; import io.grpc.ManagedChannel; -import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -69,12 +66,10 @@ public class FrostFSClient implements CommonClient { ? clientSettings.getChannel() : initGrpcChannel(clientSettings); - var ecdsa = StringUtils.isBlank(clientSettings.getWif()) - ? new ECDsa(clientSettings.getWallet(), clientSettings.getPassword()) - : new ECDsa(clientSettings.getWif()); Channel interceptChannel = ClientInterceptors.intercept(channel, MONITORING_CLIENT_INTERCEPTOR); ClientEnvironment clientEnvironment = new ClientEnvironment( - ecdsa, interceptChannel, new Version(), this, new SessionCache(0) + clientSettings.getKey(), interceptChannel, new Version(), this, + new SessionCache(0) ); Validator.validate(clientEnvironment); @@ -198,7 +193,7 @@ public class FrostFSClient implements CommonClient { } @Override - public List listChains(PrmApeChainList args, CallContext ctx) { + public List listChains(PrmApeChainList args, CallContext ctx) { return apeManagerClient.listChains(args, ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java b/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java deleted file mode 100644 index eb3fe48..0000000 --- a/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java +++ /dev/null @@ -1,12 +0,0 @@ -package info.frostfs.sdk.annotations; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Retention(RetentionPolicy.RUNTIME) -@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) -public @interface ComplexAtLeastOneIsFilled { - AtLeastOneIsFilled[] value(); -} diff --git a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java index fd8ef2c..8865173 100644 --- a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java +++ b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java @@ -5,10 +5,6 @@ public class CryptoConst { public static final int RFC6979_SIGNATURE_SIZE = 64; public static final int HASH_SIGNATURE_SIZE = 65; - public static final int MURMUR_MULTIPLIER = 33; - public static final long LANDAU_PRIME_DIVISOR_64BIT = 0xc4ceb9fe1a85ec53L; - public static final long LANDAU_PRIME_DIVISOR_65BIT = 0xff51afd7ed558ccdL; - private CryptoConst() { } } diff --git a/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java b/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java index 322b615..78e656f 100644 --- a/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java +++ b/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java @@ -17,13 +17,10 @@ public class RuleConst { // https://github.com/neo-project/neo/blob/38218bbee5bbe8b33cd8f9453465a19381c9a547/src/Neo/IO/Helper.cs#L77 public static final int MAX_SLICE_LENGTH = 0x1000000; - public static final int MAX_VAR_INT_LENGTH = 10; - public static final int CHAIN_MARSHAL_VERSION = 0; - public static final long OFFSET127 = 0x7f; public static final long OFFSET128 = 0x80; - public static final int UNSIGNED_SERIALIZE_SIZE = 7; + public static final long UNSIGNED_SERIALIZE_SIZE = 7; private RuleConst() { } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java b/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java index 387882e..62e0967 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java @@ -36,10 +36,10 @@ public class ClientEnvironment { private SessionCache sessionCache; - public ClientEnvironment(ECDsa key, Channel channel, Version version, FrostFSClient frostFSClient, + public ClientEnvironment(String wif, Channel channel, Version version, FrostFSClient frostFSClient, SessionCache sessionCache) { - this.key = key; - this.ownerId = new OwnerId(key.getAccount().getAddress()); + this.key = new ECDsa(wif); + this.ownerId = new OwnerId(key.getPublicKeyByte()); this.version = version; this.channel = channel; this.frostFSClient = frostFSClient; @@ -47,6 +47,16 @@ public class ClientEnvironment { this.address = channel.authority(); } + public ClientEnvironment(ECDsa key, Channel channel, Version version, FrostFSClient frostFSClient, + SessionCache sessionCache) { + this.key = key; + this.ownerId = new OwnerId(key.getPublicKeyByte()); + this.version = version; + this.channel = channel; + this.frostFSClient = frostFSClient; + this.sessionCache = sessionCache; + } + public String getSessionKey() { if (StringUtils.isBlank(sessionKey)) { this.sessionKey = formCacheKey(address, getHexString(key.getPublicKeyByte())); diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java b/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java index 0bab648..3f8d2df 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java @@ -1,61 +1,37 @@ package info.frostfs.sdk.jdo; import info.frostfs.sdk.annotations.AtLeastOneIsFilled; -import info.frostfs.sdk.annotations.ComplexAtLeastOneIsFilled; +import info.frostfs.sdk.annotations.NotNull; import io.grpc.ChannelCredentials; import io.grpc.ManagedChannel; import lombok.Getter; import lombok.experimental.FieldNameConstants; -import java.io.File; - @Getter @FieldNameConstants -@ComplexAtLeastOneIsFilled(value = { - @AtLeastOneIsFilled(fields = {ClientSettings.Fields.host, ClientSettings.Fields.channel}), - @AtLeastOneIsFilled(fields = {ClientSettings.Fields.wif, ClientSettings.Fields.wallet}), -}) +@AtLeastOneIsFilled(fields = {ClientSettings.Fields.host, ClientSettings.Fields.channel}) public class ClientSettings { - private String wif; - private File wallet; - private String password; + @NotNull + private final String key; + private String host; private ChannelCredentials credentials; private ManagedChannel channel; - public ClientSettings(String wif, String host) { - this.wif = wif; + public ClientSettings(String key, String host) { + this.key = key; this.host = host; } - public ClientSettings(String wif, String host, ChannelCredentials credentials) { - this.wif = wif; + public ClientSettings(String key, String host, ChannelCredentials credentials) { + this.key = key; this.host = host; this.credentials = credentials; } - public ClientSettings(String wif, ManagedChannel channel) { - this.wif = wif; - this.channel = channel; - } - - public ClientSettings(File wallet, String password, String host) { - this.wallet = wallet; - this.password = password; - this.host = host; - } - - public ClientSettings(File wallet, String password, String host, ChannelCredentials credentials) { - this.wallet = wallet; - this.password = password; - this.host = host; - this.credentials = credentials; - } - - public ClientSettings(File wallet, String password, ManagedChannel channel) { - this.wallet = wallet; - this.password = password; + public ClientSettings(String key, ManagedChannel channel) { + this.key = key; this.channel = channel; } } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java b/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java index dcd5b0a..b7e4686 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java @@ -1,25 +1,14 @@ package info.frostfs.sdk.jdo; import info.frostfs.sdk.annotations.NotNull; -import info.frostfs.sdk.exceptions.FrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import io.neow3j.wallet.Account; -import io.neow3j.wallet.nep6.NEP6Account; -import io.neow3j.wallet.nep6.NEP6Wallet; import lombok.Getter; import org.apache.commons.lang3.StringUtils; -import java.io.File; -import java.io.FileInputStream; import java.security.PrivateKey; -import java.util.Optional; -import static info.frostfs.sdk.KeyExtension.loadPrivateKey; -import static info.frostfs.sdk.constants.ErrorConst.WALLET_IS_INVALID; +import static info.frostfs.sdk.KeyExtension.*; import static info.frostfs.sdk.constants.ErrorConst.WIF_IS_INVALID; -import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; -import static io.neow3j.wallet.Wallet.OBJECT_MAPPER; -import static java.util.Objects.isNull; @Getter public class ECDsa { @@ -33,41 +22,13 @@ public class ECDsa { @NotNull private final PrivateKey privateKey; - @NotNull - private final Account account; - public ECDsa(String wif) { if (StringUtils.isEmpty(wif)) { throw new ValidationFrostFSException(WIF_IS_INVALID); } - this.account = Account.fromWIF(wif); - this.privateKeyByte = account.getECKeyPair().getPrivateKey().getBytes(); - this.publicKeyByte = account.getECKeyPair().getPublicKey().getEncoded(true); + this.privateKeyByte = getPrivateKeyFromWIF(wif); + this.publicKeyByte = loadPublicKey(privateKeyByte); this.privateKey = loadPrivateKey(privateKeyByte); } - - public ECDsa(File walletFile, String password) { - if (isNull(walletFile)) { - throw new ValidationFrostFSException(WALLET_IS_INVALID); - } - - try (var walletStream = new FileInputStream(walletFile)) { - NEP6Wallet nep6Wallet = OBJECT_MAPPER.readValue(walletStream, NEP6Wallet.class); - Optional defaultAccount = nep6Wallet.getAccounts().stream() - .filter(NEP6Account::getDefault) - .findFirst(); - - var account = defaultAccount.map(Account::fromNEP6Account) - .orElseGet(() -> Account.fromNEP6Account(nep6Wallet.getAccounts().get(0))); - account.decryptPrivateKey(isNull(password) ? EMPTY_STRING : password); - - this.account = account; - this.privateKeyByte = account.getECKeyPair().getPrivateKey().getBytes(); - this.publicKeyByte = account.getECKeyPair().getPublicKey().getEncoded(true); - this.privateKey = loadPrivateKey(privateKeyByte); - } catch (Exception exp) { - throw new FrostFSException(exp.getMessage()); - } - } } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java index 58849df..9f93c9a 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java @@ -6,7 +6,9 @@ import info.frostfs.sdk.dto.object.patch.Address; import info.frostfs.sdk.dto.object.patch.Range; import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.jdo.parameters.session.SessionContext; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; import java.io.InputStream; import java.util.List; @@ -19,8 +21,12 @@ public class PrmObjectPatch implements SessionContext { @NotNull private Address address; + @NotNull private Range range; + + @NotNull private InputStream payload; + private List newAttributes; private boolean replaceAttributes; private int maxChunkLength; @@ -33,10 +39,4 @@ public class PrmObjectPatch implements SessionContext { this.payload = payload; this.maxChunkLength = maxChunkLength; } - - public PrmObjectPatch(Address address, List newAttributes, boolean replaceAttributes) { - this.address = address; - this.newAttributes = newAttributes; - this.replaceAttributes = replaceAttributes; - } } diff --git a/client/src/main/java/info/frostfs/sdk/placement/Context.java b/client/src/main/java/info/frostfs/sdk/placement/Context.java deleted file mode 100644 index 64fcb28..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/Context.java +++ /dev/null @@ -1,369 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.*; -import info.frostfs.sdk.enums.netmap.FilterOperation; -import info.frostfs.sdk.enums.netmap.SelectorClause; -import info.frostfs.sdk.exceptions.FrostFSException; -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.commons.lang3.tuple.Pair; - -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; -import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; -import static info.frostfs.sdk.constants.ErrorConst.*; - -@Getter -@Setter -public final class Context { - public static final String MAIN_FILTER_NAME = "*"; - public static final String LIKE_WILDCARD = "*"; - - // network map to operate on - private final NetmapSnapshot netMap; - - // cache of processed filters - private final Map processedFilters = new HashMap<>(); - - // cache of processed selectors - private final Map processedSelectors = new HashMap<>(); - - // stores results of selector processing - private final Map>> selections = new HashMap<>(); - - // cache of parsed numeric values - private final Map numCache = new HashMap<>(); - private final Map usedNodes = new HashMap<>(); - private final Function weightFunc; - private byte[] hrwSeed; - private long hrwSeedHash; - private int cbf; - private boolean strict; - - public Context(NetmapSnapshot netMap) { - this.netMap = netMap; - this.weightFunc = Tools.defaultWeightFunc(netMap.getNodeInfoCollection()); - } - - private static Pair calcNodesCount(Selector selector) { - return selector.getClause() == SelectorClause.SAME - ? new ImmutablePair<>(1, selector.getCount()) - : new ImmutablePair<>(selector.getCount(), 1); - } - - private static double calcBucketWeight(List ns, MeanIQRAgg a, Function wf) { - for (NodeInfo node : ns) { - a.add(wf.apply(node)); - } - return a.compute(); - } - - public void processFilters(PlacementPolicy policy) { - for (Filter filter : policy.getFilters()) { - processFilter(filter, true); - } - } - - private void processFilter(Filter filter, boolean top) { - String filterName = filter.getName(); - if (MAIN_FILTER_NAME.equals(filterName)) { - throw new FrostFSException(String.format(INVALID_FILTER_NAME_TEMPLATE, MAIN_FILTER_NAME)); - } - - if (top && (filterName == null || filterName.isEmpty())) { - throw new FrostFSException(UNNAMED_TOP_FILTER); - } - - if (!top && filterName != null && !filterName.isEmpty() && !processedFilters.containsKey(filterName)) { - throw new FrostFSException(FILTER_NOT_FOUND); - } - - if (filter.getOperation() == FilterOperation.AND || - filter.getOperation() == FilterOperation.OR || - filter.getOperation() == FilterOperation.NOT) { - - for (Filter f : filter.getFilters()) { - processFilter(f, false); - } - } else { - if (filter.getFilters().length != 0) { - throw new FrostFSException(NON_EMPTY_FILTERS); - } else if (!top && filterName != null && !filterName.isEmpty()) { - // named reference - return; - } - - switch (filter.getOperation()) { - case EQ: - case NE: - case LIKE: - break; - case GT: - case GE: - case LT: - case LE: - long n = Long.parseLong(filter.getValue()); - numCache.put(filter.getValue(), n); - break; - default: - throw new FrostFSException(String.format(INVALID_FILTER_OPERATION_TEMPLATE, filter.getOperation())); - } - } - - if (top) { - processedFilters.put(filterName, filter); - } - } - - public void processSelectors(PlacementPolicy policy) { - for (Selector selector : policy.getSelectors()) { - String filterName = selector.getFilter(); - if (!MAIN_FILTER_NAME.equals(filterName)) { - if (selector.getFilter() == null || !processedFilters.containsKey(selector.getFilter())) { - throw new FrostFSException(String.format(FILTER_NOT_FOUND_TEMPLATE, filterName)); - } - } - - processedSelectors.put(selector.getName(), selector); - List> selection = getSelection(selector); - selections.put(selector.getName(), selection); - } - } - - private NodeAttributePair[] getSelectionBase(Selector selector) { - String fName = selector.getFilter(); - if (fName == null) { - throw new FrostFSException(FILTER_NAME_IS_EMPTY); - } - - Filter f = processedFilters.get(fName); - boolean isMain = MAIN_FILTER_NAME.equals(fName); - List result = new ArrayList<>(); - - Map> nodeMap = new HashMap<>(); - String attr = selector.getAttribute(); - - for (NodeInfo node : netMap.getNodeInfoCollection()) { - if (usedNodes.containsKey(node.getHash())) { - continue; - } - - if (isMain || match(f, node)) { - if (attr == null) { - result.add(new NodeAttributePair("", new NodeInfo[]{node})); - } else { - String v = node.getAttributes().get(attr); - List nodes = nodeMap.computeIfAbsent(v, k -> new ArrayList<>()); - nodes.add(node); - } - } - } - - if (attr != null && !attr.isEmpty()) { - for (Map.Entry> entry : nodeMap.entrySet()) { - result.add(new NodeAttributePair(entry.getKey(), entry.getValue().toArray(NodeInfo[]::new))); - } - } - - if (hrwSeed != null && hrwSeed.length != 0) { - NodeAttributePair[] sortedNodes = new NodeAttributePair[result.size()]; - - for (int i = 0; i < result.size(); i++) { - double[] ws = new double[result.get(i).getNodes().length]; - NodeAttributePair res = result.get(i); - Tools.appendWeightsTo(res.getNodes(), weightFunc, ws); - sortedNodes[i] = new NodeAttributePair( - res.getAttr(), - Tools.sortHasherSliceByWeightValue(Arrays.asList(res.getNodes()), ws, hrwSeedHash) - .toArray(NodeInfo[]::new) - ); - } - - return sortedNodes; - } - return result.toArray(new NodeAttributePair[0]); - } - - public List> getSelection(Selector s) { - Pair counts = calcNodesCount(s); - int bucketCount = counts.getKey(); - int nodesInBucket = counts.getValue(); - - NodeAttributePair[] buckets = getSelectionBase(s); - - if (strict && buckets.length < bucketCount) { - throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); - } - - if (hrwSeed == null || hrwSeed.length == 0) { - if (s.getAttribute() == null || s.getAttribute().isEmpty()) { - Arrays.sort(buckets, Comparator.comparing(b -> b.getNodes()[0].getHash())); - } else { - Arrays.sort(buckets, Comparator.comparing(NodeAttributePair::getAttr)); - } - } - - int maxNodesInBucket = nodesInBucket * cbf; - - List> res = new ArrayList<>(buckets.length); - List> fallback = new ArrayList<>(buckets.length); - - for (NodeAttributePair bucket : buckets) { - List ns = Arrays.asList(bucket.getNodes()); - if (ns.size() >= maxNodesInBucket) { - res.add(new ArrayList<>(ns.subList(0, maxNodesInBucket))); - } else if (ns.size() >= nodesInBucket) { - fallback.add(new ArrayList<>(ns)); - } - } - - if (res.size() < bucketCount) { - res.addAll(fallback); - - if (strict && res.size() < bucketCount) { - throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); - } - } - - if (hrwSeed != null && hrwSeed.length != 0) { - double[] weights = new double[res.size()]; - var a = new MeanIQRAgg(); - - for (int i = 0; i < res.size(); i++) { - a.clear(); - weights[i] = calcBucketWeight(res.get(i), a, weightFunc); - } - - List hashers = res.stream() - .map(HasherList::new) - .collect(Collectors.toList()); - - hashers = Tools.sortHasherSliceByWeightValue(hashers, weights, hrwSeedHash); - - for (int i = 0; i < res.size(); i++) { - res.set(i, hashers.get(i).getNodes()); - } - } - - if (res.size() < bucketCount) { - if (strict && res.isEmpty()) { - throw new FrostFSException(NOT_ENOUGH_NODES); - } - bucketCount = res.size(); - } - - if (s.getAttribute() == null || s.getAttribute().isEmpty()) { - fallback = res.subList(bucketCount, res.size()); - res = new ArrayList<>(res.subList(0, bucketCount)); - - for (int i = 0; i < fallback.size(); i++) { - int index = i % bucketCount; - if (res.get(index).size() >= maxNodesInBucket) { - break; - } - res.get(index).addAll(fallback.get(i)); - } - } - - return res.subList(0, bucketCount); - } - - private boolean matchKeyValue(Filter f, NodeInfo nodeInfo) { - switch (f.getOperation()) { - case EQ: - return nodeInfo.getAttributes().containsKey(f.getKey()) && - nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); - case LIKE: - boolean hasPrefix = f.getValue().startsWith(LIKE_WILDCARD); - boolean hasSuffix = f.getValue().endsWith(LIKE_WILDCARD); - - int start = hasPrefix ? LIKE_WILDCARD.length() : 0; - int end = hasSuffix ? f.getValue().length() - LIKE_WILDCARD.length() : f.getValue().length(); - String str = f.getValue().substring(start, end); - - if (hasPrefix && hasSuffix) { - return nodeInfo.getAttributes().get(f.getKey()).contains(str); - } - if (hasPrefix) { - return nodeInfo.getAttributes().get(f.getKey()).endsWith(str); - } - if (hasSuffix) { - return nodeInfo.getAttributes().get(f.getKey()).startsWith(str); - } - return nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); - case NE: - return !nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); - default: - long attr; - switch (f.getKey()) { - case ATTRIBUTE_PRICE: - attr = nodeInfo.getPrice().longValue(); - break; - case ATTRIBUTE_CAPACITY: - attr = nodeInfo.getCapacity().longValue(); - break; - default: - try { - attr = Long.parseLong(nodeInfo.getAttributes().get(f.getKey())); - } catch (NumberFormatException e) { - return false; - } - break; - } - - switch (f.getOperation()) { - case GT: - return attr > numCache.get(f.getValue()); - case GE: - return attr >= numCache.get(f.getValue()); - case LT: - return attr < numCache.get(f.getValue()); - case LE: - return attr <= numCache.get(f.getValue()); - default: - break; - } - break; - } - return false; - } - - boolean match(Filter f, NodeInfo nodeInfo) { - if (f == null) { - return false; - } - - switch (f.getOperation()) { - case NOT: - Filter[] inner = f.getFilters(); - Filter fSub = inner[0]; - - if (inner[0].getName() != null && !inner[0].getName().isEmpty()) { - fSub = processedFilters.get(inner[0].getName()); - } - return !match(fSub, nodeInfo); - case AND: - case OR: - for (int i = 0; i < f.getFilters().length; i++) { - Filter currentFilter = f.getFilters()[i]; - - if (currentFilter.getName() != null && !currentFilter.getName().isEmpty()) { - currentFilter = processedFilters.get(currentFilter.getName()); - } - - boolean ok = match(currentFilter, nodeInfo); - - if (ok == (f.getOperation() == FilterOperation.OR)) { - return ok; - } - } - return f.getOperation() == FilterOperation.AND; - default: - return matchKeyValue(f, nodeInfo); - } - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/HasherList.java b/client/src/main/java/info/frostfs/sdk/placement/HasherList.java deleted file mode 100644 index ee5a7ce..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/HasherList.java +++ /dev/null @@ -1,20 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.Hasher; -import info.frostfs.sdk.dto.netmap.NodeInfo; -import lombok.AllArgsConstructor; -import lombok.Getter; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; - -@Getter -@AllArgsConstructor -public final class HasherList implements Hasher { - private final List nodes; - - @Override - public long getHash() { - return CollectionUtils.isNotEmpty(nodes) ? nodes.get(0).getHash() : 0L; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java deleted file mode 100644 index d3464d4..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java +++ /dev/null @@ -1,18 +0,0 @@ -package info.frostfs.sdk.placement; - -import java.math.BigInteger; - -public class MeanAgg { - private double mean; - private int count; - - public void add(BigInteger n) { - int c = count + 1; - mean = mean * count / c + n.doubleValue() / c; - count++; - } - - public double compute() { - return mean; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java deleted file mode 100644 index b61b25e..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java +++ /dev/null @@ -1,57 +0,0 @@ -package info.frostfs.sdk.placement; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public final class MeanIQRAgg { - private static final int MIN_LN = 4; - - private final List arr = new ArrayList<>(); - - public MeanIQRAgg() { - } - - public void add(double d) { - arr.add(d); - } - - public double compute() { - int length = arr.size(); - if (length == 0) { - return 0; - } - - List sorted = new ArrayList<>(arr); - Collections.sort(sorted); - - double minV, maxV; - - if (length < MIN_LN) { - minV = sorted.get(0); - maxV = sorted.get(length - 1); - } else { - int start = length / MIN_LN; - int end = length * 3 / MIN_LN - 1; - - minV = sorted.get(start); - maxV = sorted.get(end); - } - - int count = 0; - double sum = 0; - - for (var e : sorted) { - if (e >= minV && e <= maxV) { - sum += e; - count++; - } - } - - return count == 0 ? 0 : sum / count; - } - - public void clear() { - arr.clear(); - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java deleted file mode 100644 index e2855b5..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java +++ /dev/null @@ -1,24 +0,0 @@ -package info.frostfs.sdk.placement; - -import java.math.BigInteger; - -public class MinAgg { - private double min; - private boolean minFound; - - public void add(BigInteger n) { - if (!minFound) { - min = n.doubleValue(); - minFound = true; - return; - } - - if (n.doubleValue() < min) { - min = n.doubleValue(); - } - } - - public double compute() { - return min; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java b/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java deleted file mode 100644 index 9fd1660..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java +++ /dev/null @@ -1,15 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.NodeInfo; -import lombok.Getter; - -@Getter -public class NodeAttributePair { - private final String attr; - private final NodeInfo[] nodes; - - NodeAttributePair(String attr, NodeInfo[] nodes) { - this.attr = attr; - this.nodes = nodes; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java b/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java deleted file mode 100644 index 587150d..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java +++ /dev/null @@ -1,5 +0,0 @@ -package info.frostfs.sdk.placement; - -public interface Normalizer { - double normalize(double w); -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java b/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java deleted file mode 100644 index f20c75f..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java +++ /dev/null @@ -1,197 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.*; -import info.frostfs.sdk.exceptions.FrostFSException; -import lombok.AllArgsConstructor; -import org.apache.commons.codec.digest.MurmurHash3; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.function.Function; - -import static info.frostfs.sdk.constants.ErrorConst.SELECTOR_NOT_FOUND_TEMPLATE; -import static info.frostfs.sdk.constants.ErrorConst.VECTORS_IS_NULL; - -@AllArgsConstructor -public final class PlacementVector { - private final NetmapSnapshot netmapSnapshot; - - private static NodeInfo[] flattenNodes(List> nodes) { - int size = nodes.stream().mapToInt(List::size).sum(); - NodeInfo[] result = new NodeInfo[size]; - - int i = 0; - for (List ns : nodes) { - for (NodeInfo n : ns) { - result[i++] = n; - } - } - return result; - } - - /* - * PlacementVectors sorts container nodes returned by ContainerNodes method - * and returns placement vectors for the entity identified by the given pivot. - * For example, in order to build node list to store the object, - * binary-encoded object identifier can be used as pivot. - * Result is deterministic for the fixed NetMap and parameters. - * */ - public NodeInfo[][] placementVectors(NodeInfo[][] vectors, byte[] pivot) { - if (vectors == null) { - throw new FrostFSException(VECTORS_IS_NULL); - } - - long hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; - - Function wf = Tools.defaultWeightFunc(netmapSnapshot.getNodeInfoCollection()); - - NodeInfo[][] result = new NodeInfo[vectors.length][]; - int maxSize = Arrays.stream(vectors) - .mapToInt(v -> v.length) - .max() - .orElse(0); - - double[] spanWeights = new double[maxSize]; - - for (int i = 0; i < vectors.length; i++) { - result[i] = Arrays.copyOf(vectors[i], vectors[i].length); - - Tools.appendWeightsTo(result[i], wf, spanWeights); - - List sorted = Tools.sortHasherSliceByWeightValue( - Arrays.asList(result[i]), - spanWeights, - hash - ); - result[i] = sorted.toArray(new NodeInfo[0]); - } - - return result; - } - - /* - * SelectFilterNodes returns a two-dimensional list of nodes as a result of applying the given - * SelectFilterExpr to the NetMap. If the SelectFilterExpr contains only filters, the result contains - * a single row with the result of the last filter application. If the SelectFilterExpr contains only selectors, - * the result contains the selection rows of the last select application. - * */ - public List> selectFilterNodes(SelectFilterExpr expr) { - PlacementPolicy policy = new PlacementPolicy( - null, - false, - expr.getCbf(), - expr.getFilters().toArray(Filter[]::new), - new Selector[]{expr.getSelector()} - ); - - Context ctx = new Context(netmapSnapshot); - ctx.setCbf(expr.getCbf()); - - ctx.processFilters(policy); - ctx.processSelectors(policy); - - List> ret = new ArrayList<>(); - - if (expr.getSelector() == null) { - Filter lastFilter = expr.getFilters().get(expr.getFilters().size() - 1); - List subCollection = new ArrayList<>(); - ret.add(subCollection); - - for (NodeInfo nodeInfo : netmapSnapshot.getNodeInfoCollection()) { - if (ctx.match(ctx.getProcessedFilters().get(lastFilter.getName()), nodeInfo)) { - subCollection.add(nodeInfo); - } - } - } else if (expr.getSelector().getName() != null) { - List> sel = ctx.getSelection( - ctx.getProcessedSelectors().get(expr.getSelector().getName()) - ); - - for (List ns : sel) { - List subCollection = new ArrayList<>(ns); - ret.add(subCollection); - } - } - - return ret; - } - - /* - * ContainerNodes returns two-dimensional list of nodes as a result of applying given PlacementPolicy to the NetMap. - * Each line of the list corresponds to a replica descriptor. - * Line order corresponds to order of ReplicaDescriptor list in the policy. - * Nodes are pre-filtered according to the Filter list from the policy, and then selected by Selector list. - * Result is deterministic for the fixed NetMap and parameters. - * - * Result can be used in PlacementVectors. - * */ - public NodeInfo[][] containerNodes(PlacementPolicy p, byte[] pivot) { - Context c = new Context(netmapSnapshot); - c.setCbf(p.getBackupFactory() == 0 ? 3 : p.getBackupFactory()); - - if (pivot != null && pivot.length > 0) { - c.setHrwSeed(pivot); - - var hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; - c.setHrwSeedHash(hash); - } - - c.processFilters(p); - c.processSelectors(p); - - boolean unique = p.isUnique(); - - List> result = new ArrayList<>(p.getReplicas().length); - for (int i = 0; i < p.getReplicas().length; i++) { - result.add(new ArrayList<>()); - } - - for (int i = 0; i < p.getReplicas().length; i++) { - String sName = p.getReplicas()[i].getSelector(); - - if ((sName == null || sName.isEmpty()) && - !(p.getReplicas().length == 1 && p.getSelectors().length == 1)) { - - Selector s = new Selector( - "", p.getReplicas()[i].getCountNodes(), null, null, - Context.MAIN_FILTER_NAME - ); - - List> nodes = c.getSelection(s); - result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); - - if (unique) { - for (NodeInfo n : result.get(i)) { - c.getUsedNodes().put(n.getHash(), true); - } - } - continue; - } - - if (unique) { - Selector s = c.getProcessedSelectors().get(sName); - if (s == null) { - throw new FrostFSException(String.format(SELECTOR_NOT_FOUND_TEMPLATE, sName)); - } - - List> nodes = c.getSelection(s); - result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); - - for (NodeInfo n : result.get(i)) { - c.getUsedNodes().put(n.getHash(), true); - } - } else { - List> nodes = c.getSelections().get(sName); - result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); - } - } - - NodeInfo[][] collection = new NodeInfo[result.size()][]; - for (int i = 0; i < result.size(); i++) { - collection[i] = result.get(i).toArray(new NodeInfo[0]); - } - - return collection; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java b/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java deleted file mode 100644 index 1bcc03a..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java +++ /dev/null @@ -1,14 +0,0 @@ -package info.frostfs.sdk.placement; - -public class ReverseMinNorm implements Normalizer { - private final double min; - - public ReverseMinNorm(double min) { - this.min = min; - } - - @Override - public double normalize(double w) { - return (min + 1) / (w + 1); - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java b/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java deleted file mode 100644 index b7dd898..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java +++ /dev/null @@ -1,16 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.Filter; -import info.frostfs.sdk.dto.netmap.Selector; -import lombok.AllArgsConstructor; -import lombok.Getter; - -import java.util.List; - -@Getter -@AllArgsConstructor -public class SelectFilterExpr { - private final int cbf; - private final Selector selector; - private final List filters; -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java b/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java deleted file mode 100644 index c0e867a..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java +++ /dev/null @@ -1,19 +0,0 @@ -package info.frostfs.sdk.placement; - -public class SigmoidNorm implements Normalizer { - private final double scale; - - public SigmoidNorm(double scale) { - this.scale = scale; - } - - @Override - public double normalize(double w) { - if (scale == 0) { - return 0; - } - - double x = w / scale; - return x / (1 + x); - } -} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Tools.java b/client/src/main/java/info/frostfs/sdk/placement/Tools.java deleted file mode 100644 index 0140078..0000000 --- a/client/src/main/java/info/frostfs/sdk/placement/Tools.java +++ /dev/null @@ -1,123 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.Hasher; -import info.frostfs.sdk.dto.netmap.NodeInfo; - -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; -import java.util.function.Function; - -import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; -import static info.frostfs.sdk.constants.CryptoConst.*; - -public final class Tools { - private Tools() { - } - - public static long distance(long x, long y) { - long acc = x ^ y; - acc ^= acc >>> MURMUR_MULTIPLIER; - acc *= LANDAU_PRIME_DIVISOR_65BIT; - acc ^= acc >>> MURMUR_MULTIPLIER; - acc *= LANDAU_PRIME_DIVISOR_64BIT; - acc ^= acc >>> MURMUR_MULTIPLIER; - return acc; - } - - public static void appendWeightsTo(NodeInfo[] nodes, Function wf, double[] weights) { - if (weights.length < nodes.length) { - weights = new double[nodes.length]; - } - for (int i = 0; i < nodes.length; i++) { - weights[i] = wf.apply(nodes[i]); - } - } - - public static List sortHasherSliceByWeightValue(List nodes, double[] weights, long hash) { - if (nodes.isEmpty()) { - return nodes; - } - - boolean allEquals = true; - if (weights.length > 1) { - for (int i = 1; i < weights.length; i++) { - if (weights[i] != weights[0]) { - allEquals = false; - break; - } - } - } - - Double[] dist = new Double[nodes.size()]; - - if (allEquals) { - for (int i = 0; i < dist.length; i++) { - long x = nodes.get(i).getHash(); - dist[i] = toUnsignedBigInteger(distance(x, hash)).doubleValue(); - } - return sortHasherByDistance(nodes, dist, true); - } - - for (int i = 0; i < dist.length; i++) { - var reverse = UNSIGNED_LONG_MASK.subtract(toUnsignedBigInteger(distance(nodes.get(i).getHash(), hash))); - dist[i] = reverse.doubleValue() * weights[i]; - } - - return sortHasherByDistance(nodes, dist, false); - } - - public static > List sortHasherByDistance( - List nodes, N[] dist, boolean asc - ) { - IndexedValue[] indexes = new IndexedValue[nodes.size()]; - for (int i = 0; i < dist.length; i++) { - indexes[i] = new IndexedValue<>(nodes.get(i), dist[i]); - } - - if (asc) { - Arrays.sort(indexes, Comparator.comparing(iv -> iv.dist)); - } else { - Arrays.sort(indexes, (iv1, iv2) -> iv2.dist.compareTo(iv1.dist)); - } - - List result = new ArrayList<>(); - for (IndexedValue iv : indexes) { - result.add(iv.nodeInfo); - } - return result; - } - - public static Function defaultWeightFunc(List nodes) { - MeanAgg mean = new MeanAgg(); - MinAgg minV = new MinAgg(); - - for (NodeInfo node : nodes) { - mean.add(node.getCapacity()); - minV.add(node.getPrice()); - } - - return newWeightFunc(new SigmoidNorm(mean.compute()), new ReverseMinNorm(minV.compute())); - } - - private static BigInteger toUnsignedBigInteger(long i) { - return i >= 0 ? BigInteger.valueOf(i) : BigInteger.valueOf(i).and(UNSIGNED_LONG_MASK); - } - - private static Function newWeightFunc(Normalizer capNorm, Normalizer priceNorm) { - return nodeInfo -> capNorm.normalize(nodeInfo.getCapacity().doubleValue()) - * priceNorm.normalize(nodeInfo.getPrice().doubleValue()); - } - - private static class IndexedValue { - final T nodeInfo; - final N dist; - - IndexedValue(T nodeInfo, N dist) { - this.nodeInfo = nodeInfo; - this.dist = dist; - } - } -} diff --git a/client/src/main/java/info/frostfs/sdk/pool/Pool.java b/client/src/main/java/info/frostfs/sdk/pool/Pool.java index 2191643..d8bac59 100644 --- a/client/src/main/java/info/frostfs/sdk/pool/Pool.java +++ b/client/src/main/java/info/frostfs/sdk/pool/Pool.java @@ -1,7 +1,6 @@ package info.frostfs.sdk.pool; import frostfs.refs.Types; -import info.frostfs.sdk.dto.ape.Chain; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.container.ContainerId; import info.frostfs.sdk.dto.netmap.NetmapSnapshot; @@ -515,7 +514,7 @@ public class Pool implements CommonClient { } @Override - public List listChains(PrmApeChainList args, CallContext ctx) { + public List listChains(PrmApeChainList args, CallContext ctx) { ClientWrapper client = connection(); return client.getClient().listChains(args, ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java b/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java index 47d7827..965e458 100644 --- a/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.services; -import info.frostfs.sdk.dto.ape.Chain; +import frostfs.ape.Types; import info.frostfs.sdk.jdo.parameters.CallContext; import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; @@ -13,5 +13,5 @@ public interface ApeManagerClient { void removeChain(PrmApeChainRemove args, CallContext ctx); - List listChains(PrmApeChainList args, CallContext ctx); + List listChains(PrmApeChainList args, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java index 6e10fe3..cd81060 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java @@ -4,7 +4,6 @@ import com.google.protobuf.ByteString; import frostfs.ape.Types; import frostfs.apemanager.APEManagerServiceGrpc; import frostfs.apemanager.Service; -import info.frostfs.sdk.dto.ape.Chain; import info.frostfs.sdk.jdo.ClientEnvironment; import info.frostfs.sdk.jdo.parameters.CallContext; import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; @@ -15,12 +14,10 @@ import info.frostfs.sdk.services.ApeManagerClient; import info.frostfs.sdk.services.ContextAccessor; import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.RequestSigner; +import info.frostfs.sdk.tools.RuleSerializer; import info.frostfs.sdk.tools.Verifier; -import info.frostfs.sdk.tools.ape.RuleDeserializer; -import info.frostfs.sdk.tools.ape.RuleSerializer; import java.util.List; -import java.util.stream.Collectors; import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; import static info.frostfs.sdk.utils.Validator.validate; @@ -60,7 +57,7 @@ public class ApeManagerClientImpl extends ContextAccessor implements ApeManagerC } @Override - public List listChains(PrmApeChainList args, CallContext ctx) { + public List listChains(PrmApeChainList args, CallContext ctx) { validate(args); var request = createListChainsRequest(args); @@ -70,9 +67,7 @@ public class ApeManagerClientImpl extends ContextAccessor implements ApeManagerC Verifier.checkResponse(response); - return response.getBody().getChainsList().stream() - .map(chain -> RuleDeserializer.deserialize(chain.getRaw().toByteArray())) - .collect(Collectors.toList()); + return response.getBody().getChainsList(); } private Service.AddChainRequest createAddChainRequest(PrmApeChainAdd args) { diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java index 71741e8..46b2905 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java @@ -231,29 +231,21 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { public ObjectId patchObject(PrmObjectPatch args, CallContext ctx) { validate(args); + var request = createInitPatchRequest(args); + var protoToken = RequestConstructor.createObjectTokenContext( + getOrCreateSession(args, ctx), + request.getBody().getAddress(), + frostfs.session.Types.ObjectSessionContext.Verb.PATCH, + getContext().getKey() + ); + + var currentPos = args.getRange().getOffset(); + var chunkSize = args.getMaxChunkLength(); + byte[] chunkBuffer = new byte[chunkSize]; + var service = deadLineAfter(objectServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); PatchStreamer writer = new PatchStreamer(service); - var request = createInitPatchRequest(args, ctx); - writer.write(request.build()); - - if (nonNull(args.getPayload())) { - patchObjectPayload(request, args, writer); - } - - var response = writer.complete(); - - Verifier.checkResponse(response); - - return ObjectIdMapper.toModel(response.getBody().getObjectId()); - } - - private void patchObjectPayload(Service.PatchRequest.Builder request, PrmObjectPatch args, PatchStreamer writer) { - var currentPos = args.getRange().getOffset(); - - var chunkSize = args.getMaxChunkLength() > 0 ? args.getMaxChunkLength() : AppConst.OBJECT_CHUNK_SIZE; - byte[] chunkBuffer = new byte[chunkSize]; - var bytesCount = readNBytes(args.getPayload(), chunkBuffer, chunkSize); while (bytesCount > 0) { var range = Service.Range.newBuilder() @@ -261,25 +253,25 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { .setLength(bytesCount) .build(); - var patch = Service.PatchRequest.Body.Patch.newBuilder() + Service.PatchRequest.Body.Patch.newBuilder() .setChunk(ByteString.copyFrom(chunkBuffer, 0, bytesCount)) .setSourceRange(range) .build(); - var body = Service.PatchRequest.Body.newBuilder() - .setAddress(request.getBody().getAddress()) - .setPatch(patch) - .build(); - request.setBody(body); + currentPos += bytesCount; - RequestConstructor.addMetaHeader(request, args.getXHeaders(), request.getMetaHeader().getSessionToken()); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); writer.write(request.build()); - - currentPos += bytesCount; bytesCount = readNBytes(args.getPayload(), chunkBuffer, chunkSize); } + + var response = writer.complete(); + + Verifier.checkResponse(response); + + return ObjectIdMapper.toModel(response.getBody().getObjectId()); } private ObjectFrostFS getObject(Service.GetRequest request, CallContext ctx) { @@ -643,26 +635,14 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { return request.build(); } - private Service.PatchRequest.Builder createInitPatchRequest(PrmObjectPatch args, CallContext ctx) { + private Service.PatchRequest.Builder createInitPatchRequest(PrmObjectPatch args) { var address = AddressMapper.toGrpcMessage(args.getAddress()); var body = Service.PatchRequest.Body.newBuilder() .setAddress(address) .setReplaceAttributes(args.isReplaceAttributes()) .addAllNewAttributes(ObjectAttributeMapper.toGrpcMessages(args.getNewAttributes())) .build(); - var request = Service.PatchRequest.newBuilder() + return Service.PatchRequest.newBuilder() .setBody(body); - - var protoToken = RequestConstructor.createObjectTokenContext( - getOrCreateSession(args, ctx), - request.getBody().getAddress(), - frostfs.session.Types.ObjectSessionContext.Verb.PATCH, - getContext().getKey() - ); - - RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); - sign(request, getContext().getKey()); - - return request; } } diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java b/client/src/main/java/info/frostfs/sdk/tools/MarshalFunction.java similarity index 70% rename from client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java rename to client/src/main/java/info/frostfs/sdk/tools/MarshalFunction.java index 1b08d5c..accd7b1 100644 --- a/client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java +++ b/client/src/main/java/info/frostfs/sdk/tools/MarshalFunction.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.tools.ape; +package info.frostfs.sdk.tools; public interface MarshalFunction { int marshal(byte[] buf, int offset, T t); diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java b/client/src/main/java/info/frostfs/sdk/tools/RuleSerializer.java similarity index 97% rename from client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java rename to client/src/main/java/info/frostfs/sdk/tools/RuleSerializer.java index 43e7210..097708d 100644 --- a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java +++ b/client/src/main/java/info/frostfs/sdk/tools/RuleSerializer.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.tools.ape; +package info.frostfs.sdk.tools; import info.frostfs.sdk.dto.ape.*; import info.frostfs.sdk.exceptions.ValidationFrostFSException; @@ -17,10 +17,6 @@ public class RuleSerializer { } public static byte[] serialize(Chain chain) { - if (isNull(chain)) { - throw new ValidationFrostFSException(String.format(INPUT_PARAM_IS_MISSING_TEMPLATE, Chain.class.getName())); - } - int s = U_INT_8_SIZE // Marshaller version + U_INT_8_SIZE // Chain version + sliceSize(chain.getId(), b -> BYTE_SIZE) diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java b/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java deleted file mode 100644 index cbba2e2..0000000 --- a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java +++ /dev/null @@ -1,198 +0,0 @@ -package info.frostfs.sdk.tools.ape; - -import info.frostfs.sdk.dto.ape.*; -import info.frostfs.sdk.enums.ConditionKindType; -import info.frostfs.sdk.enums.ConditionType; -import info.frostfs.sdk.enums.RuleMatchType; -import info.frostfs.sdk.enums.RuleStatus; -import info.frostfs.sdk.exceptions.FrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import org.apache.commons.lang3.ArrayUtils; - -import java.lang.reflect.Array; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.atomic.AtomicInteger; - -import static info.frostfs.sdk.constants.ErrorConst.*; -import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; -import static info.frostfs.sdk.constants.RuleConst.*; - -public class RuleDeserializer { - private RuleDeserializer() { - } - - public static Chain deserialize(byte[] data) { - if (ArrayUtils.isEmpty(data)) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - AtomicInteger offset = new AtomicInteger(0); - Chain chain = new Chain(); - - var version = uInt8Unmarshal(data, offset); - if (version != VERSION) { - throw new FrostFSException(String.format(UNSUPPORTED_MARSHALLER_VERSION_TEMPLATE, version)); - } - - var chainVersion = uInt8Unmarshal(data, offset); - if (chainVersion != CHAIN_MARSHAL_VERSION) { - throw new FrostFSException(String.format(UNSUPPORTED_CHAIN_VERSION_TEMPLATE, chainVersion)); - } - - chain.setId(sliceUnmarshal(data, offset, Byte.class, RuleDeserializer::uInt8Unmarshal)); - chain.setRules(sliceUnmarshal(data, offset, Rule.class, RuleDeserializer::unmarshalRule)); - chain.setMatchType(RuleMatchType.get(uInt8Unmarshal(data, offset))); - - verifyUnmarshal(data, offset); - - return chain; - } - - private static Byte uInt8Unmarshal(byte[] buf, AtomicInteger offset) { - if (buf.length - offset.get() < 1) { - throw new FrostFSException( - String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, Byte.class.getName(), offset.get())); - } - - return buf[offset.getAndAdd(1)]; - } - - public static long varInt(byte[] buf, AtomicInteger offset) { - long ux = uVarInt(buf, offset); // ok to continue in presence of error - long x = ux >> 1; - if ((ux & 1) != 0) { - x = ~x; - } - return x; - } - - - public static long uVarInt(byte[] buf, AtomicInteger offset) { - long x = 0; - int s = 0; - - for (int i = offset.get(); i < buf.length; i++) { - long b = buf[i]; - if (i == MAX_VAR_INT_LENGTH) { - offset.set(-(i + 1)); - return 0; // overflow - } - if (b >= 0) { - if (i == MAX_VAR_INT_LENGTH - 1 && b > 1) { - offset.set(-(i + 1)); - return 0; // overflow - } - offset.set(i + 1); - return x | (b << s); - } - x |= (b & OFFSET127) << s; - s += UNSIGNED_SERIALIZE_SIZE; - } - offset.set(0); - return 0; - } - - private static T[] sliceUnmarshal(byte[] buf, - AtomicInteger offset, - Class clazz, - UnmarshalFunction unmarshalT) { - var size = (int) varInt(buf, offset); - if (size == NULL_SLICE) { - return null; - } - - if (size > MAX_SLICE_LENGTH) { - throw new ValidationFrostFSException(String.format(SLICE_IS_TOO_BIG_TEMPLATE, size)); - } - - if (size < 0) { - throw new ValidationFrostFSException(String.format(SLICE_SIZE_IS_INVALID_TEMPLATE, size)); - } - - T[] result = (T[]) Array.newInstance(clazz, size); - for (int i = 0; i < result.length; i++) { - result[i] = unmarshalT.unmarshal(buf, offset); - } - - return result; - } - - private static boolean boolUnmarshal(byte[] buf, AtomicInteger offset) { - return uInt8Unmarshal(buf, offset) == BYTE_TRUE; - } - - private static long int64Unmarshal(byte[] buf, AtomicInteger offset) { - if (buf.length - offset.get() < Long.BYTES) { - throw new ValidationFrostFSException( - String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, Long.class.getName(), offset.get()) - ); - } - - return varInt(buf, offset); - } - - private static String stringUnmarshal(byte[] buf, AtomicInteger offset) { - int size = (int) int64Unmarshal(buf, offset); - if (size == 0) { - return EMPTY_STRING; - } - - if (size > MAX_SLICE_LENGTH) { - throw new ValidationFrostFSException(String.format(STRING_IS_TOO_BIG_TEMPLATE, size)); - } - - if (size < 0) { - throw new ValidationFrostFSException(String.format(STRING_SIZE_IS_INVALID_TEMPLATE, size)); - } - - if (buf.length - offset.get() < size) { - throw new ValidationFrostFSException( - String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, String.class.getName(), offset.get()) - ); - } - - return new String(buf, offset.getAndAdd(size), size, StandardCharsets.UTF_8); - } - - private static Actions unmarshalActions(byte[] buf, AtomicInteger offset) { - Actions actions = new Actions(); - actions.setInverted(boolUnmarshal(buf, offset)); - actions.setNames(sliceUnmarshal(buf, offset, String.class, RuleDeserializer::stringUnmarshal)); - return actions; - } - - private static Condition unmarshalCondition(byte[] buf, AtomicInteger offset) { - Condition condition = new Condition(); - condition.setOp(ConditionType.get(uInt8Unmarshal(buf, offset))); - condition.setKind(ConditionKindType.get(uInt8Unmarshal(buf, offset))); - condition.setKey(stringUnmarshal(buf, offset)); - condition.setValue(stringUnmarshal(buf, offset)); - - return condition; - } - - private static Rule unmarshalRule(byte[] buf, AtomicInteger offset) { - Rule rule = new Rule(); - rule.setStatus(RuleStatus.get(uInt8Unmarshal(buf, offset))); - rule.setActions(unmarshalActions(buf, offset)); - rule.setResources(unmarshalResources(buf, offset)); - rule.setAny(boolUnmarshal(buf, offset)); - rule.setConditions(sliceUnmarshal(buf, offset, Condition.class, RuleDeserializer::unmarshalCondition)); - - return rule; - } - - private static Resources unmarshalResources(byte[] buf, AtomicInteger offset) { - Resources resources = new Resources(); - resources.setInverted(boolUnmarshal(buf, offset)); - resources.setNames(sliceUnmarshal(buf, offset, String.class, RuleDeserializer::stringUnmarshal)); - - return resources; - } - - private static void verifyUnmarshal(byte[] buf, AtomicInteger offset) { - if (buf.length != offset.get()) { - throw new ValidationFrostFSException(UNMARSHAL_SIZE_DIFFERS); - } - } -} diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java b/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java deleted file mode 100644 index ce0977e..0000000 --- a/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java +++ /dev/null @@ -1,7 +0,0 @@ -package info.frostfs.sdk.tools.ape; - -import java.util.concurrent.atomic.AtomicInteger; - -public interface UnmarshalFunction { - T unmarshal(byte[] buf, AtomicInteger offset); -} diff --git a/client/src/main/java/info/frostfs/sdk/utils/Validator.java b/client/src/main/java/info/frostfs/sdk/utils/Validator.java index 5d61493..316f4c8 100644 --- a/client/src/main/java/info/frostfs/sdk/utils/Validator.java +++ b/client/src/main/java/info/frostfs/sdk/utils/Validator.java @@ -1,6 +1,9 @@ package info.frostfs.sdk.utils; -import info.frostfs.sdk.annotations.*; +import info.frostfs.sdk.annotations.AtLeastOneIsFilled; +import info.frostfs.sdk.annotations.NotBlank; +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.annotations.Validate; import info.frostfs.sdk.constants.ErrorConst; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; @@ -34,10 +37,6 @@ public class Validator { Class clazz = object.getClass(); - if (clazz.isAnnotationPresent(ComplexAtLeastOneIsFilled.class)) { - processComplexAtLeastOneIsFilled(object, clazz, errorMessage); - } - if (clazz.isAnnotationPresent(AtLeastOneIsFilled.class)) { processAtLeastOneIsFilled(object, clazz, errorMessage); } @@ -84,22 +83,8 @@ public class Validator { process(getFieldValue(object, field), errorMessage); } - private static void processComplexAtLeastOneIsFilled(T object, Class clazz, StringBuilder errorMessage) { - var annotation = clazz.getAnnotation(ComplexAtLeastOneIsFilled.class); - for (AtLeastOneIsFilled value : annotation.value()) { - processAtLeastOneIsFilled(object, clazz, errorMessage, value); - } - } - private static void processAtLeastOneIsFilled(T object, Class clazz, StringBuilder errorMessage) { var annotation = clazz.getAnnotation(AtLeastOneIsFilled.class); - processAtLeastOneIsFilled(object, clazz, errorMessage, annotation); - } - - private static void processAtLeastOneIsFilled(T object, - Class clazz, - StringBuilder errorMessage, - AtLeastOneIsFilled annotation) { var emptyFieldsCount = 0; for (String fieldName : annotation.fields()) { var field = getField(clazz, fieldName); @@ -121,7 +106,6 @@ public class Validator { } } - private static Object getFieldValue(T object, Field field) { try { return field.get(object); diff --git a/client/src/test/java/info/frostfs/sdk/FileUtils.java b/client/src/test/java/info/frostfs/sdk/FileUtils.java new file mode 100644 index 0000000..0eb9e80 --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/FileUtils.java @@ -0,0 +1,31 @@ +package info.frostfs.sdk; + +import com.google.common.io.ByteStreams; +import lombok.SneakyThrows; +import org.apache.commons.lang3.StringUtils; + +import java.io.InputStream; + +import static java.util.Objects.isNull; + +public class FileUtils { + + @SneakyThrows + public static byte[] resourceToBytes(String resourcePath) { + if (StringUtils.isBlank(resourcePath)) { + throw new IllegalArgumentException("Blank filename!"); + } + + ClassLoader loader = FileUtils.class.getClassLoader(); + if (isNull(loader)) { + throw new RuntimeException("Class loader is null!"); + } + + InputStream certStream = loader.getResourceAsStream(resourcePath); + if (isNull(certStream)) { + throw new RuntimeException("Resource could not be found!"); + } + + return ByteStreams.toByteArray(certStream); + } +} diff --git a/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java b/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java deleted file mode 100644 index 8ae8d48..0000000 --- a/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java +++ /dev/null @@ -1,238 +0,0 @@ -package info.frostfs.sdk.placement; - -import info.frostfs.sdk.dto.netmap.*; -import info.frostfs.sdk.enums.netmap.FilterOperation; -import info.frostfs.sdk.enums.netmap.NodeState; -import info.frostfs.sdk.enums.netmap.SelectorClause; -import lombok.Getter; -import lombok.SneakyThrows; -import org.junit.jupiter.api.Test; -import org.yaml.snakeyaml.Yaml; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.*; - -public class PlacementVectorTest { - private static final Yaml YAML = new Yaml(); - - private static void compareNodes(Map attrs, NodeInfo nodeInfo) { - assertEquals(attrs.size(), nodeInfo.getAttributes().size()); - assertEquals( - attrs.entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()), - nodeInfo.getAttributes().entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()) - ); - } - - @SneakyThrows - @Test - public void placementTest() { - Path resourceDirYaml = Paths.get(Objects.requireNonNull(getClass().getClassLoader() - .getResource("placement")).toURI()); - - List yamlFiles; - try (Stream paths = Files.walk(resourceDirYaml)) { - yamlFiles = paths.filter(Files::isRegularFile).collect(Collectors.toList()); - } - - Version v = new Version(2, 13); - String[] addresses = {"localhost", "server1"}; - - for (Path file : yamlFiles) { - TestCase testCase = YAML.loadAs(Files.newInputStream(file), TestCase.class); - - assertNotNull(testCase); - assertNotNull(testCase.nodes); - assertTrue(testCase.nodes.length > 0); - - List nodes = Arrays.stream(testCase.nodes) - .map(n -> new NodeInfo( - n.state, - v, - List.of(addresses), - n.attributes != null ? - Arrays.stream(n.attributes) - .collect(Collectors.toMap(KeyValuePair::getKey, KeyValuePair::getValue)) : - Collections.emptyMap(), - n.getPublicKeyBytes() - )) - .collect(Collectors.toList()); - - NetmapSnapshot netmap = new NetmapSnapshot(100L, nodes); - - assertNotNull(testCase.tests); - - for (var entry : testCase.tests.entrySet()) { - var test = entry.getValue(); - PlacementPolicy policy = new PlacementPolicy( - test.policy.replicas != null ? - Arrays.stream(test.policy.replicas) - .map(r -> new Replica(r.count, r.selector)) - .toArray(Replica[]::new) : - new Replica[0], - test.policy.unique, - test.policy.containerBackupFactor, - test.policy.filters != null - ? Arrays.stream(test.policy.filters) - .map(FilterDto::getFilter) - .toArray(Filter[]::new) - : new Filter[]{}, - test.policy.selectors != null - ? Arrays.stream(test.policy.selectors) - .map(SelectorDto::getSelector) - .toArray(Selector[]::new) - : new Selector[]{} - ); - - try { - var vector = new PlacementVector(netmap); - NodeInfo[][] result = vector.containerNodes(policy, test.getPivotBytes()); - - if (test.result == null) { - if (test.error != null && !test.error.isEmpty()) { - fail("Error is expected but has not been thrown"); - } else { - assertNotNull(test.policy.replicas); - assertEquals(result.length, test.policy.replicas.length); - - for (NodeInfo[] nodesArr : result) { - assertEquals(0, nodesArr.length); - } - } - } else { - assertEquals(test.result.length, result.length); - - for (int i = 0; i < test.result.length; i++) { - assertEquals(test.result[i].length, result[i].length); - for (int j = 0; j < test.result[i].length; j++) { - compareNodes(nodes.get(test.result[i][j]).getAttributes(), result[i][j]); - } - } - - if (test.placement != null - && test.placement.result != null - && test.placement.getPivotBytes() != null) { - NodeInfo[][] placementResult = vector.placementVectors( - result, test.placement.getPivotBytes() - ); - - assertEquals(test.placement.result.length, placementResult.length); - - for (int i = 0; i < placementResult.length; i++) { - assertEquals(test.placement.result[i].length, placementResult[i].length); - for (int j = 0; j < placementResult[i].length; j++) { - compareNodes( - nodes.get(test.placement.result[i][j]).getAttributes(), - placementResult[i][j] - ); - } - } - } - } - } catch (Exception ex) { - if (test.error != null && !test.error.isEmpty()) { - assertTrue(ex.getMessage().contains(test.error)); - } else { - throw ex; - } - } - } - } - } - - - public static class TestCase { - public String name; - public String comment; - public Node[] nodes; - public Map tests; - } - - public static class Node { - public KeyValuePair[] attributes; - public String publicKey; - public String[] addresses; - public NodeState state = NodeState.ONLINE; - - public byte[] getPublicKeyBytes() { - return publicKey == null || publicKey.isEmpty() ? new byte[0] : Base64.getDecoder().decode(publicKey); - } - } - - @Getter - public static class KeyValuePair { - public String key; - public String value; - } - - public static class TestData { - public PolicyDto policy; - public String pivot; - public int[][] result; - public String error; - public ResultData placement; - - public byte[] getPivotBytes() { - return pivot == null ? null : Base64.getDecoder().decode(pivot); - } - } - - public static class PolicyDto { - public boolean unique; - public int containerBackupFactor; - public FilterDto[] filters; - public ReplicaDto[] replicas; - public SelectorDto[] selectors; - } - - public static class SelectorDto { - public int count; - public String name; - public SelectorClause clause; - public String attribute; - public String filter; - - public Selector getSelector() { - return new Selector(name != null ? name : "", count, clause, attribute, filter); - } - } - - public static class FilterDto { - public String name; - public String key; - public FilterOperation op; - public String value; - public FilterDto[] filters; - - public Filter getFilter() { - return new Filter( - name != null ? name : "", - key != null ? key : "", - op, - value != null ? value : "", - filters != null - ? Arrays.stream(filters).map(FilterDto::getFilter).toArray(Filter[]::new) - : new Filter[0] - ); - } - } - - public static class ReplicaDto { - public int count; - public String selector; - } - - public static class ResultData { - public String pivot; - public int[][] result; - - public byte[] getPivotBytes() { - return pivot == null ? null : Base64.getDecoder().decode(pivot); - } - } -} diff --git a/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java b/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java index 741475c..b137af2 100644 --- a/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java +++ b/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java @@ -2,6 +2,7 @@ package info.frostfs.sdk.services; import frostfs.accounting.AccountingServiceGrpc; import frostfs.accounting.Service; +import info.frostfs.sdk.Base58; import info.frostfs.sdk.dto.object.OwnerId; import info.frostfs.sdk.jdo.ClientEnvironment; import info.frostfs.sdk.jdo.parameters.CallContext; @@ -11,7 +12,6 @@ import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.RequestSigner; import info.frostfs.sdk.tools.Verifier; import io.grpc.Channel; -import io.neow3j.crypto.Base58; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java b/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java index d6a4e4b..5e62c74 100644 --- a/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java +++ b/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java @@ -208,7 +208,8 @@ class ApeManagerClientTest { ); verifierMock.verify(() -> Verifier.checkResponse(response), times(1)); - assertThat(result).hasSize(10); + var expected = response.getBody().getChainsList(); + assertThat(result).hasSize(10).containsAll(expected); var request = captor.getValue(); assertEquals(chainTarget.getName(), request.getBody().getTarget().getName()); diff --git a/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java b/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java index a050114..1a9e497 100644 --- a/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java +++ b/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java @@ -3,21 +3,22 @@ package info.frostfs.sdk.testgenerator; import com.google.protobuf.ByteString; import frostfs.ape.Types; import frostfs.apemanager.Service; +import info.frostfs.sdk.FileUtils; import info.frostfs.sdk.Helper; -import org.bouncycastle.util.encoders.Base64; import java.util.ArrayList; public class ApeManagerGenerator { - private static final String BASE64_CHAIN = "AAAaY2hhaW4taWQtdGVzdAIAAAICKgACHm5hdGl2ZTpvYmplY3QvKgAAAA=="; private static ByteString generateChainID() { return ByteString.copyFrom(Helper.getByteArrayFromHex("616c6c6f774f626a476574436e72")); } private static Types.Chain generateRawChain() { + byte[] chainRaw = FileUtils.resourceToBytes("test_chain_raw.json"); + return Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(Base64.decode(BASE64_CHAIN))) + .setRaw(ByteString.copyFrom(chainRaw)) .build(); } diff --git a/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java b/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java deleted file mode 100644 index e3df60c..0000000 --- a/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java +++ /dev/null @@ -1,179 +0,0 @@ -package info.frostfs.sdk.tools.ape; - -import info.frostfs.sdk.dto.ape.*; -import info.frostfs.sdk.enums.ConditionKindType; -import info.frostfs.sdk.enums.ConditionType; -import info.frostfs.sdk.enums.RuleMatchType; -import info.frostfs.sdk.enums.RuleStatus; -import info.frostfs.sdk.exceptions.FrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import org.apache.commons.lang3.ArrayUtils; -import org.bouncycastle.util.encoders.Base64; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; - -import static java.util.Objects.isNull; -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.*; - -public class ApeRuleTest { - - @Test - void apeRuleTest() { - //Given - var resources = new Resources(false, new String[]{"native:object/*"}); - var actions = new Actions(false, new String[]{"*"}); - var rule = new Rule(); - rule.setStatus(RuleStatus.ALLOW); - rule.setResources(resources); - rule.setActions(actions); - rule.setAny(false); - rule.setConditions(new Condition[]{}); - - - var chain = new Chain(); - chain.setId(ArrayUtils.toObject("chain-id-test".getBytes(StandardCharsets.UTF_8))); - chain.setRules(new Rule[]{rule}); - chain.setMatchType(RuleMatchType.DENY_PRIORITY); - - //When - var serialized = RuleSerializer.serialize(chain); - var t = Base64.encode(serialized); - var restoredChain = RuleDeserializer.deserialize(serialized); - - //Then - assertThat(restoredChain.getId()).isNotEmpty().containsOnly(chain.getId()); - assertEquals(chain.getMatchType(), restoredChain.getMatchType()); - compareRules(chain.getRules(), restoredChain.getRules()); - } - - @Test - void apeRuleTest2() { - //Given - var resources = new Resources(true, new String[]{"native:object/*", "U.S.S. ENTERPRISE"}); - var actions = new Actions(true, new String[]{"put", "get"}); - var cond1 = new Condition( - ConditionType.COND_STRING_EQUALS, ConditionKindType.RESOURCE, "key1", "value1" - ); - var cond2 = new Condition( - ConditionType.COND_NUMERIC_GREATER_THAN, ConditionKindType.REQUEST, "key2", "value2" - ); - var rule = new Rule(); - rule.setStatus(RuleStatus.ACCESS_DENIED); - rule.setResources(resources); - rule.setActions(actions); - rule.setAny(true); - rule.setConditions(new Condition[]{cond1, cond2}); - - - var chain = new Chain(); - chain.setId(ArrayUtils.toObject("dumptext".getBytes(StandardCharsets.UTF_8))); - chain.setRules(new Rule[]{rule}); - chain.setMatchType(RuleMatchType.FIRST_MATCH); - - //When - var serialized = RuleSerializer.serialize(chain); - var restoredChain = RuleDeserializer.deserialize(serialized); - - //Then - assertThat(restoredChain.getId()).isNotEmpty().containsOnly(chain.getId()); - assertEquals(chain.getMatchType(), restoredChain.getMatchType()); - compareRules(chain.getRules(), restoredChain.getRules()); - } - - @Test - void apeRuleTest3() { - //Given - var chain = new Chain(); - chain.setMatchType(RuleMatchType.DENY_PRIORITY); - - //When - var serialized = RuleSerializer.serialize(chain); - var restoredChain = RuleDeserializer.deserialize(serialized); - - //Then - assertNull(restoredChain.getId()); - assertEquals(chain.getMatchType(), restoredChain.getMatchType()); - assertNull(restoredChain.getRules()); - } - - @Test - void apeRule_deserialize_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(null)); - assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{})); - assertThrows(FrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{1, 2, 3})); - assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{ - 0x00, 0x00, 0x3A, 0x77, 0x73, 0x3A, 0x69, 0x61, 0x6D, 0x3A, 0x3A, 0x6E, 0x61, 0x6D, 0x65, 0x73, - 0x70, 0x61, 0x63, 0x65, 0x3A, 0x67, 0x72, 0x6F, 0x75, 0x70, 0x2F, 0x73, 0x6F, (byte) 0x82, (byte) 0x82, - (byte) 0x82, (byte) 0x82, (byte) 0x82, (byte) 0x82, 0x75, (byte) 0x82 - })); - } - - @Test - void apeRule_serialize_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> RuleSerializer.serialize(null)); - } - - private void compareRules(Rule[] rules1, Rule[] rules2) { - assertThat(rules1).isNotEmpty(); - assertThat(rules2).isNotEmpty(); - - assertEquals(rules1.length, rules2.length); - - for (int ri = 0; ri < rules1.length; ri++) { - var rule1 = rules1[ri]; - var rule2 = rules2[ri]; - - assertEquals(rule1.getStatus(), rule2.getStatus()); - assertEquals(rule1.isAny(), rule2.isAny()); - - compareActions(rule1.getActions(), rule2.getActions()); - compareResources(rule1.getResources(), rule2.getResources()); - compareConditions(rule1.getConditions(), rule2.getConditions()); - } - } - - private void compareActions(Actions actions1, Actions actions2) { - if (isNull(actions1) && isNull(actions2)) { - return; - } - - assertEquals(actions1.isInverted(), actions2.isInverted()); - if (ArrayUtils.isEmpty(actions1.getNames()) && ArrayUtils.isEmpty(actions2.getNames())) { - return; - } - - assertThat(actions2.getNames()).hasSize(actions1.getNames().length).containsOnly(actions1.getNames()); - } - - private void compareResources(Resources resources1, Resources resources2) { - if (isNull(resources1) && isNull(resources2)) { - return; - } - - assertEquals(resources1.isInverted(), resources2.isInverted()); - if (ArrayUtils.isEmpty(resources1.getNames()) && ArrayUtils.isEmpty(resources2.getNames())) { - return; - } - - assertThat(resources2.getNames()).hasSize(resources1.getNames().length).containsOnly(resources1.getNames()); - } - - private void compareConditions(Condition[] conditions1, Condition[] conditions2) { - if (ArrayUtils.isEmpty(conditions1) && ArrayUtils.isEmpty(conditions2)) { - return; - } - - assertEquals(conditions1.length, conditions2.length); - for (int i = 0; i < conditions1.length; i++) { - assertEquals(conditions1[i].getOp(), conditions2[i].getOp()); - assertEquals(conditions1[i].getKind(), conditions2[i].getKind()); - assertEquals(conditions1[i].getKey(), conditions2[i].getKey()); - assertEquals(conditions1[i].getValue(), conditions2[i].getValue()); - } - } - -} diff --git a/client/src/test/resources/placement/cbf_default.yml b/client/src/test/resources/placement/cbf_default.yml deleted file mode 100644 index c43a703..0000000 --- a/client/src/test/resources/placement/cbf_default.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: default CBF is 3 -nodes: - - attributes: - - key: Location - value: Europe - - key: Country - value: RU - - key: City - value: St.Petersburg - - attributes: - - key: Location - value: Europe - - key: Country - value: RU - - key: City - value: Moscow - - attributes: - - key: Location - value: Europe - - key: Country - value: DE - - key: City - value: Berlin - - attributes: - - key: Location - value: Europe - - key: Country - value: FR - - key: City - value: Paris -tests: - set default CBF: - policy: - replicas: - - count: 1 - selector: EU - containerBackupFactor: 0 - selectors: - - name: EU - count: 1 - clause: SAME - attribute: Location - filter: '*' - filters: [] - result: - - - 0 - - 1 - - 2 diff --git a/client/src/test/resources/placement/cbf_minimal.yml b/client/src/test/resources/placement/cbf_minimal.yml deleted file mode 100644 index 2fe2642..0000000 --- a/client/src/test/resources/placement/cbf_minimal.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Real node count multiplier is in range [1, specified CBF] -nodes: - - attributes: - - key: ID - value: '1' - - key: Country - value: DE - - attributes: - - key: ID - value: '2' - - key: Country - value: DE - - attributes: - - key: ID - value: '3' - - key: Country - value: DE -tests: - select 2, CBF is 2: - policy: - replicas: - - count: 1 - selector: X - containerBackupFactor: 2 - selectors: - - name: X - count: 2 - clause: SAME - attribute: Country - filter: '*' - filters: [] - result: - - - 0 - - 1 - - 2 - select 3, CBF is 2: - policy: - replicas: - - count: 1 - selector: X - containerBackupFactor: 2 - selectors: - - name: X - count: 3 - clause: SAME - attribute: Country - filter: '*' - filters: [] - result: - - - 0 - - 1 - - 2 diff --git a/client/src/test/resources/placement/cbf_requirements.yml b/client/src/test/resources/placement/cbf_requirements.yml deleted file mode 100644 index ccd58d4..0000000 --- a/client/src/test/resources/placement/cbf_requirements.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: CBF requirements -nodes: - - attributes: - - key: ID - value: '1' - - key: Attr - value: Same - - attributes: - - key: ID - value: '2' - - key: Attr - value: Same - - attributes: - - key: ID - value: '3' - - key: Attr - value: Same - - attributes: - - key: ID - value: '4' - - key: Attr - value: Same -tests: - default CBF, no selector: - policy: - replicas: - - count: 2 - containerBackupFactor: 0 - selectors: [] - filters: [] - result: - - - 0 - - 2 - - 1 - - 3 - explicit CBF, no selector: - policy: - replicas: - - count: 2 - containerBackupFactor: 3 - selectors: [] - filters: [] - result: - - - 0 - - 2 - - 1 - - 3 - select distinct, weak CBF: - policy: - replicas: - - count: 2 - selector: X - containerBackupFactor: 3 - selectors: - - name: X - count: 2 - clause: DISTINCT - filter: '*' - filters: [] - result: - - - 0 - - 2 - - 1 - - 3 - select same, weak CBF: - policy: - replicas: - - count: 2 - selector: X - containerBackupFactor: 3 - selectors: - - name: X - count: 2 - clause: SAME - attribute: Attr - filter: '*' - filters: [] - result: - - - 0 - - 1 - - 2 - - 3 diff --git a/client/src/test/resources/placement/filter_complex.yml b/client/src/test/resources/placement/filter_complex.yml deleted file mode 100644 index 1abc46a..0000000 --- a/client/src/test/resources/placement/filter_complex.yml +++ /dev/null @@ -1,207 +0,0 @@ -name: compound filter -nodes: - - attributes: - - key: Storage - value: SSD - - key: Rating - value: '10' - - key: IntField - value: '100' - - key: Param - value: Value1 -tests: - good: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: StorageSSD - key: Storage - op: EQ - value: SSD - filters: [] - - name: GoodRating - key: Rating - op: GE - value: '4' - filters: [] - - name: Main - op: AND - filters: - - name: StorageSSD - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - key: IntField - op: LT - value: '123' - filters: [] - - name: GoodRating - op: OPERATION_UNSPECIFIED - filters: [] - - op: OR - filters: - - key: Param - op: EQ - value: Value1 - filters: [] - - key: Param - op: EQ - value: Value2 - filters: [] - result: - - - 0 - bad storage type: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: StorageSSD - key: Storage - op: EQ - value: HDD - filters: [] - - name: GoodRating - key: Rating - op: GE - value: '4' - filters: [] - - name: Main - op: AND - filters: - - name: StorageSSD - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - key: IntField - op: LT - value: '123' - filters: [] - - name: GoodRating - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - op: OR - filters: - - name: '' - key: Param - op: EQ - value: Value1 - filters: [] - - name: '' - key: Param - op: EQ - value: Value2 - filters: [] - bad rating: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: StorageSSD - key: Storage - op: EQ - value: SSD - filters: [] - - name: GoodRating - key: Rating - op: GE - value: '15' - filters: [] - - name: Main - op: AND - filters: - - name: StorageSSD - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - key: IntField - op: LT - value: '123' - filters: [] - - name: GoodRating - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - op: OR - filters: - - name: '' - key: Param - op: EQ - value: Value1 - filters: [] - - name: '' - key: Param - op: EQ - value: Value2 - filters: [] - bad param: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: StorageSSD - key: Storage - op: EQ - value: SSD - filters: [] - - name: GoodRating - key: Rating - op: GE - value: '4' - filters: [] - - name: Main - op: AND - filters: - - name: StorageSSD - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - key: IntField - op: LT - value: '123' - filters: [] - - name: GoodRating - op: OPERATION_UNSPECIFIED - filters: [] - - name: '' - op: OR - filters: - - name: '' - key: Param - op: EQ - value: Value0 - filters: [] - - name: '' - key: Param - op: EQ - value: Value2 - filters: [] diff --git a/client/src/test/resources/placement/filter_invalid_integer.yml b/client/src/test/resources/placement/filter_invalid_integer.yml deleted file mode 100644 index 6674246..0000000 --- a/client/src/test/resources/placement/filter_invalid_integer.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: invalid integer field -nodes: - - attributes: - - key: IntegerField - value: 'true' - - attributes: - - key: IntegerField - value: str -tests: - empty string is not casted to 0: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: IntegerField - op: LE - value: '8' - filters: [] - non-empty string is not casted to a number: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: IntegerField - op: GE - value: '0' - filters: [] diff --git a/client/src/test/resources/placement/filter_simple.yml b/client/src/test/resources/placement/filter_simple.yml deleted file mode 100644 index 7fdd84a..0000000 --- a/client/src/test/resources/placement/filter_simple.yml +++ /dev/null @@ -1,224 +0,0 @@ -name: single-op filters -nodes: - - attributes: - - key: Rating - value: '4' - - key: Country - value: Germany -tests: - GE true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: GE - value: '4' - filters: [] - result: - - - 0 - GE false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: GE - value: '5' - filters: [] - GT true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: GT - value: '3' - filters: [] - result: - - - 0 - GT false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: GT - value: '4' - filters: [] - LE true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: LE - value: '4' - filters: [] - result: - - - 0 - LE false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: LE - value: '3' - filters: [] - LT true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: LT - value: '5' - filters: [] - result: - - - 0 - LT false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Rating - op: LT - value: '4' - filters: [] - EQ true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Country - op: EQ - value: Germany - filters: [] - result: - - - 0 - EQ false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Country - op: EQ - value: China - filters: [] - NE true: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Country - op: NE - value: France - filters: [] - result: - - - 0 - NE false: - policy: - replicas: - - count: 1 - selector: S - containerBackupFactor: 1 - selectors: - - name: S - count: 1 - clause: DISTINCT - filter: Main - filters: - - name: Main - key: Country - op: NE - value: Germany - filters: [] diff --git a/client/src/test/resources/placement/hrw_sort.yml b/client/src/test/resources/placement/hrw_sort.yml deleted file mode 100644 index c84f7c9..0000000 --- a/client/src/test/resources/placement/hrw_sort.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: HRW ordering -nodes: - - attributes: - - key: Country - value: Germany - - key: Price - value: '2' - - key: Capacity - value: '10000' - - attributes: - - key: Country - value: Germany - - key: Price - value: '4' - - key: Capacity - value: '1' - - attributes: - - key: Country - value: France - - key: Price - value: '3' - - key: Capacity - value: '10' - - attributes: - - key: Country - value: Russia - - key: Price - value: '2' - - key: Capacity - value: '10000' - - attributes: - - key: Country - value: Russia - - key: Price - value: '1' - - key: Capacity - value: '10000' - - attributes: - - key: Country - value: Russia - - key: Capacity - value: '10000' - - attributes: - - key: Country - value: France - - key: Price - value: '100' - - key: Capacity - value: '1' - - attributes: - - key: Country - value: France - - key: Price - value: '7' - - key: Capacity - value: '10000' - - attributes: - - key: Country - value: Russia - - key: Price - value: '2' - - key: Capacity - value: '1' -tests: - select 3 nodes in 3 distinct countries, same placement: - policy: - replicas: - - count: 1 - selector: Main - containerBackupFactor: 1 - selectors: - - name: Main - count: 3 - clause: DISTINCT - attribute: Country - filter: '*' - filters: [] - pivot: Y29udGFpbmVySUQ= - result: - - - 5 - - 0 - - 7 - placement: - pivot: b2JqZWN0SUQ= - result: - - - 5 - - 0 - - 7 - select 6 nodes in 3 distinct countries, different placement: - policy: - replicas: - - count: 1 - selector: Main - containerBackupFactor: 2 - selectors: - - name: Main - count: 3 - clause: DISTINCT - attribute: Country - filter: '*' - filters: [] - pivot: Y29udGFpbmVySUQ= - result: - - - 5 - - 4 - - 0 - - 1 - - 7 - - 2 - placement: - pivot: b2JqZWN0SUQ= - result: - - - 5 - - 4 - - 0 - - 7 - - 2 - - 1 diff --git a/client/src/test/resources/placement/issue213.yml b/client/src/test/resources/placement/issue213.yml deleted file mode 100644 index 8e8aea4..0000000 --- a/client/src/test/resources/placement/issue213.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: unnamed selector (nspcc-dev/neofs-api-go#213) -nodes: - - attributes: - - key: Location - value: Europe - - key: Country - value: Russia - - key: City - value: Moscow - - attributes: - - key: Location - value: Europe - - key: Country - value: Russia - - key: City - value: Saint-Petersburg - - attributes: - - key: Location - value: Europe - - key: Country - value: Sweden - - key: City - value: Stockholm - - attributes: - - key: Location - value: Europe - - key: Country - value: Finalnd - - key: City - value: Helsinki -tests: - test: - policy: - replicas: - - count: 4 - containerBackupFactor: 1 - selectors: - - name: '' - count: 4 - clause: DISTINCT - filter: LOC_EU - filters: - - name: LOC_EU - key: Location - op: EQ - value: Europe - filters: [] - result: - - - 0 - - 1 - - 2 - - 3 diff --git a/client/src/test/resources/placement/many_selects.yml b/client/src/test/resources/placement/many_selects.yml deleted file mode 100644 index 29efd43..0000000 --- a/client/src/test/resources/placement/many_selects.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: single-op filters -nodes: - - attributes: - - key: Country - value: Russia - - key: Rating - value: '1' - - key: City - value: SPB - - attributes: - - key: Country - value: Germany - - key: Rating - value: '5' - - key: City - value: Berlin - - attributes: - - key: Country - value: Russia - - key: Rating - value: '6' - - key: City - value: Moscow - - attributes: - - key: Country - value: France - - key: Rating - value: '4' - - key: City - value: Paris - - attributes: - - key: Country - value: France - - key: Rating - value: '1' - - key: City - value: Lyon - - attributes: - - key: Country - value: Russia - - key: Rating - value: '5' - - key: City - value: SPB - - attributes: - - key: Country - value: Russia - - key: Rating - value: '7' - - key: City - value: Moscow - - attributes: - - key: Country - value: Germany - - key: Rating - value: '3' - - key: City - value: Darmstadt - - attributes: - - key: Country - value: Germany - - key: Rating - value: '7' - - key: City - value: Frankfurt - - attributes: - - key: Country - value: Russia - - key: Rating - value: '9' - - key: City - value: SPB - - attributes: - - key: Country - value: Russia - - key: Rating - value: '9' - - key: City - value: SPB -tests: - Select: - policy: - replicas: - - count: 1 - selector: SameRU - - count: 1 - selector: DistinctRU - - count: 1 - selector: Good - - count: 1 - selector: Main - containerBackupFactor: 2 - selectors: - - name: SameRU - count: 2 - clause: SAME - attribute: City - filter: FromRU - - name: DistinctRU - count: 2 - clause: DISTINCT - attribute: City - filter: FromRU - - name: Good - count: 2 - clause: DISTINCT - attribute: Country - filter: Good - - name: Main - count: 3 - clause: DISTINCT - attribute: Country - filter: '*' - filters: - - name: FromRU - key: Country - op: EQ - value: Russia - - name: Good - key: Rating - op: GE - value: '4' - result: - - - 0 - - 5 - - 9 - - 10 - - - 2 - - 6 - - 0 - - 5 - - - 1 - - 8 - - 2 - - 5 - - - 3 - - 4 - - 1 - - 7 - - 0 - - 2 diff --git a/client/src/test/resources/placement/multiple_rep.yml b/client/src/test/resources/placement/multiple_rep.yml deleted file mode 100644 index 448214f..0000000 --- a/client/src/test/resources/placement/multiple_rep.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: multiple replicas (#215) -nodes: - - attributes: - - key: City - value: Saint-Petersburg - - attributes: - - key: City - value: Moscow - - attributes: - - key: City - value: Berlin - - attributes: - - key: City - value: Paris -tests: - test: - policy: - replicas: - - count: 1 - selector: LOC_SPB_PLACE - - count: 1 - selector: LOC_MSK_PLACE - containerBackupFactor: 1 - selectors: - - name: LOC_SPB_PLACE - count: 1 - clause: CLAUSE_UNSPECIFIED - filter: LOC_SPB - - name: LOC_MSK_PLACE - count: 1 - clause: CLAUSE_UNSPECIFIED - filter: LOC_MSK - filters: - - name: LOC_SPB - key: City - op: EQ - value: Saint-Petersburg - filters: [] - - name: LOC_MSK - key: City - op: EQ - value: Moscow - filters: [] - result: - - - 0 - - - 1 diff --git a/client/src/test/resources/placement/multiple_rep_asymmetric.yml b/client/src/test/resources/placement/multiple_rep_asymmetric.yml deleted file mode 100644 index 61f8f76..0000000 --- a/client/src/test/resources/placement/multiple_rep_asymmetric.yml +++ /dev/null @@ -1,162 +0,0 @@ -name: multiple REP, asymmetric -nodes: - - attributes: - - key: ID - value: '1' - - key: Country - value: RU - - key: City - value: St.Petersburg - - key: SSD - value: '0' - - attributes: - - key: ID - value: '2' - - key: Country - value: RU - - key: City - value: St.Petersburg - - key: SSD - value: '1' - - attributes: - - key: ID - value: '3' - - key: Country - value: RU - - key: City - value: Moscow - - key: SSD - value: '1' - - attributes: - - key: ID - value: '4' - - key: Country - value: RU - - key: City - value: Moscow - - key: SSD - value: '1' - - attributes: - - key: ID - value: '5' - - key: Country - value: RU - - key: City - value: St.Petersburg - - key: SSD - value: '1' - - attributes: - - key: ID - value: '6' - - key: Continent - value: NA - - key: City - value: NewYork - - attributes: - - key: ID - value: '7' - - key: Continent - value: AF - - key: City - value: Cairo - - attributes: - - key: ID - value: '8' - - key: Continent - value: AF - - key: City - value: Cairo - - attributes: - - key: ID - value: '9' - - key: Continent - value: SA - - key: City - value: Lima - - attributes: - - key: ID - value: '10' - - key: Continent - value: AF - - key: City - value: Cairo - - attributes: - - key: ID - value: '11' - - key: Continent - value: NA - - key: City - value: NewYork - - attributes: - - key: ID - value: '12' - - key: Continent - value: NA - - key: City - value: LosAngeles - - attributes: - - key: ID - value: '13' - - key: Continent - value: SA - - key: City - value: Lima -tests: - test: - policy: - replicas: - - count: 1 - selector: SPB - - count: 2 - selector: Americas - containerBackupFactor: 2 - selectors: - - name: SPB - count: 1 - clause: SAME - attribute: City - filter: SPBSSD - - name: Americas - count: 2 - clause: DISTINCT - attribute: City - filter: Americas - filters: - - name: SPBSSD - op: AND - filters: - - name: '' - key: Country - op: EQ - value: RU - filters: [] - - name: '' - key: City - op: EQ - value: St.Petersburg - filters: [] - - name: '' - key: SSD - op: EQ - value: '1' - filters: [] - - name: Americas - op: OR - filters: - - name: '' - key: Continent - op: EQ - value: NA - filters: [] - - name: '' - key: Continent - op: EQ - value: SA - filters: [] - result: - - - 1 - - 4 - - - 8 - - 12 - - 5 - - 10 diff --git a/client/src/test/resources/placement/non_strict.yml b/client/src/test/resources/placement/non_strict.yml deleted file mode 100644 index a01986d..0000000 --- a/client/src/test/resources/placement/non_strict.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: non-strict selections -comment: These test specify loose selection behaviour, to allow fetching already PUT - objects even when there is not enough nodes to select from. -nodes: - - attributes: - - key: Country - value: Russia - - attributes: - - key: Country - value: Germany - - attributes: [] -tests: - not enough nodes (backup factor): - policy: - replicas: - - count: 1 - selector: MyStore - containerBackupFactor: 2 - selectors: - - name: MyStore - count: 2 - clause: DISTINCT - attribute: Country - filter: FromRU - filters: - - name: FromRU - key: Country - op: EQ - value: Russia - filters: [] - result: - - - 0 - not enough nodes (buckets): - policy: - replicas: - - count: 1 - selector: MyStore - containerBackupFactor: 1 - selectors: - - name: MyStore - count: 2 - clause: DISTINCT - attribute: Country - filter: FromRU - filters: - - name: FromRU - key: Country - op: EQ - value: Russia - filters: [] - result: - - - 0 diff --git a/client/src/test/resources/placement/rep_only.yml b/client/src/test/resources/placement/rep_only.yml deleted file mode 100644 index b354591..0000000 --- a/client/src/test/resources/placement/rep_only.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: REP X -nodes: - - publicKey: '' - addresses: [] - attributes: - - key: City - value: Saint-Petersburg - state: UNSPECIFIED - - publicKey: '' - addresses: [] - attributes: - - key: City - value: Moscow - state: UNSPECIFIED - - publicKey: '' - addresses: [] - attributes: - - key: City - value: Berlin - state: UNSPECIFIED - - publicKey: '' - addresses: [] - attributes: - - key: City - value: Paris - state: UNSPECIFIED -tests: - REP 1: - policy: - replicas: - - count: 1 - containerBackupFactor: 0 - selectors: [] - filters: [] - result: - - - 0 - - 1 - - 2 - REP 3: - policy: - replicas: - - count: 3 - containerBackupFactor: 0 - selectors: [] - filters: [] - result: - - - 0 - - 3 - - 1 - - 2 - REP 5: - policy: - replicas: - - count: 5 - containerBackupFactor: 0 - selectors: [] - filters: [] - result: - - - 0 - - 1 - - 2 - - 3 diff --git a/client/src/test/resources/placement/select_no_attribute.yml b/client/src/test/resources/placement/select_no_attribute.yml deleted file mode 100644 index 02046f3..0000000 --- a/client/src/test/resources/placement/select_no_attribute.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: select with unspecified attribute -nodes: - - attributes: - - key: ID - value: '1' - - key: Country - value: RU - - key: City - value: St.Petersburg - - key: SSD - value: '0' - - attributes: - - key: ID - value: '2' - - key: Country - value: RU - - key: City - value: St.Petersburg - - key: SSD - value: '1' - - attributes: - - key: ID - value: '3' - - key: Country - value: RU - - key: City - value: Moscow - - key: SSD - value: '1' - - attributes: - - key: ID - value: '4' - - key: Country - value: RU - - key: City - value: Moscow - - key: SSD - value: '1' -tests: - test: - policy: - replicas: - - count: 1 - selector: X - containerBackupFactor: 1 - selectors: - - name: X - count: 4 - clause: DISTINCT - filter: '*' - filters: [] - result: - - - 0 - - 1 - - 2 - - 3 diff --git a/client/src/test/resources/placement/selector_invalid.yml b/client/src/test/resources/placement/selector_invalid.yml deleted file mode 100644 index 9b0a539..0000000 --- a/client/src/test/resources/placement/selector_invalid.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: invalid selections -nodes: - - attributes: - - key: Country - value: Russia - - attributes: - - key: Country - value: Germany - - attributes: [] -tests: - missing filter: - policy: - replicas: - - count: 1 - selector: MyStore - containerBackupFactor: 1 - selectors: - - name: MyStore - count: 1 - clause: DISTINCT - attribute: Country - filter: FromNL - filters: - - name: FromRU - key: Country - op: EQ - value: Russia - filters: [] - error: filter not found - not enough nodes (filter results in empty set): - policy: - replicas: - - count: 1 - selector: MyStore - containerBackupFactor: 2 - selectors: - - name: MyStore - count: 2 - clause: DISTINCT - attribute: Country - filter: FromMoon - filters: - - name: FromMoon - key: Country - op: EQ - value: Moon - filters: [] diff --git a/client/src/test/resources/test_chain_raw.json b/client/src/test/resources/test_chain_raw.json new file mode 100644 index 0000000..60dfc3b --- /dev/null +++ b/client/src/test/resources/test_chain_raw.json @@ -0,0 +1,30 @@ +{ + "ID": "", + "Rules": [ + { + "Status": "Allow", + "Actions": { + "Inverted": false, + "Names": [ + "GetObject" + ] + }, + "Resources": { + "Inverted": false, + "Names": [ + "native:object/*" + ] + }, + "Any": false, + "Condition": [ + { + "Op": "StringEquals", + "Object": "Resource", + "Key": "Department", + "Value": "HR" + } + ] + } + ], + "MatchType": "DenyPriority" +} \ No newline at end of file diff --git a/cryptography/src/main/java/info/frostfs/sdk/Base58.java b/cryptography/src/main/java/info/frostfs/sdk/Base58.java new file mode 100644 index 0000000..d302763 --- /dev/null +++ b/cryptography/src/main/java/info/frostfs/sdk/Base58.java @@ -0,0 +1,141 @@ +package info.frostfs.sdk; + +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; + +import static info.frostfs.sdk.ArrayHelper.concat; +import static info.frostfs.sdk.Helper.getSha256; +import static info.frostfs.sdk.constants.ErrorConst.*; +import static java.util.Objects.isNull; + +public class Base58 { + public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + public static final int BASE58_SYMBOL_COUNT = 58; + public static final int BASE256_SYMBOL_COUNT = 256; + private static final int BYTE_DIVISION = 0xFF; + private static final char ENCODED_ZERO = ALPHABET[0]; + private static final char BASE58_ASCII_MAX_VALUE = 128; + private static final int[] INDEXES = new int[BASE58_ASCII_MAX_VALUE]; + + static { + Arrays.fill(INDEXES, -1); + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + private Base58() { + } + + public static byte[] base58CheckDecode(String input) { + if (StringUtils.isEmpty(input)) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + byte[] buffer = decode(input); + if (buffer.length < 4) { + throw new ProcessFrostFSException(String.format(DECODE_LENGTH_VALUE_TEMPLATE, buffer.length)); + } + + byte[] decode = Arrays.copyOfRange(buffer, 0, buffer.length - 4); + byte[] checksum = getSha256(getSha256(decode)); + var bufferEnd = Arrays.copyOfRange(buffer, buffer.length - 4, buffer.length); + var checksumStart = Arrays.copyOfRange(checksum, 0, 4); + if (!Arrays.equals(bufferEnd, checksumStart)) { + throw new ProcessFrostFSException(INVALID_CHECKSUM); + } + + return decode; + } + + public static String base58CheckEncode(byte[] data) { + if (isNull(data)) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + byte[] checksum = getSha256(getSha256(data)); + var buffer = concat(data, Arrays.copyOfRange(checksum, 0, 4)); + return encode(buffer); + } + + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input.length && input[zeros] == 0) { + ++zeros; + } + // Convert base-256 digits to base-58 digits (plus conversion to ASCII characters) + input = Arrays.copyOf(input, input.length); // since we modify it in-place + char[] encoded = new char[input.length * 2]; // upper bound + int outputStart = encoded.length; + for (int inputStart = zeros; inputStart < input.length; ) { + encoded[--outputStart] = ALPHABET[divmod(input, inputStart, BASE256_SYMBOL_COUNT, BASE58_SYMBOL_COUNT)]; + if (input[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Preserve exactly as many leading encoded zeros in output as there were leading zeros in input. + while (outputStart < encoded.length && encoded[outputStart] == ENCODED_ZERO) { + ++outputStart; + } + while (--zeros >= 0) { + encoded[--outputStart] = ENCODED_ZERO; + } + // Return encoded string (including encoded leading zeros). + return new String(encoded, outputStart, encoded.length - outputStart); + } + + public static byte[] decode(String input) { + if (input.isEmpty()) { + return new byte[0]; + } + // Convert the base58-encoded ASCII chars to a base58 byte sequence (base58 digits). + byte[] input58 = new byte[input.length()]; + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + int digit = c < BASE58_ASCII_MAX_VALUE ? INDEXES[c] : -1; + if (digit < 0) { + throw new ValidationFrostFSException(String.format(INVALID_BASE58_CHARACTER_TEMPLATE, (int) c)); + } + input58[i] = (byte) digit; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input58.length && input58[zeros] == 0) { + ++zeros; + } + // Convert base-58 digits to base-256 digits. + byte[] decoded = new byte[input.length()]; + int outputStart = decoded.length; + for (int inputStart = zeros; inputStart < input58.length; ) { + decoded[--outputStart] = divmod(input58, inputStart, BASE58_SYMBOL_COUNT, BASE256_SYMBOL_COUNT); + if (input58[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Ignore extra leading zeroes that were added during the calculation. + while (outputStart < decoded.length && decoded[outputStart] == 0) { + ++outputStart; + } + // Return decoded data (including original number of leading zeros). + return Arrays.copyOfRange(decoded, outputStart - zeros, decoded.length); + } + + private static byte divmod(byte[] number, int firstDigit, int base, int divisor) { + // this is just long division which accounts for the base of the input digits + int remainder = 0; + for (int i = firstDigit; i < number.length; i++) { + int digit = (int) number[i] & BYTE_DIVISION; + int temp = remainder * base + digit; + number[i] = (byte) (temp / divisor); + remainder = temp % divisor; + } + return (byte) remainder; + } +} diff --git a/cryptography/src/main/java/info/frostfs/sdk/Helper.java b/cryptography/src/main/java/info/frostfs/sdk/Helper.java index 0ca1dab..7ee18d2 100644 --- a/cryptography/src/main/java/info/frostfs/sdk/Helper.java +++ b/cryptography/src/main/java/info/frostfs/sdk/Helper.java @@ -4,6 +4,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Message; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.crypto.digests.RIPEMD160Digest; import java.math.BigInteger; import java.security.MessageDigest; @@ -14,11 +15,24 @@ import static java.util.Objects.isNull; public class Helper { private static final String SHA256 = "SHA-256"; + private static final int RIPEMD_160_HASH_BYTE_LENGTH = 20; private static final int HEX_RADIX = 16; private Helper() { } + public static byte[] getRipemd160(byte[] value) { + if (isNull(value)) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + var hash = new byte[RIPEMD_160_HASH_BYTE_LENGTH]; + var digest = new RIPEMD160Digest(); + digest.update(value, 0, value.length); + digest.doFinal(hash, 0); + return hash; + } + public static MessageDigest getSha256Instance() { try { return MessageDigest.getInstance(SHA256); diff --git a/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java b/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java index 248b717..f939c08 100644 --- a/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java +++ b/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java @@ -2,6 +2,7 @@ package info.frostfs.sdk; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.lang3.StringUtils; import org.bouncycastle.asn1.sec.SECNamedCurves; import org.bouncycastle.asn1.sec.SECObjectIdentifiers; import org.bouncycastle.asn1.x9.X9ECParameters; @@ -9,7 +10,12 @@ import org.bouncycastle.crypto.params.ECDomainParameters; import org.bouncycastle.crypto.params.ECPrivateKeyParameters; import org.bouncycastle.crypto.params.ECPublicKeyParameters; import org.bouncycastle.jce.spec.ECNamedCurveSpec; +import org.bouncycastle.math.ec.ECPoint; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -18,20 +24,63 @@ import java.security.spec.ECParameterSpec; import java.security.spec.ECPrivateKeySpec; import java.security.spec.ECPublicKeySpec; import java.security.spec.InvalidKeySpecException; +import java.util.Arrays; -import static info.frostfs.sdk.constants.ErrorConst.COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE; -import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING; +import static info.frostfs.sdk.Helper.getRipemd160; +import static info.frostfs.sdk.Helper.getSha256; +import static info.frostfs.sdk.constants.ErrorConst.*; import static java.util.Objects.isNull; import static org.bouncycastle.util.BigIntegers.fromUnsignedByteArray; public class KeyExtension { + public static final byte NEO_ADDRESS_VERSION = 0x35; private static final String CURVE_NAME = "secp256r1"; private static final String SECURITY_ALGORITHM = "EC"; + private static final int PS_IN_HASH160 = 0x0C; + private static final int DECODE_ADDRESS_LENGTH = 21; private static final int COMPRESSED_PUBLIC_KEY_LENGTH = 33; + private static final int UNCOMPRESSED_PUBLIC_KEY_LENGTH = 65; + private static final int CHECK_SIG_DESCRIPTOR = ByteBuffer + .wrap(getSha256("System.Crypto.CheckSig".getBytes(StandardCharsets.US_ASCII))) + .order(ByteOrder.LITTLE_ENDIAN).getInt(); private KeyExtension() { } + public static byte[] compress(byte[] publicKey) { + checkInputValue(publicKey); + if (publicKey.length != UNCOMPRESSED_PUBLIC_KEY_LENGTH) { + throw new ValidationFrostFSException(String.format( + UNCOMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, UNCOMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length + )); + } + + var secp256R1 = SECNamedCurves.getByOID(SECObjectIdentifiers.secp256r1); + var point = secp256R1.getCurve().decodePoint(publicKey); + return point.getEncoded(true); + } + + public static byte[] getPrivateKeyFromWIF(String wif) { + if (StringUtils.isEmpty(wif)) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + var data = Base58.base58CheckDecode(wif); + return Arrays.copyOfRange(data, 1, data.length - 1); + } + + public static byte[] loadPublicKey(byte[] privateKey) { + checkInputValue(privateKey); + + X9ECParameters params = SECNamedCurves.getByOID(SECObjectIdentifiers.secp256r1); + ECDomainParameters domain = new ECDomainParameters( + params.getCurve(), params.getG(), params.getN(), params.getH() + ); + ECPoint q = domain.getG().multiply(new BigInteger(1, privateKey)); + ECPublicKeyParameters publicParams = new ECPublicKeyParameters(q, domain); + return publicParams.getQ().getEncoded(true); + } + public static PrivateKey loadPrivateKey(byte[] privateKey) { checkInputValue(privateKey); @@ -85,6 +134,58 @@ public class KeyExtension { } } + public static byte[] getScriptHash(byte[] publicKey) { + checkInputValue(publicKey); + + var script = createSignatureRedeemScript(publicKey); + return getRipemd160(getSha256(script)); + } + + public static String publicKeyToAddress(byte[] publicKey) { + checkInputValue(publicKey); + if (publicKey.length != COMPRESSED_PUBLIC_KEY_LENGTH) { + throw new ValidationFrostFSException(String.format( + ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, COMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length + )); + } + + return toAddress(getScriptHash(publicKey)); + } + + private static String toAddress(byte[] scriptHash) { + checkInputValue(scriptHash); + byte[] data = new byte[DECODE_ADDRESS_LENGTH]; + data[0] = NEO_ADDRESS_VERSION; + System.arraycopy(scriptHash, 0, data, 1, scriptHash.length); + return Base58.base58CheckEncode(data); + } + + private static byte[] getBytes(int value) { + byte[] buffer = new byte[4]; + + for (int i = 0; i < buffer.length; i++) { + buffer[i] = (byte) (value >> i * Byte.SIZE); + } + + return buffer; + } + + private static byte[] createSignatureRedeemScript(byte[] publicKey) { + checkInputValue(publicKey); + if (publicKey.length != COMPRESSED_PUBLIC_KEY_LENGTH) { + throw new ValidationFrostFSException(String.format( + ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, COMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length + )); + } + + var script = new byte[]{PS_IN_HASH160, COMPRESSED_PUBLIC_KEY_LENGTH}; //PUSHDATA1 33 + + script = ArrayHelper.concat(script, publicKey); + script = ArrayHelper.concat(script, new byte[]{UNCOMPRESSED_PUBLIC_KEY_LENGTH}); //SYSCALL + script = ArrayHelper.concat(script, getBytes(CHECK_SIG_DESCRIPTOR)); //Neo_Crypto_CheckSig + return script; + } + private static void checkInputValue(byte[] data) { if (isNull(data) || data.length == 0) { throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); diff --git a/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java b/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java new file mode 100644 index 0000000..bebcae4 --- /dev/null +++ b/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java @@ -0,0 +1,56 @@ +package info.frostfs.sdk; + +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class Base58Test { + private static final String WIF = "L1YS4myg3xHPvi3FHeLaEt7G8upwJaWL5YLV7huviuUtXFpzBMqZ"; + private static final byte[] DECODE = new byte[]{ + -128, -128, -5, 30, -36, -118, 85, -67, -6, 81, 43, 93, -38, 106, 21, -88, 127, 15, 125, -79, -17, -40, 77, + -15, 122, -88, 72, 109, -47, 125, -80, -40, -38, 1 + }; + + @Test + void base58DecodeEncode() { + //When + Then + assertEquals(WIF, Base58.base58CheckEncode(Base58.base58CheckDecode(WIF))); + } + + @Test + void base58Decode_success() { + //When + var decode = Base58.base58CheckDecode(WIF); + + //Then + assertThat(decode).hasSize(34).containsExactly(DECODE); + } + + @Test + void base58Decode_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode(null)); + assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode("")); + assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode("WIF")); + assertThrows(ProcessFrostFSException.class, () -> Base58.base58CheckDecode("fh")); + } + + @Test + void base58Encode_success() { + //When + var encode = Base58.base58CheckEncode(DECODE); + + //Then + assertEquals(WIF, encode); + } + + @Test + void base58Encode_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckEncode(null)); + } +} diff --git a/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java b/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java index 00a9121..536e881 100644 --- a/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java +++ b/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java @@ -10,6 +10,38 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class HelperTest { + @Test + void getRipemd160_success() { + //Given + var value = new byte[]{1, 2, 3, 4, 5}; + var expected = new byte[] + {-21, -126, 92, 75, 36, -12, 37, 7, 122, 6, 124, -61, -66, -12, 87, 120, 63, 90, -41, 5}; + //When + var result = Helper.getRipemd160(value); + + //Then + assertThat(result).hasSize(20).containsExactly(expected); + } + + @Test + void getRipemd160_givenParamIsNull() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> Helper.getRipemd160(null)); + } + + @Test + void getRipemd160_givenParamsIsEmpty() { + //Given + var value = new byte[]{}; + var expected = new byte[] + {-100, 17, -123, -91, -59, -23, -4, 84, 97, 40, 8, -105, 126, -24, -11, 72, -78, 37, -115, 49}; + //When + var result = Helper.getRipemd160(value); + + //Then + assertThat(result).hasSize(20).containsExactly(expected); + } + @Test void getSha256Instance() { //When diff --git a/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java b/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java index a99fe4d..2f9b1b9 100644 --- a/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java +++ b/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java @@ -8,6 +8,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; public class KeyExtensionTest { + private static final String WIF = "L1YS4myg3xHPvi3FHeLaEt7G8upwJaWL5YLV7huviuUtXFpzBMqZ"; + private static final String OWNER_ID = "NVxUSpEEJzYXZZtUs18PrJTD9QZkLLNQ8S"; private static final byte[] PRIVATE_KEY = new byte[]{ -128, -5, 30, -36, -118, 85, -67, -6, 81, 43, 93, -38, 106, 21, -88, 127, 15, 125, -79, -17, -40, 77, -15, 122, -88, 72, 109, -47, 125, -80, -40, -38 @@ -22,6 +24,38 @@ public class KeyExtensionTest { -68, -73, 65, -57, -26, 75, 4, -51, -40, -20, 75, 89, -59, 111, 96, -80, 56, 13 }; + @Test + void getPrivateKeyFromWIF_success() { + //When + var privateKey = KeyExtension.getPrivateKeyFromWIF(WIF); + + //Then + assertThat(privateKey).hasSize(32).containsExactly(PRIVATE_KEY); + } + + @Test + void getPrivateKeyFromWIF_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPrivateKeyFromWIF("")); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPrivateKeyFromWIF(null)); + } + + @Test + void loadPublicKey_success() { + //When + var publicKey = KeyExtension.loadPublicKey(PRIVATE_KEY); + + //Then + assertThat(publicKey).hasSize(33).containsExactly(PUBLIC_KEY); + } + + @Test + void loadPublicKey_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.loadPublicKey(null)); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.loadPublicKey(new byte[]{})); + } + @Test void loadPrivateKey_success() { //When @@ -58,4 +92,61 @@ public class KeyExtensionTest { assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPublicKeyFromBytes(new byte[]{})); assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPublicKeyFromBytes(PRIVATE_KEY)); } + + @Test + void getScriptHash_success() { + //Given + var expected = new byte[]{ + 110, 42, -125, -76, -25, -44, -94, 22, -98, 117, -100, -5, 103, 74, -128, -51, 37, -116, -102, 71 + }; + + //When + var hash = KeyExtension.getScriptHash(PUBLIC_KEY); + + //Then + assertThat(hash).hasSize(20).containsExactly(expected); + } + + @Test + void getScriptHash_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(null)); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(new byte[]{})); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(PRIVATE_KEY)); + } + + @Test + void publicKeyToAddress_success() { + //When + var address = KeyExtension.publicKeyToAddress(PUBLIC_KEY); + + //Then + assertEquals(OWNER_ID, address); + } + + @Test + void publicKeyToAddress_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(null)); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(new byte[]{})); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(PRIVATE_KEY)); + } + + @Test + void compress_success() { + //When + var publicKey = KeyExtension.compress(UNCOMPRESSED_PUBLIC_KEY); + + //Then + assertThat(publicKey).hasSize(33).containsExactly(PUBLIC_KEY); + } + + @Test + void compress_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(null)); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(new byte[]{})); + assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(PUBLIC_KEY)); + } + } diff --git a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java index 8c248c9..629943e 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java +++ b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java @@ -20,7 +20,6 @@ public class ErrorConst { public static final String UNEXPECTED_MESSAGE_TYPE_TEMPLATE = "unexpected message type, expected %s, actually %s"; public static final String WIF_IS_INVALID = "WIF is invalid"; - public static final String WALLET_IS_INVALID = "wallet is not present"; public static final String UNEXPECTED_STREAM = "unexpected end of stream"; public static final String INVALID_HOST_TEMPLATE = "host %s has invalid format. Error: %s"; public static final String INVALID_RESPONSE = "invalid response"; @@ -29,7 +28,14 @@ public class ErrorConst { public static final String UNKNOWN_ENUM_VALUE_TEMPLATE = "unknown %s value: %s"; public static final String INPUT_PARAM_IS_NOT_SHA256 = "%s must be a sha256 hash"; + public static final String DECODE_LENGTH_VALUE_TEMPLATE = "decode array length must be >= 4, but %s"; + public static final String INVALID_BASE58_CHARACTER_TEMPLATE = "invalid character in Base58: 0x%04x"; + public static final String INVALID_CHECKSUM = "checksum does not match"; public static final String WRONG_SIGNATURE_SIZE_TEMPLATE = "wrong signature size. Expected length=%s, actual=%s"; + public static final String ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = + "publicKey isn't encoded compressed public key. Expected length=%s, actual=%s"; + public static final String UNCOMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = + "compress argument isn't uncompressed public key. Expected length=%s, actual=%s"; public static final String COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = "decompress argument isn't compressed public key. Expected length=%s, actual=%s"; @@ -46,30 +52,11 @@ public class ErrorConst { public static final String FIELDS_DELIMITER_COMMA = ", "; public static final String FIELDS_DELIMITER_OR = " or "; - public static final String UNSUPPORTED_MARSHALLER_VERSION_TEMPLATE = "unsupported marshaller version %s"; - public static final String UNSUPPORTED_CHAIN_VERSION_TEMPLATE = "unsupported chain version %s"; public static final String MARSHAL_SIZE_DIFFERS = "actual data size differs from expected"; - public static final String UNMARSHAL_SIZE_DIFFERS = "unmarshalled bytes left"; public static final String BYTES_ARE_OVER_FOR_SERIALIZE_TEMPLATE = "not enough bytes left to serialize value of type %s with length=%s"; - public static final String BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE = - "not enough bytes left to deserialize a value of type %s from offset=%s"; public static final String SLICE_IS_TOO_BIG_TEMPLATE = "slice size is too big=%s"; - public static final String SLICE_SIZE_IS_INVALID_TEMPLATE = "invalid slice size=%s"; public static final String STRING_IS_TOO_BIG_TEMPLATE = "string size is too big=%s"; - public static final String STRING_SIZE_IS_INVALID_TEMPLATE = "invalid string size=%s"; - - public static final String FILTER_NAME_IS_EMPTY = "Filter name for selector is empty"; - public static final String INVALID_FILTER_NAME_TEMPLATE = "filter name is invalid: '%s' is reserved"; - public static final String INVALID_FILTER_OPERATION_TEMPLATE = "invalid filter operation: %s"; - public static final String FILTER_NOT_FOUND = "filter not found"; - public static final String FILTER_NOT_FOUND_TEMPLATE = "filter not found: SELECT FROM '%s'"; - public static final String NON_EMPTY_FILTERS = "simple filter contains sub-filters"; - public static final String NOT_ENOUGH_NODES = "not enough nodes"; - public static final String NOT_ENOUGH_NODES_TEMPLATE = "not enough nodes to SELECT from '%s'"; - public static final String UNNAMED_TOP_FILTER = "unnamed top-level filter"; - public static final String VECTORS_IS_NULL = "vectors cannot be null"; - public static final String SELECTOR_NOT_FOUND_TEMPLATE = "selector not found: %s"; private ErrorConst() { } diff --git a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java index 58b0041..feb1ca5 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java @@ -1,7 +1,5 @@ package info.frostfs.sdk.constants; -import java.math.BigInteger; - public class AppConst { public static final String RESERVED_PREFIX = "__SYSTEM__"; @@ -17,8 +15,6 @@ public class AppConst { public static final int DEFAULT_GRPC_TIMEOUT = 5; public static final long DEFAULT_POLL_INTERVAL = 10; - public static final BigInteger UNSIGNED_LONG_MASK = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); - private AppConst() { } } diff --git a/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java index 964b3d4..214a9e3 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java @@ -5,17 +5,6 @@ import static info.frostfs.sdk.constants.AppConst.RESERVED_PREFIX; public class AttributeConst { public static final String DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE = RESERVED_PREFIX + "DISABLE_HOMOMORPHIC_HASHING"; - /* - * ATTRIBUTE_PRICE is a key to the node attribute that indicates - * the price in GAS tokens for storing one GB of data during one Epoch. - * */ - public static final String ATTRIBUTE_PRICE = "Price"; - - /* - * ATTRIBUTE_CAPACITY is a key to the node attribute that indicates the total available disk space in Gigabytes. - * */ - public static final String ATTRIBUTE_CAPACITY = "Capacity"; - private AttributeConst() { } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java b/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java index def97fd..393e3d0 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java @@ -1,8 +1,8 @@ package info.frostfs.sdk.dto.container; +import info.frostfs.sdk.Base58; import info.frostfs.sdk.constants.AppConst; import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java index a957dce..fe809fe 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.FilterOperation; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java deleted file mode 100644 index 286f77f..0000000 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java +++ /dev/null @@ -1,5 +0,0 @@ -package info.frostfs.sdk.dto.netmap; - -public interface Hasher { - long getHash(); -} diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java index a46cf2a..a3d28f8 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java @@ -1,31 +1,20 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.netmap.NodeState; +import info.frostfs.sdk.enums.NodeState; import lombok.Getter; -import org.apache.commons.codec.digest.MurmurHash3; -import java.math.BigInteger; import java.util.Collections; import java.util.List; import java.util.Map; -import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; -import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; -import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; -import static java.util.Objects.isNull; - @Getter -public class NodeInfo implements Hasher { +public class NodeInfo { private final NodeState state; private final Version version; private final List addresses; private final Map attributes; private final byte[] publicKey; - private long hash; - private BigInteger price = UNSIGNED_LONG_MASK; - - public NodeInfo(NodeState state, Version version, List addresses, Map attributes, byte[] publicKey) { this.state = state; @@ -34,26 +23,4 @@ public class NodeInfo implements Hasher { this.attributes = Collections.unmodifiableMap(attributes); this.publicKey = publicKey; } - - public long getHash() { - if (hash == 0) { - hash = MurmurHash3.hash128x64(publicKey, 0, publicKey.length, 0)[0]; - } - - return hash; - } - - public BigInteger getCapacity() { - var capacity = attributes.get(ATTRIBUTE_CAPACITY); - return isNull(capacity) ? BigInteger.valueOf(0) : new BigInteger(capacity); - } - - public BigInteger getPrice() { - if (price.equals(UNSIGNED_LONG_MASK)) { - var priceString = attributes.get(ATTRIBUTE_PRICE); - price = isNull(priceString) ? BigInteger.valueOf(0) : new BigInteger(priceString); - } - - return price; - } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java index 09d059d..5c28462 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java @@ -13,14 +13,6 @@ import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; public class Replica { private final int count; private final String selector; - private long ecDataCount; - private long ecParityCount; - - public Replica(int count, String selector, int ecDataCount, int ecParityCount) { - this(count, selector); - this.ecDataCount = Integer.toUnsignedLong(ecDataCount); - this.ecParityCount = Integer.toUnsignedLong(ecParityCount); - } public Replica(int count, String selector) { if (count <= 0) { @@ -40,11 +32,8 @@ public class Replica { ); } + this.count = count; this.selector = EMPTY_STRING; } - - public int getCountNodes() { - return count != 0 ? count : (int) (ecDataCount + ecParityCount); - } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java index 4dee15c..71197a1 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java @@ -1,17 +1,15 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.enums.SelectorClause; import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.Setter; @Getter -@Setter @AllArgsConstructor public class Selector { private final String name; - private int count; - private SelectorClause clause; - private String attribute; - private String filter; + private final int count; + private final SelectorClause clause; + private final String attribute; + private final String filter; } diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java b/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java index 35e9ab2..7f34f87 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java @@ -1,8 +1,8 @@ package info.frostfs.sdk.dto.object; +import info.frostfs.sdk.Base58; import info.frostfs.sdk.constants.AppConst; import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java b/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java index 82886e1..ba65652 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java @@ -1,11 +1,14 @@ package info.frostfs.sdk.dto.object; +import info.frostfs.sdk.Base58; import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import static info.frostfs.sdk.KeyExtension.publicKeyToAddress; +import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING; import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING_TEMPLATE; +import static java.util.Objects.isNull; @Getter public class OwnerId { @@ -21,6 +24,14 @@ public class OwnerId { this.value = value; } + public OwnerId(byte[] publicKey) { + if (isNull(publicKey) || publicKey.length == 0) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + this.value = publicKeyToAddress(publicKey); + } + public byte[] toHash() { return Base58.decode(value); } diff --git a/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java b/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java new file mode 100644 index 0000000..52bab99 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java @@ -0,0 +1,33 @@ +package info.frostfs.sdk.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum BasicAcl { + PRIVATE(0x1C8C8CCC), + PUBLIC_RO(0x1FBF8CFF), + PUBLIC_RW(0x1FBFBFFF), + PUBLIC_APPEND(0x1FBF9FFF), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (BasicAcl basicAcl : BasicAcl.values()) { + map.put(basicAcl.value, basicAcl); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + BasicAcl(int value) { + this.value = value; + } + + public static BasicAcl get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java b/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java index 9f1bc0e..a437254 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java +++ b/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java @@ -1,31 +1,13 @@ package info.frostfs.sdk.enums; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - public enum ConditionKindType { RESOURCE(0), REQUEST(1), ; - private static final Map ENUM_MAP_BY_VALUE; - - static { - Map map = new HashMap<>(); - for (ConditionKindType conditionKindType : ConditionKindType.values()) { - map.put(conditionKindType.value, conditionKindType); - } - ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); - } - public final int value; ConditionKindType(int value) { this.value = value; } - - public static ConditionKindType get(int value) { - return ENUM_MAP_BY_VALUE.get(value); - } } diff --git a/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java b/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java index dca427d..b23417c 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java +++ b/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java @@ -1,9 +1,5 @@ package info.frostfs.sdk.enums; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - public enum ConditionType { COND_STRING_EQUALS(0), COND_STRING_NOT_EQUALS(1), @@ -32,23 +28,9 @@ public enum ConditionType { COND_NOT_IP_ADDRESS(18), ; - private static final Map ENUM_MAP_BY_VALUE; - - static { - Map map = new HashMap<>(); - for (ConditionType conditionType : ConditionType.values()) { - map.put(conditionType.value, conditionType); - } - ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); - } - public final int value; ConditionType(int value) { this.value = value; } - - public static ConditionType get(int value) { - return ENUM_MAP_BY_VALUE.get(value); - } } diff --git a/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java b/models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java similarity index 95% rename from models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java rename to models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java index 69513c8..f49b0f2 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java +++ b/models/src/main/java/info/frostfs/sdk/enums/FilterOperation.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums.netmap; +package info.frostfs.sdk.enums; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java b/models/src/main/java/info/frostfs/sdk/enums/NodeState.java similarity index 94% rename from models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java rename to models/src/main/java/info/frostfs/sdk/enums/NodeState.java index ac6ae78..3f833e3 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java +++ b/models/src/main/java/info/frostfs/sdk/enums/NodeState.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums.netmap; +package info.frostfs.sdk.enums; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java b/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java index 6bbbe82..bfacaf6 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java +++ b/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java @@ -1,9 +1,5 @@ package info.frostfs.sdk.enums; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - public enum RuleMatchType { // DENY_PRIORITY rejects the request if any `Deny` is specified. DENY_PRIORITY(0), @@ -12,23 +8,9 @@ public enum RuleMatchType { FIRST_MATCH(1), ; - private static final Map ENUM_MAP_BY_VALUE; - - static { - Map map = new HashMap<>(); - for (RuleMatchType ruleMatchType : RuleMatchType.values()) { - map.put(ruleMatchType.value, ruleMatchType); - } - ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); - } - public final int value; RuleMatchType(int value) { this.value = value; } - - public static RuleMatchType get(int value) { - return ENUM_MAP_BY_VALUE.get(value); - } } diff --git a/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java b/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java index 45289d0..b2f6cf1 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java +++ b/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java @@ -1,9 +1,5 @@ package info.frostfs.sdk.enums; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - public enum RuleStatus { ALLOW(0), NO_RULE_FOUND(1), @@ -11,23 +7,9 @@ public enum RuleStatus { QUOTA_LIMIT_REACHED(3), ; - private static final Map ENUM_MAP_BY_VALUE; - - static { - Map map = new HashMap<>(); - for (RuleStatus ruleStatus : RuleStatus.values()) { - map.put(ruleStatus.value, ruleStatus); - } - ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); - } - public final int value; RuleStatus(int value) { this.value = value; } - - public static RuleStatus get(int value) { - return ENUM_MAP_BY_VALUE.get(value); - } } diff --git a/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java b/models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java similarity index 94% rename from models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java rename to models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java index f375f88..b10ff0c 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java +++ b/models/src/main/java/info/frostfs/sdk/enums/SelectorClause.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums.netmap; +package info.frostfs.sdk.enums; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java index 0c96e32..0571ff0 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java @@ -3,6 +3,8 @@ package info.frostfs.sdk.mappers.container; import com.google.protobuf.ByteString; import frostfs.container.Types; import info.frostfs.sdk.dto.container.Container; +import info.frostfs.sdk.enums.BasicAcl; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.mappers.netmap.PlacementPolicyMapper; import info.frostfs.sdk.mappers.netmap.VersionMapper; import info.frostfs.sdk.mappers.object.OwnerIdMapper; @@ -12,6 +14,7 @@ import java.util.stream.Collectors; import static info.frostfs.sdk.UuidExtension.asBytes; import static info.frostfs.sdk.UuidExtension.asUuid; +import static info.frostfs.sdk.constants.ErrorConst.UNKNOWN_ENUM_VALUE_TEMPLATE; import static java.util.Objects.isNull; public class ContainerMapper { @@ -48,6 +51,13 @@ public class ContainerMapper { return null; } + var basicAcl = BasicAcl.get(containerGrpc.getBasicAcl()); + if (isNull(basicAcl)) { + throw new ProcessFrostFSException( + String.format(UNKNOWN_ENUM_VALUE_TEMPLATE, BasicAcl.class.getName(), containerGrpc.getBasicAcl()) + ); + } + var attributes = containerGrpc.getAttributesList().stream() .collect(Collectors.toMap(Types.Container.Attribute::getKey, Types.Container.Attribute::getValue)); diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java index 529b26d..3f63b3b 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Filter; -import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.FilterOperation; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.collections4.CollectionUtils; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java index c28cd35..f319583 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java @@ -4,7 +4,7 @@ import frostfs.netmap.Service; import frostfs.netmap.Types; import frostfs.netmap.Types.NodeInfo.Attribute; import info.frostfs.sdk.dto.netmap.NodeInfo; -import info.frostfs.sdk.enums.netmap.NodeState; +import info.frostfs.sdk.enums.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import java.util.stream.Collectors; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java index 3ac0105..959b9d6 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java @@ -36,17 +36,15 @@ public class ReplicaMapper { return Types.Replica.newBuilder() .setCount(replica.getCount()) .setSelector(replica.getSelector()) - .setEcDataCount((int) replica.getEcDataCount()) - .setEcParityCount((int) replica.getEcParityCount()) .build(); } - public static Replica[] toModels(List replicas) { - if (CollectionUtils.isEmpty(replicas)) { + public static Replica[] toModels(List filters) { + if (CollectionUtils.isEmpty(filters)) { return null; } - return replicas.stream().map(ReplicaMapper::toModel).toArray(Replica[]::new); + return filters.stream().map(ReplicaMapper::toModel).toArray(Replica[]::new); } public static Replica toModel(Types.Replica replica) { @@ -54,11 +52,6 @@ public class ReplicaMapper { return null; } - return new Replica( - replica.getCount(), - replica.getSelector(), - replica.getEcDataCount(), - replica.getEcParityCount() - ); + return new Replica(replica.getCount(), replica.getSelector()); } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java index 462519a..8b9067d 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Selector; -import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.enums.SelectorClause; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.collections4.CollectionUtils; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java index 535905f..e1047b8 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java @@ -2,8 +2,8 @@ package info.frostfs.sdk.mappers.object; import com.google.protobuf.ByteString; import frostfs.refs.Types; +import info.frostfs.sdk.Base58; import info.frostfs.sdk.dto.object.OwnerId; -import io.neow3j.crypto.Base58; import static java.util.Objects.isNull; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java index c390e40..fadde6a 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java @@ -7,8 +7,12 @@ import info.frostfs.sdk.dto.netmap.PlacementPolicy; import info.frostfs.sdk.dto.netmap.Replica; import info.frostfs.sdk.dto.netmap.Version; import info.frostfs.sdk.dto.object.OwnerId; +import info.frostfs.sdk.enums.BasicAcl; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.mappers.object.OwnerIdMapper; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.UUID; @@ -102,8 +106,9 @@ public class ContainerMapperTest { assertNull(ContainerMapper.toGrpcMessage(null)); } - @Test - void toModel_success() { + @ParameterizedTest + @EnumSource(value = BasicAcl.class) + void toModel_success(BasicAcl basicAcl) { //Given var version = frostfs.refs.Types.Version.newBuilder() .setMajor(1) @@ -132,6 +137,7 @@ public class ContainerMapperTest { .build(); var container = Types.Container.newBuilder() + .setBasicAcl(basicAcl.value) .setNonce(ByteString.copyFrom(asBytes(UUID.randomUUID()))) .setVersion(version) .setPlacementPolicy(placementPolicy) @@ -172,4 +178,15 @@ public class ContainerMapperTest { assertNull(ContainerMapper.toModel(null)); assertNull(ContainerMapper.toModel(Types.Container.getDefaultInstance())); } + + @Test + void toModel_notValid() { + //Given + var container = Types.Container.newBuilder() + .setBasicAcl(-1) + .build(); + + //When + Then + assertThrows(ProcessFrostFSException.class, () -> ContainerMapper.toModel(container)); + } } diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java index 788d71e..4b1649d 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Filter; -import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.FilterOperation; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.junit.jupiter.api.Test; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java index dd2dc1f..e2b6c29 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java @@ -3,7 +3,7 @@ package info.frostfs.sdk.mappers.netmap; import com.google.protobuf.ByteString; import frostfs.netmap.Service; import frostfs.netmap.Types; -import info.frostfs.sdk.enums.netmap.NodeState; +import info.frostfs.sdk.enums.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java index 6d46553..4e55ae0 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java @@ -2,7 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Selector; -import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.enums.SelectorClause; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.junit.jupiter.api.Test; diff --git a/pom.xml b/pom.xml index d8de384..8f58e1f 100644 --- a/pom.xml +++ b/pom.xml @@ -17,7 +17,7 @@ - 0.12.0 + 0.7.0 11 11 @@ -28,7 +28,6 @@ 3.26.3 1.18.34 3.23.0 - 3.23.0 @@ -42,11 +41,6 @@ commons-lang3 3.14.0 - - commons-codec - commons-codec - 1.18.0 - org.projectlombok lombok @@ -54,11 +48,6 @@ provided true - - io.neow3j - contract - ${neow3j.version} - org.junit.jupiter junit-jupiter @@ -83,11 +72,6 @@ ${mockito.version} test - - org.yaml - snakeyaml - 2.4 - @@ -170,20 +154,4 @@ - - - TrueCloudLab - https://git.frostfs.info/api/packages/TrueCloudLab/maven - - - - - TrueCloudLab - https://git.frostfs.info/api/packages/TrueCloudLab/maven - - - TrueCloudLab - https://git.frostfs.info/api/packages/TrueCloudLab/maven - - - + \ No newline at end of file