diff --git a/.forgejo/workflows/publish.yml b/.forgejo/workflows/publish.yml new file mode 100644 index 0000000..7c067ca --- /dev/null +++ b/.forgejo/workflows/publish.yml @@ -0,0 +1,22 @@ +on: + push: + workflow_dispatch: + +jobs: + image: + name: Publish Maven packages + runs-on: docker + container: git.frostfs.info/truecloudlab/env:openjdk-11-maven-3.8.6 + steps: + - name: Clone git repo + uses: actions/checkout@v3 + + - name: Publish release packages + run: mvn clean --batch-mode --update-snapshots deploy + if: >- + startsWith(github.ref, 'refs/tags/v') && + (github.event_name == 'workflow_dispatch' || github.event_name == 'push') + env: + MAVEN_REGISTRY: TrueCloudLab + MAVEN_REGISTRY_USER: ${{secrets.MAVEN_REGISTRY_USER}} + MAVEN_REGISTRY_PASSWORD: ${{secrets.MAVEN_REGISTRY_PASSWORD}} diff --git a/.gitignore b/.gitignore index c3f0616..82d73ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ target/ !.mvn/wrapper/maven-wrapper.jar !**/src/main/**/target/ !**/src/test/**/target/ +**/.flattened-pom.xml ### IntelliJ IDEA ### .idea/modules.xml diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e859051 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,62 @@ +# Changelog + +## [0.12.0] - 2025-04-24 + +### Fixed + +- Patch logic +- Patch payload requirements + +## [0.11.0] - 2025-04-23 + +### Added + +- Placement policy vectors + +## [0.10.0] - 2025-03-10 + +### Added + +- Auto deploy to forgejo + +## [0.9.0] - 2025-03-05 + +### Added + +- APE rule deserializer + +## [0.9.0] - 2025-03-05 + +### Added + +- APE rule deserializer + +## [0.8.0] - 2025-03-04 + +### Added + +- Creating client via wallet and password + +## [0.7.0] - 2025-02-20 + +### Added + +- Expanding the parameters for creating a container + +### Fixed + +- Creating a session for working with objects + +## [0.6.0] - 2025-02-13 + +### Added + +- APE rules serializer + +## [0.5.0] - 2025-02-11 + +### Fixed + +- Loading large objects in chunks +- .gitignore +- pom revision \ No newline at end of file diff --git a/README.md b/README.md index 737c204..991c5fa 100644 --- a/README.md +++ b/README.md @@ -21,31 +21,39 @@ neo-go wallet export -w -d ### Container operations ```java +import info.frostfs.sdk.FrostFSClient; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.netmap.PlacementPolicy; import info.frostfs.sdk.dto.netmap.Replica; -import info.frostfs.sdk.enums.BasicAcl; import info.frostfs.sdk.jdo.ClientSettings; -import info.frostfs.sdk.FrostFSClient; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerDelete; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGet; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGetAll; public class ContainerExample { public void example() { + var callContext = new CallContext(); ClientSettings clientSettings = new ClientSettings(, ); FrostFSClient frostFSClient = new FrostFSClient(clientSettings); // Create container - var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(1)}, Boolean.TRUE); - var containerId = frostFSClient.createContainer(new Container(BasicAcl.PUBLIC_RW, placementPolicy)); + var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(3)}, true, 1); + var prmContainerCreate = new PrmContainerCreate(new Container(placementPolicy)); + var containerId = frostFSClient.createContainer(prmContainerCreate, callContext); // Get container - var container = frostFSClient.getContainer(containerId); + var prmContainerGet = new PrmContainerGet(containerId); + var container = frostFSClient.getContainer(prmContainerGet, callContext); // List containers - var containerIds = frostFSClient.listContainers(); + var containerIds = frostFSClient.listContainers(new PrmContainerGetAll(), callContext); // Delete container - frostFSClient.deleteContainer(containerId); + var prmContainerDelete = new PrmContainerDelete(containerId); + frostFSClient.deleteContainer(prmContainerDelete, callContext); } } ``` @@ -53,45 +61,104 @@ public class ContainerExample { ### Object operations ```java +import info.frostfs.sdk.dto.object.*; import info.frostfs.sdk.enums.ObjectType; -import info.frostfs.sdk.dto.container.ContainerId; -import info.frostfs.sdk.dto.object.ObjectAttribute; -import info.frostfs.sdk.dto.object.ObjectFilter; -import info.frostfs.sdk.dto.object.ObjectHeader; -import info.frostfs.sdk.dto.object.ObjectId; -import info.frostfs.sdk.jdo.PutObjectParameters; -import info.frostfs.sdk.FrostFSClient; +import info.frostfs.sdk.jdo.ClientSettings; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.object.*; +import org.apache.commons.lang3.ArrayUtils; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import static java.util.Objects.isNull; + public class ObjectExample { public void example() { + CallContext callContext = new CallContext(); ClientSettings clientSettings = new ClientSettings(, ); FrostFSClient frostFSClient = new FrostFSClient(clientSettings); // Put object ObjectId objectId; - try (FileInputStream fis = new FileInputStream("cat.jpg")) { - var cat = new ObjectHeader( - containerId, ObjectType.REGULAR, new ObjectAttribute[]{new ObjectAttribute("Filename", "cat.jpg")} - ); + try (FileInputStream file = new FileInputStream("/path/to/file/cat.jpg")) { + var attribute = new ObjectAttribute("Filename", "cat.jpg"); + var cat = new ObjectHeader(containerId, ObjectType.REGULAR, attribute); + var prmObjectPut = PrmObjectPut.builder().objectHeader(cat).build(); + var writer = frostFSClient.putObject(prmObjectPut, callContext); - var params = new PutObjectParameters(cat, fis); - objectId = frostFSClient.putObject(params); + writer.write(file.readAllBytes()); + objectId = writer.complete(); } catch (IOException e) { throw new RuntimeException(e); } // Get object - var obj = frostFSClient.getObject(containerId, objectId); + var prmObjectGet = new PrmObjectGet(containerId, oid); + ObjectFrostFS object = frostFSClient.getObject(prmObjectGet, callContext); + + var reader = object.getObjectReader(); + var chunk = reader.readChunk(); + var length = chunk.length; + byte[] buffer = null; + while (length > 0) { + buffer = isNull(buffer) ? chunk : ArrayHelper.concat(buffer, chunk); + + chunk = object.getObjectReader().readChunk(); + length = ArrayUtils.isEmpty(chunk) ? 0 : chunk.length; + } + + try (FileOutputStream fos = new FileOutputStream("/path/to/file/newCat.jpg")) { + fos.write(buffer); + } catch (Exception ignored) { + } // Get object header - var objectHeader = frostFSClient.getObjectHead(containerId, objectId); + var prmObjectHeadGet = new PrmObjectHeadGet(containerId, objectId); + var objectHeader = frostFSClient.getObjectHead(prmObjectHeadGet, callContext); // Search regular objects - var objectIds = frostFSClient.searchObjects(containerId, new ObjectFilter.FilterByRootObject()); + var prmObjectSearch = new PrmObjectSearch(containerId, new ObjectFilter.FilterByRootObject()); + var objectIds = frostFSClient.searchObjects(prmObjectSearch, callContext); + + // Delete object + var prmObjectDelete = new PrmObjectDelete(containerId, objectId); + frostFSClient.deleteObject(prmObjectDelete, callContext); + } +} +``` + +### Pool init + +```java +import info.frostfs.sdk.jdo.ECDsa; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.pool.NodeParameters; +import info.frostfs.sdk.jdo.pool.PoolInitParameters; +import info.frostfs.sdk.pool.Pool; + +public class PoolExample { + + public static void example() { + CallContext callContext = new CallContext(); + + //Init + var nodeParam1 = new NodeParameters(1, , 1); + var nodeParam2 = new NodeParameters(1, , 1); + var nodeParam3 = new NodeParameters(1, , 1); + var nodeParam4 = new NodeParameters(1, , 1); + + PoolInitParameters initParameters = new PoolInitParameters(); + initParameters.setKey(new ECDsa()); + initParameters.setNodeParams(new NodeParameters[]{nodeParam1, nodeParam2, nodeParam3, nodeParam4}); + + + Pool pool = new Pool(initParameters); + + //Dial (Required!) + pool.dial(callContext); } } ``` \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml index ebe929e..1bb24e7 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -76,7 +76,7 @@ - + diff --git a/client/pom.xml b/client/pom.xml index a3f7ff7..2d87b21 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} client @@ -21,17 +21,17 @@ info.frostfs.sdk cryptography - 0.1.0 + ${revision} info.frostfs.sdk models - 0.1.0 + ${revision} info.frostfs.sdk exceptions - 0.1.0 + ${revision} commons-codec @@ -54,5 +54,10 @@ simpleclient_common 0.16.0 + + org.slf4j + slf4j-api + 2.0.16 + \ No newline at end of file diff --git a/client/src/main/java/info/frostfs/sdk/FrostFSClient.java b/client/src/main/java/info/frostfs/sdk/FrostFSClient.java index 634c956..4aa685c 100644 --- a/client/src/main/java/info/frostfs/sdk/FrostFSClient.java +++ b/client/src/main/java/info/frostfs/sdk/FrostFSClient.java @@ -1,13 +1,12 @@ package info.frostfs.sdk; -import info.frostfs.sdk.dto.chain.Chain; -import info.frostfs.sdk.dto.chain.ChainTarget; +import frostfs.accounting.Types; +import info.frostfs.sdk.dto.ape.Chain; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.container.ContainerId; import info.frostfs.sdk.dto.netmap.NetmapSnapshot; import info.frostfs.sdk.dto.netmap.NodeInfo; import info.frostfs.sdk.dto.netmap.Version; -import info.frostfs.sdk.dto.object.ObjectFilter; import info.frostfs.sdk.dto.object.ObjectFrostFS; import info.frostfs.sdk.dto.object.ObjectHeader; import info.frostfs.sdk.dto.object.ObjectId; @@ -15,15 +14,35 @@ import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; import info.frostfs.sdk.jdo.ClientSettings; +import info.frostfs.sdk.jdo.ECDsa; import info.frostfs.sdk.jdo.NetworkSettings; -import info.frostfs.sdk.jdo.PutObjectParameters; -import info.frostfs.sdk.services.*; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainRemove; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerDelete; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGet; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGetAll; +import info.frostfs.sdk.jdo.parameters.object.*; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmObjectPatch; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeGet; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeHashGet; +import info.frostfs.sdk.jdo.parameters.session.PrmSessionCreate; +import info.frostfs.sdk.jdo.result.ObjectHeaderResult; +import info.frostfs.sdk.pool.SessionCache; +import info.frostfs.sdk.pool.WrapperPrm; +import info.frostfs.sdk.services.CommonClient; import info.frostfs.sdk.services.impl.*; import info.frostfs.sdk.services.impl.interceptor.Configuration; import info.frostfs.sdk.services.impl.interceptor.MonitoringClientInterceptor; +import info.frostfs.sdk.services.impl.rwhelper.ObjectWriter; +import info.frostfs.sdk.services.impl.rwhelper.RangeReader; import info.frostfs.sdk.utils.Validator; import io.grpc.Channel; import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannel; +import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -31,26 +50,32 @@ import static info.frostfs.sdk.constants.ErrorConst.VERSION_UNSUPPORTED_TEMPLATE import static info.frostfs.sdk.tools.GrpcClient.initGrpcChannel; import static java.util.Objects.nonNull; -public class FrostFSClient - implements ContainerClient, ObjectClient, ApeManagerClient, NetmapClient, SessionClient, ToolsClient { +public class FrostFSClient implements CommonClient { + private static final MonitoringClientInterceptor MONITORING_CLIENT_INTERCEPTOR = + MonitoringClientInterceptor.create(Configuration.allMetrics()); + private final ContainerClientImpl containerClientImpl; private final ObjectClientImpl objectClientImpl; private final ApeManagerClientImpl apeManagerClient; private final NetmapClientImpl netmapClientImpl; private final SessionClientImpl sessionClientImpl; private final ObjectToolsImpl objectToolsImpl; + private final AccountingClientImpl accountingClient; + private final ManagedChannel channel; public FrostFSClient(ClientSettings clientSettings) { Validator.validate(clientSettings); - Channel channel = nonNull(clientSettings.getChannel()) + this.channel = nonNull(clientSettings.getChannel()) ? clientSettings.getChannel() : initGrpcChannel(clientSettings); - MonitoringClientInterceptor monitoringClientInterceptor = MonitoringClientInterceptor - .create(Configuration.allMetrics()); - channel = ClientInterceptors.intercept(channel, monitoringClientInterceptor); - ClientEnvironment clientEnvironment = - new ClientEnvironment(clientSettings.getKey(), channel, new Version(), this); + var ecdsa = StringUtils.isBlank(clientSettings.getWif()) + ? new ECDsa(clientSettings.getWallet(), clientSettings.getPassword()) + : new ECDsa(clientSettings.getWif()); + Channel interceptChannel = ClientInterceptors.intercept(channel, MONITORING_CLIENT_INTERCEPTOR); + ClientEnvironment clientEnvironment = new ClientEnvironment( + ecdsa, interceptChannel, new Version(), this, new SessionCache(0) + ); Validator.validate(clientEnvironment); @@ -60,11 +85,31 @@ public class FrostFSClient this.netmapClientImpl = new NetmapClientImpl(clientEnvironment); this.sessionClientImpl = new SessionClientImpl(clientEnvironment); this.objectToolsImpl = new ObjectToolsImpl(clientEnvironment); - checkFrostFsVersionSupport(clientEnvironment.getVersion()); + this.accountingClient = new AccountingClientImpl(clientEnvironment); + checkFrostFSVersionSupport(clientEnvironment.getVersion()); } - private void checkFrostFsVersionSupport(Version version) { - var localNodeInfo = netmapClientImpl.getLocalNodeInfo(); + public FrostFSClient(WrapperPrm prm, SessionCache cache) { + this.channel = initGrpcChannel(prm.getAddress()); + + Channel interceptChannel = ClientInterceptors.intercept(channel, MONITORING_CLIENT_INTERCEPTOR); + ClientEnvironment clientEnvironment = + new ClientEnvironment(prm.getKey(), interceptChannel, new Version(), this, cache); + + Validator.validate(clientEnvironment); + + this.containerClientImpl = new ContainerClientImpl(clientEnvironment); + this.objectClientImpl = new ObjectClientImpl(clientEnvironment); + this.apeManagerClient = new ApeManagerClientImpl(clientEnvironment); + this.netmapClientImpl = new NetmapClientImpl(clientEnvironment); + this.sessionClientImpl = new SessionClientImpl(clientEnvironment); + this.objectToolsImpl = new ObjectToolsImpl(clientEnvironment); + this.accountingClient = new AccountingClientImpl(clientEnvironment); + checkFrostFSVersionSupport(clientEnvironment.getVersion()); + } + + private void checkFrostFSVersionSupport(Version version) { + var localNodeInfo = netmapClientImpl.getLocalNodeInfo(new CallContext()); if (!localNodeInfo.getVersion().isSupported(version)) { throw new ProcessFrostFSException( String.format(VERSION_UNSUPPORTED_TEMPLATE, localNodeInfo.getVersion()) @@ -73,96 +118,131 @@ public class FrostFSClient } @Override - public Container getContainer(ContainerId cid) { - return containerClientImpl.getContainer(cid); + public Container getContainer(PrmContainerGet args, CallContext ctx) { + return containerClientImpl.getContainer(args, ctx); } @Override - public List listContainers() { - return containerClientImpl.listContainers(); + public List listContainers(PrmContainerGetAll args, CallContext ctx) { + return containerClientImpl.listContainers(args, ctx); } @Override - public ContainerId createContainer(Container container) { - return containerClientImpl.createContainer(container); + public ContainerId createContainer(PrmContainerCreate args, CallContext ctx) { + return containerClientImpl.createContainer(args, ctx); } @Override - public void deleteContainer(ContainerId cid) { - containerClientImpl.deleteContainer(cid); + public void deleteContainer(PrmContainerDelete args, CallContext ctx) { + containerClientImpl.deleteContainer(args, ctx); } @Override - public ObjectHeader getObjectHead(ContainerId containerId, ObjectId objectId) { - return objectClientImpl.getObjectHead(containerId, objectId); + public ObjectHeaderResult getObjectHead(PrmObjectHeadGet args, CallContext ctx) { + return objectClientImpl.getObjectHead(args, ctx); } @Override - public ObjectFrostFS getObject(ContainerId containerId, ObjectId objectId) { - return objectClientImpl.getObject(containerId, objectId); + public ObjectFrostFS getObject(PrmObjectGet args, CallContext ctx) { + return objectClientImpl.getObject(args, ctx); } @Override - public ObjectId putObject(PutObjectParameters parameters) { - return objectClientImpl.putObject(parameters); + public ObjectWriter putObject(PrmObjectPut args, CallContext ctx) { + return objectClientImpl.putObject(args, ctx); } @Override - public ObjectId putSingleObject(ObjectFrostFS objectFrostFS) { - return objectClientImpl.putSingleObject(objectFrostFS); + public ObjectId putClientCutObject(PrmObjectClientCutPut args, CallContext ctx) { + return objectClientImpl.putClientCutObject(args, ctx); } @Override - public void deleteObject(ContainerId containerId, ObjectId objectId) { - objectClientImpl.deleteObject(containerId, objectId); + public ObjectId putSingleObject(PrmObjectSinglePut args, CallContext ctx) { + return objectClientImpl.putSingleObject(args, ctx); } @Override - public Iterable searchObjects(ContainerId cid, ObjectFilter... filters) { - return objectClientImpl.searchObjects(cid, filters); + public void deleteObject(PrmObjectDelete args, CallContext ctx) { + objectClientImpl.deleteObject(args, ctx); } @Override - public byte[] addChain(Chain chain, ChainTarget chainTarget) { - return apeManagerClient.addChain(chain, chainTarget); + public Iterable searchObjects(PrmObjectSearch args, CallContext ctx) { + return objectClientImpl.searchObjects(args, ctx); } @Override - public void removeChain(Chain chain, ChainTarget chainTarget) { - apeManagerClient.removeChain(chain, chainTarget); + public RangeReader getRange(PrmRangeGet args, CallContext ctx) { + return objectClientImpl.getRange(args, ctx); } @Override - public List listChains(ChainTarget chainTarget) { - return apeManagerClient.listChains(chainTarget); + public byte[][] getRangeHash(PrmRangeHashGet args, CallContext ctx) { + return objectClientImpl.getRangeHash(args, ctx); } @Override - public NetmapSnapshot getNetmapSnapshot() { - return netmapClientImpl.getNetmapSnapshot(); + public ObjectId patchObject(PrmObjectPatch args, CallContext ctx) { + return objectClientImpl.patchObject(args, ctx); } @Override - public NodeInfo getLocalNodeInfo() { - return netmapClientImpl.getLocalNodeInfo(); + public byte[] addChain(PrmApeChainAdd args, CallContext ctx) { + return apeManagerClient.addChain(args, ctx); } @Override - public NetworkSettings getNetworkSettings() { - return netmapClientImpl.getNetworkSettings(); + public void removeChain(PrmApeChainRemove args, CallContext ctx) { + apeManagerClient.removeChain(args, ctx); } @Override - public SessionToken createSession(long expiration) { - return sessionClientImpl.createSession(expiration); + public List listChains(PrmApeChainList args, CallContext ctx) { + return apeManagerClient.listChains(args, ctx); } - public frostfs.session.Types.SessionToken createSessionInternal(long expiration) { - return sessionClientImpl.createSessionInternal(expiration); + @Override + public NetmapSnapshot getNetmapSnapshot(CallContext ctx) { + return netmapClientImpl.getNetmapSnapshot(ctx); + } + + @Override + public NodeInfo getLocalNodeInfo(CallContext ctx) { + return netmapClientImpl.getLocalNodeInfo(ctx); + } + + @Override + public NetworkSettings getNetworkSettings(CallContext ctx) { + return netmapClientImpl.getNetworkSettings(ctx); + } + + @Override + public SessionToken createSession(PrmSessionCreate args, CallContext ctx) { + return sessionClientImpl.createSession(args, ctx); + } + + public frostfs.session.Types.SessionToken createSessionInternal(PrmSessionCreate args, CallContext ctx) { + return sessionClientImpl.createSessionInternal(args, ctx); } @Override public ObjectId calculateObjectId(ObjectHeader header) { return objectToolsImpl.calculateObjectId(header); } + + @Override + public Types.Decimal getBalance(CallContext ctx) { + return accountingClient.getBalance(ctx); + } + + @Override + public String dial(CallContext ctx) { + accountingClient.getBalance(ctx); + return null; + } + + public void close() { + channel.shutdown(); + } } diff --git a/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java b/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java new file mode 100644 index 0000000..eb3fe48 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/annotations/ComplexAtLeastOneIsFilled.java @@ -0,0 +1,12 @@ +package info.frostfs.sdk.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +public @interface ComplexAtLeastOneIsFilled { + AtLeastOneIsFilled[] value(); +} diff --git a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java index 8865173..fd8ef2c 100644 --- a/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java +++ b/client/src/main/java/info/frostfs/sdk/constants/CryptoConst.java @@ -5,6 +5,10 @@ public class CryptoConst { public static final int RFC6979_SIGNATURE_SIZE = 64; public static final int HASH_SIGNATURE_SIZE = 65; + public static final int MURMUR_MULTIPLIER = 33; + public static final long LANDAU_PRIME_DIVISOR_64BIT = 0xc4ceb9fe1a85ec53L; + public static final long LANDAU_PRIME_DIVISOR_65BIT = 0xff51afd7ed558ccdL; + private CryptoConst() { } } diff --git a/client/src/main/java/info/frostfs/sdk/constants/PoolConst.java b/client/src/main/java/info/frostfs/sdk/constants/PoolConst.java new file mode 100644 index 0000000..30d853f --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/constants/PoolConst.java @@ -0,0 +1,14 @@ +package info.frostfs.sdk.constants; + +public class PoolConst { + public static final int DEFAULT_SESSION_TOKEN_EXPIRATION_DURATION = 100; // in epochs + public static final int DEFAULT_ERROR_THRESHOLD = 100; + public static final int DEFAULT_GRACEFUL_CLOSE_ON_SWITCH_TIMEOUT = 10; // Seconds + public static final int DEFAULT_REBALANCE_INTERVAL = 15; // Seconds + public static final int DEFAULT_HEALTHCHECK_TIMEOUT = 4; // Seconds + public static final int DEFAULT_DIAL_TIMEOUT = 5; // Seconds + public static final int DEFAULT_STREAM_TIMEOUT = 10; // Seconds + + private PoolConst() { + } +} diff --git a/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java b/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java new file mode 100644 index 0000000..322b615 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/constants/RuleConst.java @@ -0,0 +1,30 @@ +package info.frostfs.sdk.constants; + +public class RuleConst { + public static final byte VERSION = 0; + + public static final int BYTE_SIZE = 1; + public static final int U_INT_8_SIZE = BYTE_SIZE; + public static final int BOOL_SIZE = BYTE_SIZE; + + public static final long NULL_SLICE = -1L; + public static final int NULL_SLICE_SIZE = 1; + + public static final byte BYTE_TRUE = 1; + public static final byte BYTE_FALSE = 0; + + // maxSliceLen taken from + // https://github.com/neo-project/neo/blob/38218bbee5bbe8b33cd8f9453465a19381c9a547/src/Neo/IO/Helper.cs#L77 + public static final int MAX_SLICE_LENGTH = 0x1000000; + + public static final int MAX_VAR_INT_LENGTH = 10; + + public static final int CHAIN_MARSHAL_VERSION = 0; + + public static final long OFFSET127 = 0x7f; + public static final long OFFSET128 = 0x80; + public static final int UNSIGNED_SERIALIZE_SIZE = 7; + + private RuleConst() { + } +} diff --git a/client/src/main/java/info/frostfs/sdk/enums/HealthyStatus.java b/client/src/main/java/info/frostfs/sdk/enums/HealthyStatus.java new file mode 100644 index 0000000..f88a8db --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/enums/HealthyStatus.java @@ -0,0 +1,23 @@ +package info.frostfs.sdk.enums; + +public enum HealthyStatus { + // status HEALTHY is set when connection is ready to be used by the pool. + HEALTHY(1), + + // status UNHEALTHY_ON_REQUEST is set when communication after dialing to the + // endpoint is failed due to immediate or accumulated errors, connection is + // available and pool should close it before re-establishing connection once again. + UNHEALTHY_ON_REQUEST(2), + + // status UNHEALTHY_ON_DIAL is set when dialing to the endpoint is failed, + // so there is no connection to the endpoint, and pool should not close it + // before re-establishing connection once again. + UNHEALTHY_ON_DIAL(3), + ; + + public final int value; + + HealthyStatus(int value) { + this.value = value; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/enums/MethodIndex.java b/client/src/main/java/info/frostfs/sdk/enums/MethodIndex.java new file mode 100644 index 0000000..af8f4a5 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/enums/MethodIndex.java @@ -0,0 +1,29 @@ +package info.frostfs.sdk.enums; + +public enum MethodIndex { + METHOD_BALANCE_GET("balanceGet"), + METHOD_CONTAINER_PUT("containerPut"), + METHOD_CONTAINER_GET("ContainerGet"), + METHOD_CONTAINER_LIST("ContainerList"), + METHOD_CONTAINER_DELETE("ContainerDelete"), + METHOD_ENDPOINT_INFO("EndpointInfo"), + METHOD_NETWORK_INFO("NetworkInfo"), + METHOD_NETMAP_SNAPSHOT("NetMapSnapshot"), + METHOD_OBJECT_PUT("ObjectPut"), + METHOD_OBJECT_DELETE("ObjectDelete"), + METHOD_OBJECT_GET("ObjectGet"), + METHOD_OBJECT_HEAD("ObjectHead"), + METHOD_OBJECT_RANGE("ObjectRange"), + METHOD_OBJECT_PATCH("ObjectPatch"), + METHOD_SESSION_CREATE("SessionCreate"), + METHOD_APE_MANAGER_ADD_CHAIN("APEManagerAddChain"), + METHOD_APE_MANAGER_REMOVE_CHAIN("APEManagerRemoveChain"), + METHOD_APE_MANAGER_LIST_CHAINS("APEManagerListChains"), + ; + + public final String methodName; + + MethodIndex(String methodName) { + this.methodName = methodName; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/exceptions/ResponseFrostFSException.java b/client/src/main/java/info/frostfs/sdk/exceptions/ResponseFrostFSException.java index 6b216a6..c266395 100644 --- a/client/src/main/java/info/frostfs/sdk/exceptions/ResponseFrostFSException.java +++ b/client/src/main/java/info/frostfs/sdk/exceptions/ResponseFrostFSException.java @@ -4,7 +4,7 @@ import info.frostfs.sdk.dto.response.ResponseStatus; import lombok.Getter; @Getter -public class ResponseFrostFSException extends RuntimeException { +public class ResponseFrostFSException extends FrostFSException { private final ResponseStatus status; public ResponseFrostFSException(ResponseStatus status) { diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java b/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java index 34eca67..387882e 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ClientEnvironment.java @@ -5,9 +5,14 @@ import info.frostfs.sdk.annotations.NotNull; import info.frostfs.sdk.annotations.Validate; import info.frostfs.sdk.dto.netmap.Version; import info.frostfs.sdk.dto.object.OwnerId; +import info.frostfs.sdk.pool.SessionCache; import io.grpc.Channel; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; + +import static info.frostfs.sdk.Helper.getHexString; +import static info.frostfs.sdk.pool.Pool.formCacheKey; @Getter @Setter @@ -15,27 +20,38 @@ public class ClientEnvironment { @NotNull private final OwnerId ownerId; - @NotNull private final Version version; - @NotNull @Validate private final ECDsa key; - @NotNull private final Channel channel; - @NotNull private final FrostFSClient frostFSClient; + private String sessionKey; + private String address; private NetworkSettings networkSettings; - public ClientEnvironment(String wif, Channel channel, Version version, FrostFSClient frostFSClient) { - this.key = new ECDsa(wif); - this.ownerId = new OwnerId(key.getPublicKeyByte()); + private SessionCache sessionCache; + + public ClientEnvironment(ECDsa key, Channel channel, Version version, FrostFSClient frostFSClient, + SessionCache sessionCache) { + this.key = key; + this.ownerId = new OwnerId(key.getAccount().getAddress()); this.version = version; this.channel = channel; this.frostFSClient = frostFSClient; + this.sessionCache = sessionCache; + this.address = channel.authority(); + } + + public String getSessionKey() { + if (StringUtils.isBlank(sessionKey)) { + this.sessionKey = formCacheKey(address, getHexString(key.getPublicKeyByte())); + } + + return sessionKey; } } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java b/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java index bca70ce..0bab648 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ClientSettings.java @@ -1,37 +1,61 @@ package info.frostfs.sdk.jdo; import info.frostfs.sdk.annotations.AtLeastOneIsFilled; -import info.frostfs.sdk.annotations.NotNull; -import io.grpc.Channel; +import info.frostfs.sdk.annotations.ComplexAtLeastOneIsFilled; import io.grpc.ChannelCredentials; +import io.grpc.ManagedChannel; import lombok.Getter; import lombok.experimental.FieldNameConstants; +import java.io.File; + @Getter @FieldNameConstants -@AtLeastOneIsFilled(fields = {ClientSettings.Fields.host, ClientSettings.Fields.channel}) +@ComplexAtLeastOneIsFilled(value = { + @AtLeastOneIsFilled(fields = {ClientSettings.Fields.host, ClientSettings.Fields.channel}), + @AtLeastOneIsFilled(fields = {ClientSettings.Fields.wif, ClientSettings.Fields.wallet}), +}) public class ClientSettings { - @NotNull - private final String key; - + private String wif; + private File wallet; + private String password; private String host; private ChannelCredentials credentials; - private Channel channel; + private ManagedChannel channel; - public ClientSettings(String key, String host) { - this.key = key; + public ClientSettings(String wif, String host) { + this.wif = wif; this.host = host; } - public ClientSettings(String key, String host, ChannelCredentials credentials) { - this.key = key; + public ClientSettings(String wif, String host, ChannelCredentials credentials) { + this.wif = wif; this.host = host; this.credentials = credentials; } - public ClientSettings(String key, Channel channel) { - this.key = key; + public ClientSettings(String wif, ManagedChannel channel) { + this.wif = wif; + this.channel = channel; + } + + public ClientSettings(File wallet, String password, String host) { + this.wallet = wallet; + this.password = password; + this.host = host; + } + + public ClientSettings(File wallet, String password, String host, ChannelCredentials credentials) { + this.wallet = wallet; + this.password = password; + this.host = host; + this.credentials = credentials; + } + + public ClientSettings(File wallet, String password, ManagedChannel channel) { + this.wallet = wallet; + this.password = password; this.channel = channel; } } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java b/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java index b7e4686..dcd5b0a 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/ECDsa.java @@ -1,14 +1,25 @@ package info.frostfs.sdk.jdo; import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.exceptions.FrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import io.neow3j.wallet.Account; +import io.neow3j.wallet.nep6.NEP6Account; +import io.neow3j.wallet.nep6.NEP6Wallet; import lombok.Getter; import org.apache.commons.lang3.StringUtils; +import java.io.File; +import java.io.FileInputStream; import java.security.PrivateKey; +import java.util.Optional; -import static info.frostfs.sdk.KeyExtension.*; +import static info.frostfs.sdk.KeyExtension.loadPrivateKey; +import static info.frostfs.sdk.constants.ErrorConst.WALLET_IS_INVALID; import static info.frostfs.sdk.constants.ErrorConst.WIF_IS_INVALID; +import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; +import static io.neow3j.wallet.Wallet.OBJECT_MAPPER; +import static java.util.Objects.isNull; @Getter public class ECDsa { @@ -22,13 +33,41 @@ public class ECDsa { @NotNull private final PrivateKey privateKey; + @NotNull + private final Account account; + public ECDsa(String wif) { if (StringUtils.isEmpty(wif)) { throw new ValidationFrostFSException(WIF_IS_INVALID); } - this.privateKeyByte = getPrivateKeyFromWIF(wif); - this.publicKeyByte = loadPublicKey(privateKeyByte); + this.account = Account.fromWIF(wif); + this.privateKeyByte = account.getECKeyPair().getPrivateKey().getBytes(); + this.publicKeyByte = account.getECKeyPair().getPublicKey().getEncoded(true); this.privateKey = loadPrivateKey(privateKeyByte); } + + public ECDsa(File walletFile, String password) { + if (isNull(walletFile)) { + throw new ValidationFrostFSException(WALLET_IS_INVALID); + } + + try (var walletStream = new FileInputStream(walletFile)) { + NEP6Wallet nep6Wallet = OBJECT_MAPPER.readValue(walletStream, NEP6Wallet.class); + Optional defaultAccount = nep6Wallet.getAccounts().stream() + .filter(NEP6Account::getDefault) + .findFirst(); + + var account = defaultAccount.map(Account::fromNEP6Account) + .orElseGet(() -> Account.fromNEP6Account(nep6Wallet.getAccounts().get(0))); + account.decryptPrivateKey(isNull(password) ? EMPTY_STRING : password); + + this.account = account; + this.privateKeyByte = account.getECKeyPair().getPrivateKey().getBytes(); + this.publicKeyByte = account.getECKeyPair().getPublicKey().getEncoded(true); + this.privateKey = loadPrivateKey(privateKeyByte); + } catch (Exception exp) { + throw new FrostFSException(exp.getMessage()); + } + } } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/PutObjectParameters.java b/client/src/main/java/info/frostfs/sdk/jdo/PutObjectParameters.java deleted file mode 100644 index b995eac..0000000 --- a/client/src/main/java/info/frostfs/sdk/jdo/PutObjectParameters.java +++ /dev/null @@ -1,40 +0,0 @@ -package info.frostfs.sdk.jdo; - -import info.frostfs.sdk.annotations.NotNull; -import info.frostfs.sdk.dto.object.ObjectHeader; -import info.frostfs.sdk.dto.session.SessionToken; -import lombok.Getter; -import lombok.Setter; - -import java.io.InputStream; - -@Getter -@Setter -public class PutObjectParameters { - - @NotNull - private ObjectHeader header; - - @NotNull - private InputStream payload; - - private boolean clientCut; - private int bufferMaxSize; - private byte[] customerBuffer; - private SessionToken sessionToken; - private int maxObjectSizeCache; - private long currentStreamPosition; - private long fullLength; - - public PutObjectParameters(ObjectHeader header, InputStream payload, boolean clientCut, int bufferMaxSize) { - this.header = header; - this.payload = payload; - this.clientCut = clientCut; - this.bufferMaxSize = bufferMaxSize; - } - - public PutObjectParameters(ObjectHeader header, InputStream payload) { - this.header = header; - this.payload = payload; - } -} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/CallContext.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/CallContext.java new file mode 100644 index 0000000..4831e99 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/CallContext.java @@ -0,0 +1,23 @@ +package info.frostfs.sdk.jdo.parameters; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.concurrent.TimeUnit; + +import static info.frostfs.sdk.constants.AppConst.DEFAULT_GRPC_TIMEOUT; + +@Getter +@Builder +@AllArgsConstructor +public class CallContext { + private final long timeout; + private final TimeUnit timeUnit; + + public CallContext() { + this.timeout = DEFAULT_GRPC_TIMEOUT; + this.timeUnit = TimeUnit.SECONDS; + } +} + diff --git a/client/src/main/java/info/frostfs/sdk/jdo/WaitParameters.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/PrmWait.java similarity index 86% rename from client/src/main/java/info/frostfs/sdk/jdo/WaitParameters.java rename to client/src/main/java/info/frostfs/sdk/jdo/parameters/PrmWait.java index 7fdd965..c109bd4 100644 --- a/client/src/main/java/info/frostfs/sdk/jdo/WaitParameters.java +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/PrmWait.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.jdo; +package info.frostfs.sdk.jdo.parameters; import lombok.AllArgsConstructor; import lombok.Getter; @@ -8,14 +8,14 @@ import java.time.LocalDateTime; @Getter @AllArgsConstructor -public class WaitParameters { +public class PrmWait { private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(120); private static final Duration DEFAULT_POLL_INTERVAL = Duration.ofSeconds(5); private final Duration timeout; private final Duration pollInterval; - public WaitParameters() { + public PrmWait() { this.timeout = DEFAULT_TIMEOUT; this.pollInterval = DEFAULT_POLL_INTERVAL; } diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainAdd.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainAdd.java new file mode 100644 index 0000000..e7ae05c --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainAdd.java @@ -0,0 +1,27 @@ +package info.frostfs.sdk.jdo.parameters.ape; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.ape.Chain; +import info.frostfs.sdk.dto.chain.ChainTarget; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmApeChainAdd { + @NotNull + private Chain chain; + @NotNull + private ChainTarget chainTarget; + + private Map xHeaders; + + public PrmApeChainAdd(Chain chain, ChainTarget chainTarget) { + this.chain = chain; + this.chainTarget = chainTarget; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainList.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainList.java new file mode 100644 index 0000000..70dc193 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainList.java @@ -0,0 +1,23 @@ +package info.frostfs.sdk.jdo.parameters.ape; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.chain.ChainTarget; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmApeChainList { + @NotNull + private ChainTarget chainTarget; + + private Map xHeaders; + + public PrmApeChainList(ChainTarget chainTarget) { + this.chainTarget = chainTarget; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainRemove.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainRemove.java new file mode 100644 index 0000000..00bc2e9 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/ape/PrmApeChainRemove.java @@ -0,0 +1,27 @@ +package info.frostfs.sdk.jdo.parameters.ape; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.chain.ChainTarget; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmApeChainRemove { + @NotNull + private byte[] chainId; + + @NotNull + private ChainTarget chainTarget; + + private Map xHeaders; + + public PrmApeChainRemove(byte[] chainId, ChainTarget chainTarget) { + this.chainId = chainId; + this.chainTarget = chainTarget; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerCreate.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerCreate.java new file mode 100644 index 0000000..758bfd8 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerCreate.java @@ -0,0 +1,33 @@ +package info.frostfs.sdk.jdo.parameters.container; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.Container; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.PrmWait; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmContainerCreate implements SessionContext { + @NotNull + private Container container; + + private PrmWait waitParams; + private SessionToken sessionToken; + private Map xHeaders; + + public PrmContainerCreate(Container container, PrmWait waitParams) { + this.container = container; + this.waitParams = waitParams; + } + + public PrmContainerCreate(Container container) { + this.container = container; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerDelete.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerDelete.java new file mode 100644 index 0000000..c9baeaf --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerDelete.java @@ -0,0 +1,33 @@ +package info.frostfs.sdk.jdo.parameters.container; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.PrmWait; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmContainerDelete implements SessionContext { + @NotNull + private ContainerId containerId; + + private PrmWait waitParams; + private SessionToken sessionToken; + private Map xHeaders; + + public PrmContainerDelete(ContainerId containerId, PrmWait waitParams) { + this.containerId = containerId; + this.waitParams = waitParams; + } + + public PrmContainerDelete(ContainerId containerId) { + this.containerId = containerId; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGet.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGet.java new file mode 100644 index 0000000..aec9924 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGet.java @@ -0,0 +1,23 @@ +package info.frostfs.sdk.jdo.parameters.container; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmContainerGet { + @NotNull + private ContainerId containerId; + + private Map xHeaders; + + public PrmContainerGet(ContainerId containerId) { + this.containerId = containerId; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGetAll.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGetAll.java new file mode 100644 index 0000000..56b8d7b --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/container/PrmContainerGetAll.java @@ -0,0 +1,16 @@ +package info.frostfs.sdk.jdo.parameters.container; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class PrmContainerGetAll { + private Map xHeaders; +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectClientCutPut.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectClientCutPut.java new file mode 100644 index 0000000..9703a4c --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectClientCutPut.java @@ -0,0 +1,28 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.object.ObjectHeader; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.io.InputStream; +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectClientCutPut implements PrmObjectPutBase, SessionContext { + @NotNull + private final PutObjectContext putObjectContext = new PutObjectContext(); + @NotNull + private ObjectHeader objectHeader; + @NotNull + private InputStream payload; + private int bufferMaxSize; + private byte[] customerBuffer; + private SessionToken sessionToken; + private Map xHeaders; +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectDelete.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectDelete.java new file mode 100644 index 0000000..326834f --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectDelete.java @@ -0,0 +1,30 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectDelete implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectId objectId; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectDelete(ContainerId containerId, ObjectId objectId) { + this.containerId = containerId; + this.objectId = objectId; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectGet.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectGet.java new file mode 100644 index 0000000..e8eb3d7 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectGet.java @@ -0,0 +1,30 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectGet implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectId objectId; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectGet(ContainerId containerId, ObjectId objectId) { + this.containerId = containerId; + this.objectId = objectId; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectHeadGet.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectHeadGet.java new file mode 100644 index 0000000..8263db7 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectHeadGet.java @@ -0,0 +1,31 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectHeadGet implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectId objectId; + + private boolean raw; + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectHeadGet(ContainerId containerId, ObjectId objectId) { + this.containerId = containerId; + this.objectId = objectId; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPut.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPut.java new file mode 100644 index 0000000..ac99504 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPut.java @@ -0,0 +1,28 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.object.ObjectHeader; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectPut implements PrmObjectPutBase, SessionContext { + @NotNull + private final PutObjectContext putObjectContext = new PutObjectContext(); + @NotNull + private ObjectHeader objectHeader; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectPut(ObjectHeader objectHeader) { + this.objectHeader = objectHeader; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPutBase.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPutBase.java new file mode 100644 index 0000000..7ddfafb --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectPutBase.java @@ -0,0 +1,12 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.dto.object.ObjectHeader; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; + +import java.util.Map; + +public interface PrmObjectPutBase extends SessionContext { + ObjectHeader getObjectHeader(); + + Map getXHeaders(); +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSearch.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSearch.java new file mode 100644 index 0000000..0bc414b --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSearch.java @@ -0,0 +1,30 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectFilter; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectSearch implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectFilter[] filters; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectSearch(ContainerId containerId, ObjectFilter... filters) { + this.containerId = containerId; + this.filters = filters; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSinglePut.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSinglePut.java new file mode 100644 index 0000000..2965d0c --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PrmObjectSinglePut.java @@ -0,0 +1,26 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.object.ObjectFrostFS; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectSinglePut implements SessionContext { + @NotNull + private ObjectFrostFS objectFrostFS; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectSinglePut(ObjectFrostFS objectFrostFS) { + this.objectFrostFS = objectFrostFS; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PutObjectContext.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PutObjectContext.java new file mode 100644 index 0000000..fc97dd0 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/PutObjectContext.java @@ -0,0 +1,14 @@ +package info.frostfs.sdk.jdo.parameters.object; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@NoArgsConstructor +public class PutObjectContext { + private int maxObjectSizeCache; + private long currentStreamPosition; + private long fullLength; +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java new file mode 100644 index 0000000..58849df --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmObjectPatch.java @@ -0,0 +1,42 @@ +package info.frostfs.sdk.jdo.parameters.object.patch; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.object.ObjectAttribute; +import info.frostfs.sdk.dto.object.patch.Address; +import info.frostfs.sdk.dto.object.patch.Range; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.*; + +import java.io.InputStream; +import java.util.List; +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmObjectPatch implements SessionContext { + @NotNull + private Address address; + + private Range range; + private InputStream payload; + private List newAttributes; + private boolean replaceAttributes; + private int maxChunkLength; + private SessionToken sessionToken; + private Map xHeaders; + + public PrmObjectPatch(Address address, Range range, InputStream payload, int maxChunkLength) { + this.address = address; + this.range = range; + this.payload = payload; + this.maxChunkLength = maxChunkLength; + } + + public PrmObjectPatch(Address address, List newAttributes, boolean replaceAttributes) { + this.address = address; + this.newAttributes = newAttributes; + this.replaceAttributes = replaceAttributes; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeGet.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeGet.java new file mode 100644 index 0000000..add15b7 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeGet.java @@ -0,0 +1,35 @@ +package info.frostfs.sdk.jdo.parameters.object.patch; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.patch.Range; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmRangeGet implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectId objectId; + @NotNull + private Range range; + + private boolean raw; + private SessionToken sessionToken; + private Map xHeaders; + + public PrmRangeGet(ContainerId containerId, ObjectId objectId, Range range) { + this.containerId = containerId; + this.objectId = objectId; + this.range = range; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeHashGet.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeHashGet.java new file mode 100644 index 0000000..d860133 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/object/patch/PrmRangeHashGet.java @@ -0,0 +1,38 @@ +package info.frostfs.sdk.jdo.parameters.object.patch; + +import info.frostfs.sdk.annotations.NotNull; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.patch.Range; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.List; +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmRangeHashGet implements SessionContext { + @NotNull + private ContainerId containerId; + @NotNull + private ObjectId objectId; + @NotNull + private List ranges; + @NotNull + private byte[] salt; + + private SessionToken sessionToken; + private Map xHeaders; + + public PrmRangeHashGet(ContainerId containerId, ObjectId objectId, List ranges, byte[] salt) { + this.containerId = containerId; + this.objectId = objectId; + this.ranges = ranges; + this.salt = salt; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/PrmSessionCreate.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/PrmSessionCreate.java new file mode 100644 index 0000000..c73dd2b --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/PrmSessionCreate.java @@ -0,0 +1,19 @@ +package info.frostfs.sdk.jdo.parameters.session; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; + +import java.util.Map; + +@Getter +@Builder +@AllArgsConstructor +public class PrmSessionCreate { + private long expiration; //-1 is max + private Map xHeaders; + + public PrmSessionCreate(long expiration) { + this.expiration = expiration; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/SessionContext.java b/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/SessionContext.java new file mode 100644 index 0000000..536017a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/parameters/session/SessionContext.java @@ -0,0 +1,7 @@ +package info.frostfs.sdk.jdo.parameters.session; + +import info.frostfs.sdk.dto.session.SessionToken; + +public interface SessionContext { + SessionToken getSessionToken(); +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/pool/NodeParameters.java b/client/src/main/java/info/frostfs/sdk/jdo/pool/NodeParameters.java new file mode 100644 index 0000000..01c9df0 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/pool/NodeParameters.java @@ -0,0 +1,12 @@ +package info.frostfs.sdk.jdo.pool; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class NodeParameters { + private final int priority; + private final String address; + private final double weight; +} diff --git a/client/src/main/java/info/frostfs/sdk/jdo/pool/PoolInitParameters.java b/client/src/main/java/info/frostfs/sdk/jdo/pool/PoolInitParameters.java new file mode 100644 index 0000000..71eb670 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/pool/PoolInitParameters.java @@ -0,0 +1,44 @@ +package info.frostfs.sdk.jdo.pool; + +import info.frostfs.sdk.jdo.ECDsa; +import info.frostfs.sdk.pool.ClientWrapper; +import io.grpc.ClientInterceptors; +import io.netty.channel.ChannelOption; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.function.Function; + +@Getter +@Setter +public class PoolInitParameters { + private ECDsa key; + + private long nodeDialTimeout; + + private long nodeStreamTimeout; + + private long healthCheckTimeout; + + private long clientRebalanceInterval; + + private long sessionExpirationDuration; + + private int errorThreshold; + + private NodeParameters[] nodeParams; + + private ChannelOption[] dialOptions; + + private Function clientBuilder; + + private long gracefulCloseOnSwitchTimeout; + + private Logger logger; + + private Collection interceptors = new ArrayList<>(); +} + diff --git a/client/src/main/java/info/frostfs/sdk/jdo/result/ObjectHeaderResult.java b/client/src/main/java/info/frostfs/sdk/jdo/result/ObjectHeaderResult.java new file mode 100644 index 0000000..a3fbb1e --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/jdo/result/ObjectHeaderResult.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.jdo.result; + +import info.frostfs.sdk.dto.object.ObjectHeader; +import info.frostfs.sdk.dto.object.SplitInfo; +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Builder +@Getter +@Setter +public class ObjectHeaderResult { + private ObjectHeader headerInfo; + private SplitInfo splitInfo; +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Context.java b/client/src/main/java/info/frostfs/sdk/placement/Context.java new file mode 100644 index 0000000..64fcb28 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Context.java @@ -0,0 +1,369 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.exceptions.FrostFSException; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; +import static info.frostfs.sdk.constants.ErrorConst.*; + +@Getter +@Setter +public final class Context { + public static final String MAIN_FILTER_NAME = "*"; + public static final String LIKE_WILDCARD = "*"; + + // network map to operate on + private final NetmapSnapshot netMap; + + // cache of processed filters + private final Map processedFilters = new HashMap<>(); + + // cache of processed selectors + private final Map processedSelectors = new HashMap<>(); + + // stores results of selector processing + private final Map>> selections = new HashMap<>(); + + // cache of parsed numeric values + private final Map numCache = new HashMap<>(); + private final Map usedNodes = new HashMap<>(); + private final Function weightFunc; + private byte[] hrwSeed; + private long hrwSeedHash; + private int cbf; + private boolean strict; + + public Context(NetmapSnapshot netMap) { + this.netMap = netMap; + this.weightFunc = Tools.defaultWeightFunc(netMap.getNodeInfoCollection()); + } + + private static Pair calcNodesCount(Selector selector) { + return selector.getClause() == SelectorClause.SAME + ? new ImmutablePair<>(1, selector.getCount()) + : new ImmutablePair<>(selector.getCount(), 1); + } + + private static double calcBucketWeight(List ns, MeanIQRAgg a, Function wf) { + for (NodeInfo node : ns) { + a.add(wf.apply(node)); + } + return a.compute(); + } + + public void processFilters(PlacementPolicy policy) { + for (Filter filter : policy.getFilters()) { + processFilter(filter, true); + } + } + + private void processFilter(Filter filter, boolean top) { + String filterName = filter.getName(); + if (MAIN_FILTER_NAME.equals(filterName)) { + throw new FrostFSException(String.format(INVALID_FILTER_NAME_TEMPLATE, MAIN_FILTER_NAME)); + } + + if (top && (filterName == null || filterName.isEmpty())) { + throw new FrostFSException(UNNAMED_TOP_FILTER); + } + + if (!top && filterName != null && !filterName.isEmpty() && !processedFilters.containsKey(filterName)) { + throw new FrostFSException(FILTER_NOT_FOUND); + } + + if (filter.getOperation() == FilterOperation.AND || + filter.getOperation() == FilterOperation.OR || + filter.getOperation() == FilterOperation.NOT) { + + for (Filter f : filter.getFilters()) { + processFilter(f, false); + } + } else { + if (filter.getFilters().length != 0) { + throw new FrostFSException(NON_EMPTY_FILTERS); + } else if (!top && filterName != null && !filterName.isEmpty()) { + // named reference + return; + } + + switch (filter.getOperation()) { + case EQ: + case NE: + case LIKE: + break; + case GT: + case GE: + case LT: + case LE: + long n = Long.parseLong(filter.getValue()); + numCache.put(filter.getValue(), n); + break; + default: + throw new FrostFSException(String.format(INVALID_FILTER_OPERATION_TEMPLATE, filter.getOperation())); + } + } + + if (top) { + processedFilters.put(filterName, filter); + } + } + + public void processSelectors(PlacementPolicy policy) { + for (Selector selector : policy.getSelectors()) { + String filterName = selector.getFilter(); + if (!MAIN_FILTER_NAME.equals(filterName)) { + if (selector.getFilter() == null || !processedFilters.containsKey(selector.getFilter())) { + throw new FrostFSException(String.format(FILTER_NOT_FOUND_TEMPLATE, filterName)); + } + } + + processedSelectors.put(selector.getName(), selector); + List> selection = getSelection(selector); + selections.put(selector.getName(), selection); + } + } + + private NodeAttributePair[] getSelectionBase(Selector selector) { + String fName = selector.getFilter(); + if (fName == null) { + throw new FrostFSException(FILTER_NAME_IS_EMPTY); + } + + Filter f = processedFilters.get(fName); + boolean isMain = MAIN_FILTER_NAME.equals(fName); + List result = new ArrayList<>(); + + Map> nodeMap = new HashMap<>(); + String attr = selector.getAttribute(); + + for (NodeInfo node : netMap.getNodeInfoCollection()) { + if (usedNodes.containsKey(node.getHash())) { + continue; + } + + if (isMain || match(f, node)) { + if (attr == null) { + result.add(new NodeAttributePair("", new NodeInfo[]{node})); + } else { + String v = node.getAttributes().get(attr); + List nodes = nodeMap.computeIfAbsent(v, k -> new ArrayList<>()); + nodes.add(node); + } + } + } + + if (attr != null && !attr.isEmpty()) { + for (Map.Entry> entry : nodeMap.entrySet()) { + result.add(new NodeAttributePair(entry.getKey(), entry.getValue().toArray(NodeInfo[]::new))); + } + } + + if (hrwSeed != null && hrwSeed.length != 0) { + NodeAttributePair[] sortedNodes = new NodeAttributePair[result.size()]; + + for (int i = 0; i < result.size(); i++) { + double[] ws = new double[result.get(i).getNodes().length]; + NodeAttributePair res = result.get(i); + Tools.appendWeightsTo(res.getNodes(), weightFunc, ws); + sortedNodes[i] = new NodeAttributePair( + res.getAttr(), + Tools.sortHasherSliceByWeightValue(Arrays.asList(res.getNodes()), ws, hrwSeedHash) + .toArray(NodeInfo[]::new) + ); + } + + return sortedNodes; + } + return result.toArray(new NodeAttributePair[0]); + } + + public List> getSelection(Selector s) { + Pair counts = calcNodesCount(s); + int bucketCount = counts.getKey(); + int nodesInBucket = counts.getValue(); + + NodeAttributePair[] buckets = getSelectionBase(s); + + if (strict && buckets.length < bucketCount) { + throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); + } + + if (hrwSeed == null || hrwSeed.length == 0) { + if (s.getAttribute() == null || s.getAttribute().isEmpty()) { + Arrays.sort(buckets, Comparator.comparing(b -> b.getNodes()[0].getHash())); + } else { + Arrays.sort(buckets, Comparator.comparing(NodeAttributePair::getAttr)); + } + } + + int maxNodesInBucket = nodesInBucket * cbf; + + List> res = new ArrayList<>(buckets.length); + List> fallback = new ArrayList<>(buckets.length); + + for (NodeAttributePair bucket : buckets) { + List ns = Arrays.asList(bucket.getNodes()); + if (ns.size() >= maxNodesInBucket) { + res.add(new ArrayList<>(ns.subList(0, maxNodesInBucket))); + } else if (ns.size() >= nodesInBucket) { + fallback.add(new ArrayList<>(ns)); + } + } + + if (res.size() < bucketCount) { + res.addAll(fallback); + + if (strict && res.size() < bucketCount) { + throw new FrostFSException(String.format(NOT_ENOUGH_NODES_TEMPLATE, s.getName())); + } + } + + if (hrwSeed != null && hrwSeed.length != 0) { + double[] weights = new double[res.size()]; + var a = new MeanIQRAgg(); + + for (int i = 0; i < res.size(); i++) { + a.clear(); + weights[i] = calcBucketWeight(res.get(i), a, weightFunc); + } + + List hashers = res.stream() + .map(HasherList::new) + .collect(Collectors.toList()); + + hashers = Tools.sortHasherSliceByWeightValue(hashers, weights, hrwSeedHash); + + for (int i = 0; i < res.size(); i++) { + res.set(i, hashers.get(i).getNodes()); + } + } + + if (res.size() < bucketCount) { + if (strict && res.isEmpty()) { + throw new FrostFSException(NOT_ENOUGH_NODES); + } + bucketCount = res.size(); + } + + if (s.getAttribute() == null || s.getAttribute().isEmpty()) { + fallback = res.subList(bucketCount, res.size()); + res = new ArrayList<>(res.subList(0, bucketCount)); + + for (int i = 0; i < fallback.size(); i++) { + int index = i % bucketCount; + if (res.get(index).size() >= maxNodesInBucket) { + break; + } + res.get(index).addAll(fallback.get(i)); + } + } + + return res.subList(0, bucketCount); + } + + private boolean matchKeyValue(Filter f, NodeInfo nodeInfo) { + switch (f.getOperation()) { + case EQ: + return nodeInfo.getAttributes().containsKey(f.getKey()) && + nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + case LIKE: + boolean hasPrefix = f.getValue().startsWith(LIKE_WILDCARD); + boolean hasSuffix = f.getValue().endsWith(LIKE_WILDCARD); + + int start = hasPrefix ? LIKE_WILDCARD.length() : 0; + int end = hasSuffix ? f.getValue().length() - LIKE_WILDCARD.length() : f.getValue().length(); + String str = f.getValue().substring(start, end); + + if (hasPrefix && hasSuffix) { + return nodeInfo.getAttributes().get(f.getKey()).contains(str); + } + if (hasPrefix) { + return nodeInfo.getAttributes().get(f.getKey()).endsWith(str); + } + if (hasSuffix) { + return nodeInfo.getAttributes().get(f.getKey()).startsWith(str); + } + return nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + case NE: + return !nodeInfo.getAttributes().get(f.getKey()).equals(f.getValue()); + default: + long attr; + switch (f.getKey()) { + case ATTRIBUTE_PRICE: + attr = nodeInfo.getPrice().longValue(); + break; + case ATTRIBUTE_CAPACITY: + attr = nodeInfo.getCapacity().longValue(); + break; + default: + try { + attr = Long.parseLong(nodeInfo.getAttributes().get(f.getKey())); + } catch (NumberFormatException e) { + return false; + } + break; + } + + switch (f.getOperation()) { + case GT: + return attr > numCache.get(f.getValue()); + case GE: + return attr >= numCache.get(f.getValue()); + case LT: + return attr < numCache.get(f.getValue()); + case LE: + return attr <= numCache.get(f.getValue()); + default: + break; + } + break; + } + return false; + } + + boolean match(Filter f, NodeInfo nodeInfo) { + if (f == null) { + return false; + } + + switch (f.getOperation()) { + case NOT: + Filter[] inner = f.getFilters(); + Filter fSub = inner[0]; + + if (inner[0].getName() != null && !inner[0].getName().isEmpty()) { + fSub = processedFilters.get(inner[0].getName()); + } + return !match(fSub, nodeInfo); + case AND: + case OR: + for (int i = 0; i < f.getFilters().length; i++) { + Filter currentFilter = f.getFilters()[i]; + + if (currentFilter.getName() != null && !currentFilter.getName().isEmpty()) { + currentFilter = processedFilters.get(currentFilter.getName()); + } + + boolean ok = match(currentFilter, nodeInfo); + + if (ok == (f.getOperation() == FilterOperation.OR)) { + return ok; + } + } + return f.getOperation() == FilterOperation.AND; + default: + return matchKeyValue(f, nodeInfo); + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/HasherList.java b/client/src/main/java/info/frostfs/sdk/placement/HasherList.java new file mode 100644 index 0000000..ee5a7ce --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/HasherList.java @@ -0,0 +1,20 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Hasher; +import info.frostfs.sdk.dto.netmap.NodeInfo; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +@Getter +@AllArgsConstructor +public final class HasherList implements Hasher { + private final List nodes; + + @Override + public long getHash() { + return CollectionUtils.isNotEmpty(nodes) ? nodes.get(0).getHash() : 0L; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java new file mode 100644 index 0000000..d3464d4 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MeanAgg.java @@ -0,0 +1,18 @@ +package info.frostfs.sdk.placement; + +import java.math.BigInteger; + +public class MeanAgg { + private double mean; + private int count; + + public void add(BigInteger n) { + int c = count + 1; + mean = mean * count / c + n.doubleValue() / c; + count++; + } + + public double compute() { + return mean; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java new file mode 100644 index 0000000..b61b25e --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MeanIQRAgg.java @@ -0,0 +1,57 @@ +package info.frostfs.sdk.placement; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public final class MeanIQRAgg { + private static final int MIN_LN = 4; + + private final List arr = new ArrayList<>(); + + public MeanIQRAgg() { + } + + public void add(double d) { + arr.add(d); + } + + public double compute() { + int length = arr.size(); + if (length == 0) { + return 0; + } + + List sorted = new ArrayList<>(arr); + Collections.sort(sorted); + + double minV, maxV; + + if (length < MIN_LN) { + minV = sorted.get(0); + maxV = sorted.get(length - 1); + } else { + int start = length / MIN_LN; + int end = length * 3 / MIN_LN - 1; + + minV = sorted.get(start); + maxV = sorted.get(end); + } + + int count = 0; + double sum = 0; + + for (var e : sorted) { + if (e >= minV && e <= maxV) { + sum += e; + count++; + } + } + + return count == 0 ? 0 : sum / count; + } + + public void clear() { + arr.clear(); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java b/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java new file mode 100644 index 0000000..e2855b5 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/MinAgg.java @@ -0,0 +1,24 @@ +package info.frostfs.sdk.placement; + +import java.math.BigInteger; + +public class MinAgg { + private double min; + private boolean minFound; + + public void add(BigInteger n) { + if (!minFound) { + min = n.doubleValue(); + minFound = true; + return; + } + + if (n.doubleValue() < min) { + min = n.doubleValue(); + } + } + + public double compute() { + return min; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java b/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java new file mode 100644 index 0000000..9fd1660 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/NodeAttributePair.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.NodeInfo; +import lombok.Getter; + +@Getter +public class NodeAttributePair { + private final String attr; + private final NodeInfo[] nodes; + + NodeAttributePair(String attr, NodeInfo[] nodes) { + this.attr = attr; + this.nodes = nodes; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java b/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java new file mode 100644 index 0000000..587150d --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Normalizer.java @@ -0,0 +1,5 @@ +package info.frostfs.sdk.placement; + +public interface Normalizer { + double normalize(double w); +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java b/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java new file mode 100644 index 0000000..f20c75f --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/PlacementVector.java @@ -0,0 +1,197 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.exceptions.FrostFSException; +import lombok.AllArgsConstructor; +import org.apache.commons.codec.digest.MurmurHash3; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.function.Function; + +import static info.frostfs.sdk.constants.ErrorConst.SELECTOR_NOT_FOUND_TEMPLATE; +import static info.frostfs.sdk.constants.ErrorConst.VECTORS_IS_NULL; + +@AllArgsConstructor +public final class PlacementVector { + private final NetmapSnapshot netmapSnapshot; + + private static NodeInfo[] flattenNodes(List> nodes) { + int size = nodes.stream().mapToInt(List::size).sum(); + NodeInfo[] result = new NodeInfo[size]; + + int i = 0; + for (List ns : nodes) { + for (NodeInfo n : ns) { + result[i++] = n; + } + } + return result; + } + + /* + * PlacementVectors sorts container nodes returned by ContainerNodes method + * and returns placement vectors for the entity identified by the given pivot. + * For example, in order to build node list to store the object, + * binary-encoded object identifier can be used as pivot. + * Result is deterministic for the fixed NetMap and parameters. + * */ + public NodeInfo[][] placementVectors(NodeInfo[][] vectors, byte[] pivot) { + if (vectors == null) { + throw new FrostFSException(VECTORS_IS_NULL); + } + + long hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; + + Function wf = Tools.defaultWeightFunc(netmapSnapshot.getNodeInfoCollection()); + + NodeInfo[][] result = new NodeInfo[vectors.length][]; + int maxSize = Arrays.stream(vectors) + .mapToInt(v -> v.length) + .max() + .orElse(0); + + double[] spanWeights = new double[maxSize]; + + for (int i = 0; i < vectors.length; i++) { + result[i] = Arrays.copyOf(vectors[i], vectors[i].length); + + Tools.appendWeightsTo(result[i], wf, spanWeights); + + List sorted = Tools.sortHasherSliceByWeightValue( + Arrays.asList(result[i]), + spanWeights, + hash + ); + result[i] = sorted.toArray(new NodeInfo[0]); + } + + return result; + } + + /* + * SelectFilterNodes returns a two-dimensional list of nodes as a result of applying the given + * SelectFilterExpr to the NetMap. If the SelectFilterExpr contains only filters, the result contains + * a single row with the result of the last filter application. If the SelectFilterExpr contains only selectors, + * the result contains the selection rows of the last select application. + * */ + public List> selectFilterNodes(SelectFilterExpr expr) { + PlacementPolicy policy = new PlacementPolicy( + null, + false, + expr.getCbf(), + expr.getFilters().toArray(Filter[]::new), + new Selector[]{expr.getSelector()} + ); + + Context ctx = new Context(netmapSnapshot); + ctx.setCbf(expr.getCbf()); + + ctx.processFilters(policy); + ctx.processSelectors(policy); + + List> ret = new ArrayList<>(); + + if (expr.getSelector() == null) { + Filter lastFilter = expr.getFilters().get(expr.getFilters().size() - 1); + List subCollection = new ArrayList<>(); + ret.add(subCollection); + + for (NodeInfo nodeInfo : netmapSnapshot.getNodeInfoCollection()) { + if (ctx.match(ctx.getProcessedFilters().get(lastFilter.getName()), nodeInfo)) { + subCollection.add(nodeInfo); + } + } + } else if (expr.getSelector().getName() != null) { + List> sel = ctx.getSelection( + ctx.getProcessedSelectors().get(expr.getSelector().getName()) + ); + + for (List ns : sel) { + List subCollection = new ArrayList<>(ns); + ret.add(subCollection); + } + } + + return ret; + } + + /* + * ContainerNodes returns two-dimensional list of nodes as a result of applying given PlacementPolicy to the NetMap. + * Each line of the list corresponds to a replica descriptor. + * Line order corresponds to order of ReplicaDescriptor list in the policy. + * Nodes are pre-filtered according to the Filter list from the policy, and then selected by Selector list. + * Result is deterministic for the fixed NetMap and parameters. + * + * Result can be used in PlacementVectors. + * */ + public NodeInfo[][] containerNodes(PlacementPolicy p, byte[] pivot) { + Context c = new Context(netmapSnapshot); + c.setCbf(p.getBackupFactory() == 0 ? 3 : p.getBackupFactory()); + + if (pivot != null && pivot.length > 0) { + c.setHrwSeed(pivot); + + var hash = MurmurHash3.hash128x64(pivot, 0, pivot.length, 0)[0]; + c.setHrwSeedHash(hash); + } + + c.processFilters(p); + c.processSelectors(p); + + boolean unique = p.isUnique(); + + List> result = new ArrayList<>(p.getReplicas().length); + for (int i = 0; i < p.getReplicas().length; i++) { + result.add(new ArrayList<>()); + } + + for (int i = 0; i < p.getReplicas().length; i++) { + String sName = p.getReplicas()[i].getSelector(); + + if ((sName == null || sName.isEmpty()) && + !(p.getReplicas().length == 1 && p.getSelectors().length == 1)) { + + Selector s = new Selector( + "", p.getReplicas()[i].getCountNodes(), null, null, + Context.MAIN_FILTER_NAME + ); + + List> nodes = c.getSelection(s); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + + if (unique) { + for (NodeInfo n : result.get(i)) { + c.getUsedNodes().put(n.getHash(), true); + } + } + continue; + } + + if (unique) { + Selector s = c.getProcessedSelectors().get(sName); + if (s == null) { + throw new FrostFSException(String.format(SELECTOR_NOT_FOUND_TEMPLATE, sName)); + } + + List> nodes = c.getSelection(s); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + + for (NodeInfo n : result.get(i)) { + c.getUsedNodes().put(n.getHash(), true); + } + } else { + List> nodes = c.getSelections().get(sName); + result.get(i).addAll(Arrays.asList(flattenNodes(nodes))); + } + } + + NodeInfo[][] collection = new NodeInfo[result.size()][]; + for (int i = 0; i < result.size(); i++) { + collection[i] = result.get(i).toArray(new NodeInfo[0]); + } + + return collection; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java b/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java new file mode 100644 index 0000000..1bcc03a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/ReverseMinNorm.java @@ -0,0 +1,14 @@ +package info.frostfs.sdk.placement; + +public class ReverseMinNorm implements Normalizer { + private final double min; + + public ReverseMinNorm(double min) { + this.min = min; + } + + @Override + public double normalize(double w) { + return (min + 1) / (w + 1); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java b/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java new file mode 100644 index 0000000..b7dd898 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/SelectFilterExpr.java @@ -0,0 +1,16 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Filter; +import info.frostfs.sdk.dto.netmap.Selector; +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.util.List; + +@Getter +@AllArgsConstructor +public class SelectFilterExpr { + private final int cbf; + private final Selector selector; + private final List filters; +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java b/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java new file mode 100644 index 0000000..c0e867a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/SigmoidNorm.java @@ -0,0 +1,19 @@ +package info.frostfs.sdk.placement; + +public class SigmoidNorm implements Normalizer { + private final double scale; + + public SigmoidNorm(double scale) { + this.scale = scale; + } + + @Override + public double normalize(double w) { + if (scale == 0) { + return 0; + } + + double x = w / scale; + return x / (1 + x); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/placement/Tools.java b/client/src/main/java/info/frostfs/sdk/placement/Tools.java new file mode 100644 index 0000000..0140078 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/placement/Tools.java @@ -0,0 +1,123 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.Hasher; +import info.frostfs.sdk.dto.netmap.NodeInfo; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.function.Function; + +import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; +import static info.frostfs.sdk.constants.CryptoConst.*; + +public final class Tools { + private Tools() { + } + + public static long distance(long x, long y) { + long acc = x ^ y; + acc ^= acc >>> MURMUR_MULTIPLIER; + acc *= LANDAU_PRIME_DIVISOR_65BIT; + acc ^= acc >>> MURMUR_MULTIPLIER; + acc *= LANDAU_PRIME_DIVISOR_64BIT; + acc ^= acc >>> MURMUR_MULTIPLIER; + return acc; + } + + public static void appendWeightsTo(NodeInfo[] nodes, Function wf, double[] weights) { + if (weights.length < nodes.length) { + weights = new double[nodes.length]; + } + for (int i = 0; i < nodes.length; i++) { + weights[i] = wf.apply(nodes[i]); + } + } + + public static List sortHasherSliceByWeightValue(List nodes, double[] weights, long hash) { + if (nodes.isEmpty()) { + return nodes; + } + + boolean allEquals = true; + if (weights.length > 1) { + for (int i = 1; i < weights.length; i++) { + if (weights[i] != weights[0]) { + allEquals = false; + break; + } + } + } + + Double[] dist = new Double[nodes.size()]; + + if (allEquals) { + for (int i = 0; i < dist.length; i++) { + long x = nodes.get(i).getHash(); + dist[i] = toUnsignedBigInteger(distance(x, hash)).doubleValue(); + } + return sortHasherByDistance(nodes, dist, true); + } + + for (int i = 0; i < dist.length; i++) { + var reverse = UNSIGNED_LONG_MASK.subtract(toUnsignedBigInteger(distance(nodes.get(i).getHash(), hash))); + dist[i] = reverse.doubleValue() * weights[i]; + } + + return sortHasherByDistance(nodes, dist, false); + } + + public static > List sortHasherByDistance( + List nodes, N[] dist, boolean asc + ) { + IndexedValue[] indexes = new IndexedValue[nodes.size()]; + for (int i = 0; i < dist.length; i++) { + indexes[i] = new IndexedValue<>(nodes.get(i), dist[i]); + } + + if (asc) { + Arrays.sort(indexes, Comparator.comparing(iv -> iv.dist)); + } else { + Arrays.sort(indexes, (iv1, iv2) -> iv2.dist.compareTo(iv1.dist)); + } + + List result = new ArrayList<>(); + for (IndexedValue iv : indexes) { + result.add(iv.nodeInfo); + } + return result; + } + + public static Function defaultWeightFunc(List nodes) { + MeanAgg mean = new MeanAgg(); + MinAgg minV = new MinAgg(); + + for (NodeInfo node : nodes) { + mean.add(node.getCapacity()); + minV.add(node.getPrice()); + } + + return newWeightFunc(new SigmoidNorm(mean.compute()), new ReverseMinNorm(minV.compute())); + } + + private static BigInteger toUnsignedBigInteger(long i) { + return i >= 0 ? BigInteger.valueOf(i) : BigInteger.valueOf(i).and(UNSIGNED_LONG_MASK); + } + + private static Function newWeightFunc(Normalizer capNorm, Normalizer priceNorm) { + return nodeInfo -> capNorm.normalize(nodeInfo.getCapacity().doubleValue()) + * priceNorm.normalize(nodeInfo.getPrice().doubleValue()); + } + + private static class IndexedValue { + final T nodeInfo; + final N dist; + + IndexedValue(T nodeInfo, N dist) { + this.nodeInfo = nodeInfo; + this.dist = dist; + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/ClientStatus.java b/client/src/main/java/info/frostfs/sdk/pool/ClientStatus.java new file mode 100644 index 0000000..53b04c1 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/ClientStatus.java @@ -0,0 +1,27 @@ +package info.frostfs.sdk.pool; + +public interface ClientStatus { + // isHealthy checks if the connection can handle requests. + boolean isHealthy(); + + // isDialed checks if the connection was created. + boolean isDialed(); + + // setUnhealthy marks client as unhealthy. + void setUnhealthy(); + + // address return address of endpoint. + String getAddress(); + + // currentErrorRate returns current errors rate. + // After specific threshold connection is considered as unhealthy. + // Pool.startRebalance routine can make this connection healthy again. + int getCurrentErrorRate(); + + // overallErrorRate returns the number of all happened errors. + long getOverallErrorRate(); + + // methodsStatus returns statistic for all used methods. + StatusSnapshot[] getMethodsStatus(); +} + diff --git a/client/src/main/java/info/frostfs/sdk/pool/ClientStatusMonitor.java b/client/src/main/java/info/frostfs/sdk/pool/ClientStatusMonitor.java new file mode 100644 index 0000000..0770482 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/ClientStatusMonitor.java @@ -0,0 +1,114 @@ +package info.frostfs.sdk.pool; + +import info.frostfs.sdk.enums.HealthyStatus; +import info.frostfs.sdk.enums.MethodIndex; +import info.frostfs.sdk.utils.FrostFSMessages; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; + +@Getter +@Setter +public class ClientStatusMonitor implements ClientStatus { + private final ReentrantLock lock = new ReentrantLock(); + private final Logger logger; + private final AtomicInteger healthy = new AtomicInteger(); + + private final String address; + private final MethodStatus[] methods; + + private int errorThreshold; + private int currentErrorCount; + private long overallErrorCount; + + public ClientStatusMonitor(Logger logger, String address) { + this.logger = logger; + this.healthy.set(HealthyStatus.HEALTHY.value); + + this.address = address; + this.methods = Arrays.stream(MethodIndex.values()) + .map(t -> new MethodStatus(t.methodName)) + .toArray(MethodStatus[]::new); + } + + @Override + public boolean isHealthy() { + return healthy.get() == HealthyStatus.HEALTHY.value; + } + + @Override + public boolean isDialed() { + return healthy.get() != HealthyStatus.UNHEALTHY_ON_DIAL.value; + } + + public void setHealthy() { + healthy.set(HealthyStatus.HEALTHY.ordinal()); + } + + @Override + public void setUnhealthy() { + healthy.set(HealthyStatus.UNHEALTHY_ON_REQUEST.value); + } + + public void setUnhealthyOnDial() { + healthy.set(HealthyStatus.UNHEALTHY_ON_DIAL.value); + } + + public void incErrorRate() { + boolean thresholdReached; + lock.lock(); + try { + currentErrorCount++; + overallErrorCount++; + + thresholdReached = currentErrorCount >= errorThreshold; + + if (thresholdReached) { + setUnhealthy(); + currentErrorCount = 0; + } + } finally { + lock.unlock(); + } + + if (thresholdReached && logger != null) { + FrostFSMessages.errorThresholdReached(logger, address, errorThreshold); + } + } + + @Override + public int getCurrentErrorRate() { + lock.lock(); + try { + return currentErrorCount; + } finally { + lock.unlock(); + } + } + + @Override + public long getOverallErrorRate() { + lock.lock(); + try { + return overallErrorCount; + } finally { + lock.unlock(); + } + } + + @Override + public StatusSnapshot[] getMethodsStatus() { + StatusSnapshot[] result = new StatusSnapshot[methods.length]; + + for (int i = 0; i < result.length; i++) { + result[i] = methods[i].getSnapshot(); + } + + return result; + } +} + diff --git a/client/src/main/java/info/frostfs/sdk/pool/ClientWrapper.java b/client/src/main/java/info/frostfs/sdk/pool/ClientWrapper.java new file mode 100644 index 0000000..54163d1 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/ClientWrapper.java @@ -0,0 +1,131 @@ +package info.frostfs.sdk.pool; + +import info.frostfs.sdk.FrostFSClient; +import info.frostfs.sdk.enums.MethodIndex; +import info.frostfs.sdk.exceptions.ResponseFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.utils.WaitUtil; +import lombok.AccessLevel; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import static info.frostfs.sdk.constants.ErrorConst.POOL_CLIENT_UNHEALTHY; + +@Getter +public class ClientWrapper extends ClientStatusMonitor { + @Getter(value = AccessLevel.NONE) + private final Lock lock = new ReentrantLock(); + private final SessionCache sessionCache; + private final WrapperPrm wrapperPrm; + private FrostFSClient client; + + public ClientWrapper(WrapperPrm wrapperPrm, Pool pool) { + super(wrapperPrm.getLogger(), wrapperPrm.getAddress()); + this.wrapperPrm = wrapperPrm; + setErrorThreshold(wrapperPrm.getErrorThreshold()); + + this.sessionCache = pool.getSessionCache(); + this.client = new FrostFSClient(wrapperPrm, sessionCache); + } + + + public FrostFSClient getClient() { + lock.lock(); + try { + if (isHealthy()) { + return client; + } + return null; + } finally { + lock.unlock(); + } + } + + public void dial(CallContext ctx) { + FrostFSClient client = getClient(); + if (client == null) { + throw new ValidationFrostFSException(POOL_CLIENT_UNHEALTHY); + } + client.dial(ctx); + } + + public void handleError(Exception exp) { + if (exp instanceof ResponseFrostFSException && ((ResponseFrostFSException) exp).getStatus() != null) { + switch (((ResponseFrostFSException) exp).getStatus().getCode()) { + case INTERNAL: + case WRONG_MAGIC_NUMBER: + case SIGNATURE_VERIFICATION_FAILURE: + case NODE_UNDER_MAINTENANCE: + incErrorRate(); + } + return; + } + + incErrorRate(); + } + + private void scheduleGracefulClose() { + if (client == null) { + return; + } + + WaitUtil.sleep(wrapperPrm.getGracefulCloseOnSwitchTimeout()); + client.close(); + } + + public CompletableFuture restartIfUnhealthy(CallContext ctx) { + try { + client.getLocalNodeInfo(ctx); + return CompletableFuture.completedFuture(false); + } catch (Exception ignored) { + } + + if (isDialed()) { + scheduleGracefulClose(); + } + + return CompletableFuture.completedFuture(restartClient(ctx)); + } + + private boolean restartClient(CallContext ctx) { + FrostFSClient newClient = null; + try { + newClient = new FrostFSClient(wrapperPrm, sessionCache); + + var error = newClient.dial(ctx); + if (StringUtils.isNotBlank(error)) { + setUnhealthyOnDial(); + newClient.close(); + return true; + } + + lock.lock(); + client = newClient; + lock.unlock(); + } catch (Exception exp) { + if (newClient != null) { + newClient.close(); + } + } + + try { + client.getLocalNodeInfo(ctx); + } catch (Exception exp) { + setUnhealthy(); + return true; + } + + setHealthy(); + return false; + } + + public void incRequests(long elapsed, MethodIndex method) { + var methodStat = getMethods()[method.ordinal()]; + methodStat.incRequests(elapsed); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/InnerPool.java b/client/src/main/java/info/frostfs/sdk/pool/InnerPool.java new file mode 100644 index 0000000..94a8be4 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/InnerPool.java @@ -0,0 +1,54 @@ +package info.frostfs.sdk.pool; + +import java.util.concurrent.locks.ReentrantLock; + +class InnerPool { + private static final int ATTEMPTS_COUNT = 3; + private final ReentrantLock lock = new ReentrantLock(); + private final ClientWrapper[] clients; + private Sampler sampler; + + InnerPool(Sampler sampler, ClientWrapper[] clients) { + this.sampler = sampler; + this.clients = clients; + } + + Sampler getSampler() { + return sampler; + } + + void setSampler(Sampler sampler) { + this.sampler = sampler; + } + + ClientWrapper[] getClients() { + return clients; + } + + ClientWrapper connection() { + lock.lock(); + try { + if (clients.length == 1) { + ClientWrapper client = clients[0]; + if (client.isHealthy()) { + return client; + } + } else { + int attempts = ATTEMPTS_COUNT * clients.length; + + for (int i = 0; i < attempts; i++) { + int index = sampler.next(); + + if (clients[index].isHealthy()) { + return clients[index]; + } + } + } + + return null; + } finally { + lock.unlock(); + } + } +} + diff --git a/client/src/main/java/info/frostfs/sdk/pool/MethodStatus.java b/client/src/main/java/info/frostfs/sdk/pool/MethodStatus.java new file mode 100644 index 0000000..5309c93 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/MethodStatus.java @@ -0,0 +1,31 @@ +package info.frostfs.sdk.pool; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.Setter; + +import java.util.concurrent.locks.ReentrantLock; + +@Setter +@Getter +public class MethodStatus { + @Getter(AccessLevel.NONE) + private final ReentrantLock lock = new ReentrantLock(); + private final String name; + private StatusSnapshot snapshot; + + public MethodStatus(String name) { + this.name = name; + this.snapshot = new StatusSnapshot(); + } + + void incRequests(long elapsed) { + lock.lock(); + try { + snapshot.setAllTime(snapshot.getAllTime() + elapsed); + snapshot.setAllRequests(snapshot.getAllRequests() + 1); + } finally { + lock.unlock(); + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/NodeStatistic.java b/client/src/main/java/info/frostfs/sdk/pool/NodeStatistic.java new file mode 100644 index 0000000..2ad4cdc --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/NodeStatistic.java @@ -0,0 +1,13 @@ +package info.frostfs.sdk.pool; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class NodeStatistic { + private String address; + private StatusSnapshot[] methods; + private long overallErrors; + private int currentErrors; +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/NodesParam.java b/client/src/main/java/info/frostfs/sdk/pool/NodesParam.java new file mode 100644 index 0000000..0221852 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/NodesParam.java @@ -0,0 +1,19 @@ +package info.frostfs.sdk.pool; + +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class NodesParam { + private final int priority; + private final List address; + private final List weight; + + public NodesParam(int priority) { + this.priority = priority; + this.address = new ArrayList<>(); + this.weight = new ArrayList<>(); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/Pool.java b/client/src/main/java/info/frostfs/sdk/pool/Pool.java new file mode 100644 index 0000000..2191643 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/Pool.java @@ -0,0 +1,558 @@ +package info.frostfs.sdk.pool; + +import frostfs.refs.Types; +import info.frostfs.sdk.dto.ape.Chain; +import info.frostfs.sdk.dto.container.Container; +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.netmap.NetmapSnapshot; +import info.frostfs.sdk.dto.netmap.NodeInfo; +import info.frostfs.sdk.dto.object.ObjectFrostFS; +import info.frostfs.sdk.dto.object.ObjectHeader; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.OwnerId; +import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.exceptions.FrostFSException; +import info.frostfs.sdk.exceptions.SessionExpiredFrostFSException; +import info.frostfs.sdk.exceptions.SessionNotFoundFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import info.frostfs.sdk.jdo.ECDsa; +import info.frostfs.sdk.jdo.NetworkSettings; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainRemove; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerDelete; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGet; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGetAll; +import info.frostfs.sdk.jdo.parameters.object.*; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmObjectPatch; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeGet; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeHashGet; +import info.frostfs.sdk.jdo.parameters.session.PrmSessionCreate; +import info.frostfs.sdk.jdo.pool.NodeParameters; +import info.frostfs.sdk.jdo.pool.PoolInitParameters; +import info.frostfs.sdk.jdo.result.ObjectHeaderResult; +import info.frostfs.sdk.services.CommonClient; +import info.frostfs.sdk.services.impl.rwhelper.ObjectWriter; +import info.frostfs.sdk.services.impl.rwhelper.RangeReader; +import info.frostfs.sdk.utils.FrostFSMessages; +import info.frostfs.sdk.utils.WaitUtil; +import lombok.Getter; +import org.slf4j.Logger; + +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; + +import static info.frostfs.sdk.Helper.getHexString; +import static info.frostfs.sdk.constants.ErrorConst.*; +import static info.frostfs.sdk.constants.PoolConst.*; + +@Getter +public class Pool implements CommonClient { + + private final ReentrantLock lock = new ReentrantLock(); + private final ECDsa key; + private final SessionCache sessionCache; + private final long sessionTokenDuration; + private final RebalanceParameters rebalanceParams; + private final Function clientBuilder; + private final Logger logger; + private InnerPool[] innerPools; + private Types.OwnerID ownerID; + private OwnerId ownerId; + private boolean disposedValue; + private long maxObjectSize; + private ClientStatus clientStatus; + + public Pool(PoolInitParameters options) { + if (options == null || options.getKey() == null) { + throw new ValidationFrostFSException( + String.format( + PARAMS_ARE_MISSING_TEMPLATE, + String.join( + FIELDS_DELIMITER_COMMA, PoolInitParameters.class.getName(), ECDsa.class.getName() + ) + ) + ); + } + + List nodesParams = adjustNodeParams(options.getNodeParams()); + SessionCache cache = new SessionCache(options.getSessionExpirationDuration()); + fillDefaultInitParams(options, this); + + this.key = options.getKey(); + this.sessionCache = cache; + this.logger = options.getLogger(); + this.sessionTokenDuration = options.getSessionExpirationDuration(); + + this.rebalanceParams = new RebalanceParameters( + nodesParams.toArray(new NodesParam[0]), + options.getHealthCheckTimeout(), + options.getClientRebalanceInterval(), + options.getSessionExpirationDuration()); + + this.clientBuilder = options.getClientBuilder(); + } + + private static List adjustNodeParams(NodeParameters[] nodeParams) { + if (nodeParams == null || nodeParams.length == 0) { + throw new ValidationFrostFSException(POOL_PEERS_IS_MISSING); + } + + Map nodesParamsDict = new HashMap<>(nodeParams.length); + for (NodeParameters nodeParam : nodeParams) { + var nodesParam = nodesParamsDict + .computeIfAbsent(nodeParam.getPriority(), k -> new NodesParam(nodeParam.getPriority())); + nodesParam.getAddress().add(nodeParam.getAddress()); + nodesParam.getWeight().add(nodeParam.getWeight()); + } + + List nodesParams = new ArrayList<>(nodesParamsDict.values()); + nodesParams.sort(Comparator.comparingInt(NodesParam::getPriority)); + + for (NodesParam nodes : nodesParams) { + double[] newWeights = adjustWeights(nodes.getWeight().stream().mapToDouble(Double::doubleValue).toArray()); + nodes.getWeight().clear(); + for (double weight : newWeights) { + nodes.getWeight().add(weight); + } + } + + return nodesParams; + } + + private static double[] adjustWeights(double[] weights) { + double[] adjusted = new double[weights.length]; + double sum = Arrays.stream(weights).sum(); + + if (sum > 0) { + for (int i = 0; i < weights.length; i++) { + adjusted[i] = weights[i] / sum; + } + } + + return adjusted; + } + + private static void fillDefaultInitParams(PoolInitParameters parameters, Pool pool) { + if (parameters.getSessionExpirationDuration() == 0) { + parameters.setSessionExpirationDuration(DEFAULT_SESSION_TOKEN_EXPIRATION_DURATION); + } + + if (parameters.getErrorThreshold() == 0) { + parameters.setErrorThreshold(DEFAULT_ERROR_THRESHOLD); + } + + if (parameters.getClientRebalanceInterval() <= 0) { + parameters.setClientRebalanceInterval(DEFAULT_REBALANCE_INTERVAL); + } + + if (parameters.getGracefulCloseOnSwitchTimeout() <= 0) { + parameters.setGracefulCloseOnSwitchTimeout(DEFAULT_GRACEFUL_CLOSE_ON_SWITCH_TIMEOUT); + } + + if (parameters.getHealthCheckTimeout() <= 0) { + parameters.setHealthCheckTimeout(DEFAULT_HEALTHCHECK_TIMEOUT); + } + + if (parameters.getNodeDialTimeout() <= 0) { + parameters.setNodeDialTimeout(DEFAULT_DIAL_TIMEOUT); + } + + if (parameters.getNodeStreamTimeout() <= 0) { + parameters.setNodeStreamTimeout(DEFAULT_STREAM_TIMEOUT); + } + + if (parameters.getSessionExpirationDuration() == 0) { + parameters.setSessionExpirationDuration(DEFAULT_SESSION_TOKEN_EXPIRATION_DURATION); + } + + if (parameters.getClientBuilder() == null) { + parameters.setClientBuilder(address -> { + WrapperPrm wrapperPrm = new WrapperPrm(); + wrapperPrm.setAddress(address); + wrapperPrm.setKey(parameters.getKey()); + wrapperPrm.setLogger(parameters.getLogger()); + wrapperPrm.setDialTimeout(parameters.getNodeDialTimeout()); + wrapperPrm.setStreamTimeout(parameters.getNodeStreamTimeout()); + wrapperPrm.setErrorThreshold(parameters.getErrorThreshold()); + wrapperPrm.setGracefulCloseOnSwitchTimeout(parameters.getGracefulCloseOnSwitchTimeout()); + wrapperPrm.setInterceptors(parameters.getInterceptors()); + return new ClientWrapper(wrapperPrm, pool); + }); + } + } + + private static SessionToken initSessionForDuration(CallContext ctx, ClientWrapper cw, long duration) { + var client = cw.getClient(); + NetworkSettings networkInfo = client.getNetworkSettings(ctx); + + long epoch = networkInfo.getEpochDuration(); + long exp = (Long.MAX_VALUE - epoch < duration) ? Long.MAX_VALUE : (epoch + duration); + + return client.createSession(new PrmSessionCreate(exp), ctx); + } + + public static String formCacheKey(String address, String key) { + return address + key; + } + + @Override + public String dial(CallContext ctx) { + InnerPool[] inner = new InnerPool[rebalanceParams.getNodesParams().length]; + boolean atLeastOneHealthy = false; + int i = 0; + + for (NodesParam nodeParams : rebalanceParams.getNodesParams()) { + ClientWrapper[] clients = new ClientWrapper[nodeParams.getWeight().size()]; + + for (int j = 0; j < nodeParams.getAddress().size(); j++) { + ClientWrapper client = clients[j] = clientBuilder.apply(nodeParams.getAddress().get(j)); + boolean dialed = false; + + try { + client.dial(ctx); + dialed = true; + + SessionToken token = initSessionForDuration( + ctx, client, rebalanceParams.getSessionExpirationDuration() + ); + String cacheKey = formCacheKey( + nodeParams.getAddress().get(j), + getHexString(key.getPublicKeyByte()) + ); + sessionCache.setValue(cacheKey, token); + + atLeastOneHealthy = true; + } catch (ValidationFrostFSException exp) { + break; + } catch (Exception exp) { + if (!dialed) { + client.setUnhealthyOnDial(); + } else { + client.setUnhealthy(); + } + + if (logger != null) { + FrostFSMessages + .sessionCreationError(logger, client.getWrapperPrm().getAddress(), exp.getMessage()); + } + } + } + + Sampler sampler = new Sampler(nodeParams.getWeight().stream().mapToDouble(Double::doubleValue).toArray()); + inner[i] = new InnerPool(sampler, clients); + i++; + } + + if (!atLeastOneHealthy) { + return POOL_NODES_UNHEALTHY; + } + + this.innerPools = inner; + + NetworkSettings networkSettings = getNetworkSettings(ctx); + + this.maxObjectSize = networkSettings.getMaxObjectSize(); + startRebalance(ctx); + + return null; + } + + private ClientWrapper connection() { + for (InnerPool pool : innerPools) { + ClientWrapper client = pool.connection(); + if (client != null) { + return client; + } + } + + throw new FrostFSException(POOL_CLIENTS_UNHEALTHY); + } + + public void close() { + if (innerPools != null) { + for (InnerPool innerPool : innerPools) { + for (ClientWrapper client : innerPool.getClients()) { + if (client.isDialed()) { + client.getClient().close(); + } + } + } + } + } + + public void startRebalance(CallContext ctx) { + double[][] buffers = new double[rebalanceParams.getNodesParams().length][]; + + for (int i = 0; i < rebalanceParams.getNodesParams().length; i++) { + NodesParam parameters = rebalanceParams.getNodesParams()[i]; + buffers[i] = new double[parameters.getWeight().size()]; + + CompletableFuture.runAsync(() -> { + WaitUtil.sleep(rebalanceParams.getClientRebalanceInterval()); + updateNodesHealth(ctx, buffers); + }); + } + } + + private void updateNodesHealth(CallContext ctx, double[][] buffers) { + CompletableFuture[] tasks = new CompletableFuture[innerPools.length]; + + for (int i = 0; i < innerPools.length; i++) { + double[] bufferWeights = buffers[i]; + int finalI = i; + tasks[i] = CompletableFuture.runAsync(() -> updateInnerNodesHealth(ctx, finalI, bufferWeights)); + } + + CompletableFuture.allOf(tasks).join(); + } + + private void updateInnerNodesHealth(CallContext ctx, int poolIndex, double[] bufferWeights) { + if (poolIndex > innerPools.length - 1) { + return; + } + + InnerPool pool = innerPools[poolIndex]; + RebalanceParameters options = rebalanceParams; + int[] healthyChanged = {0}; + + CompletableFuture[] tasks = new CompletableFuture[pool.getClients().length]; + + for (int j = 0; j < pool.getClients().length; j++) { + ClientWrapper client = pool.getClients()[j]; + AtomicBoolean healthy = new AtomicBoolean(false); + AtomicReference error = new AtomicReference<>(); + AtomicBoolean changed = new AtomicBoolean(false); + + int finalJ = j; + tasks[j] = client.restartIfUnhealthy(ctx).handle((unused, throwable) -> { + if (throwable != null) { + error.set(throwable.getMessage()); + bufferWeights[finalJ] = 0; + sessionCache.deleteByPrefix(client.getAddress()); + } else { + changed.set(unused); + healthy.set(true); + bufferWeights[finalJ] = options.getNodesParams()[poolIndex].getWeight().get(finalJ); + } + return null; + }).thenRun(() -> { + if (changed.get()) { + if (error.get() != null && logger != null) { + FrostFSMessages.healthChanged(logger, client.getAddress(), healthy.get(), error.get()); + } + healthyChanged[0] = 1; + } + }); + } + + CompletableFuture.allOf(tasks).thenRun(() -> { + if (healthyChanged[0] == 1) { + double[] probabilities = adjustWeights(bufferWeights); + lock.lock(); + try { + pool.setSampler(new Sampler(probabilities)); + } finally { + lock.unlock(); + } + } + }); + } + + private boolean checkSessionTokenErr(Exception error, String address) { + if (error == null) { + return false; + } + + if (error instanceof SessionNotFoundFrostFSException || error instanceof SessionExpiredFrostFSException) { + sessionCache.deleteByPrefix(address); + return true; + } + + return false; + } + + public Statistic statistic() { + if (innerPools == null) { + throw new ValidationFrostFSException(POOL_NOT_DIALED); + } + + Statistic statistics = new Statistic(); + + for (InnerPool inner : innerPools) { + int valueIndex = 0; + String[] nodes = new String[inner.getClients().length]; + + lock.lock(); + try { + for (ClientWrapper client : inner.getClients()) { + if (client.isHealthy()) { + nodes[valueIndex] = client.getAddress(); + } + + NodeStatistic node = new NodeStatistic(); + node.setAddress(client.getAddress()); + node.setMethods(client.getMethodsStatus()); + node.setOverallErrors(client.getOverallErrorRate()); + node.setCurrentErrors(client.getCurrentErrorRate()); + + statistics.getNodes().add(node); + valueIndex++; + statistics.setOverallErrors(statistics.getOverallErrors() + node.getOverallErrors()); + } + + if (statistics.getCurrentNodes() == null || statistics.getCurrentNodes().length == 0) { + statistics.setCurrentNodes(nodes); + } + } finally { + lock.unlock(); + } + } + + return statistics; + } + + @Override + public Container getContainer(PrmContainerGet args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getContainer(args, ctx); + } + + @Override + public List listContainers(PrmContainerGetAll args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().listContainers(args, ctx); + } + + @Override + public ContainerId createContainer(PrmContainerCreate args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().createContainer(args, ctx); + } + + @Override + public void deleteContainer(PrmContainerDelete args, CallContext ctx) { + ClientWrapper client = connection(); + client.getClient().deleteContainer(args, ctx); + } + + @Override + public ObjectHeaderResult getObjectHead(PrmObjectHeadGet args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getObjectHead(args, ctx); + } + + @Override + public ObjectFrostFS getObject(PrmObjectGet args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getObject(args, ctx); + } + + @Override + public ObjectWriter putObject(PrmObjectPut args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().putObject(args, ctx); + } + + @Override + public ObjectId putClientCutObject(PrmObjectClientCutPut args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().putClientCutObject(args, ctx); + } + + @Override + public ObjectId putSingleObject(PrmObjectSinglePut args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().putSingleObject(args, ctx); + } + + @Override + public void deleteObject(PrmObjectDelete args, CallContext ctx) { + ClientWrapper client = connection(); + client.getClient().deleteObject(args, ctx); + } + + @Override + public Iterable searchObjects(PrmObjectSearch args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().searchObjects(args, ctx); + } + + @Override + public RangeReader getRange(PrmRangeGet args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getRange(args, ctx); + } + + @Override + public byte[][] getRangeHash(PrmRangeHashGet args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getRangeHash(args, ctx); + } + + @Override + public ObjectId patchObject(PrmObjectPatch args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().patchObject(args, ctx); + } + + @Override + public byte[] addChain(PrmApeChainAdd args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().addChain(args, ctx); + } + + @Override + public void removeChain(PrmApeChainRemove args, CallContext ctx) { + ClientWrapper client = connection(); + client.getClient().removeChain(args, ctx); + } + + @Override + public List listChains(PrmApeChainList args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().listChains(args, ctx); + } + + @Override + public NetmapSnapshot getNetmapSnapshot(CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getNetmapSnapshot(ctx); + } + + @Override + public NodeInfo getLocalNodeInfo(CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getLocalNodeInfo(ctx); + } + + @Override + public NetworkSettings getNetworkSettings(CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getNetworkSettings(ctx); + } + + @Override + public SessionToken createSession(PrmSessionCreate args, CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().createSession(args, ctx); + } + + @Override + public ObjectId calculateObjectId(ObjectHeader header) { + ClientWrapper client = connection(); + return client.getClient().calculateObjectId(header); + } + + @Override + public frostfs.accounting.Types.Decimal getBalance(CallContext ctx) { + ClientWrapper client = connection(); + return client.getClient().getBalance(ctx); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/RebalanceParameters.java b/client/src/main/java/info/frostfs/sdk/pool/RebalanceParameters.java new file mode 100644 index 0000000..d7bd790 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/RebalanceParameters.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.pool; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +public class RebalanceParameters { + private NodesParam[] nodesParams; + private long nodeRequestTimeout; + private long clientRebalanceInterval; + private long sessionExpirationDuration; +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/RequestInfo.java b/client/src/main/java/info/frostfs/sdk/pool/RequestInfo.java new file mode 100644 index 0000000..68d8312 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/RequestInfo.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.pool; + +import info.frostfs.sdk.enums.MethodIndex; +import lombok.Getter; +import lombok.Setter; + +import java.time.Duration; + +@Getter +@Setter +public class RequestInfo { + private String address; + private MethodIndex methodIndex; + private Duration elapsed; +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/Sampler.java b/client/src/main/java/info/frostfs/sdk/pool/Sampler.java new file mode 100644 index 0000000..505243a --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/Sampler.java @@ -0,0 +1,77 @@ +package info.frostfs.sdk.pool; + +import java.util.ArrayList; +import java.util.Random; + +class Sampler { + private final Object lock = new Object(); + private final Random random = new Random(); + private final double[] probabilities; + private final int[] alias; + + Sampler(double[] probabilities) { + ArrayList small = new ArrayList<>(); + ArrayList large = new ArrayList<>(); + + int n = probabilities.length; + + this.probabilities = new double[n]; + this.alias = new int[n]; + + // Compute scaled probabilities. + double[] p = new double[n]; + + for (int i = 0; i < n; i++) { + p[i] = probabilities[i] * n; + if (p[i] < 1) { + small.add(i); + } else { + large.add(i); + } + } + + while (!small.isEmpty() && !large.isEmpty()) { + int l = small.remove(small.size() - 1); + int g = large.remove(large.size() - 1); + + this.probabilities[l] = p[l]; + this.alias[l] = g; + + p[g] = p[g] + p[l] - 1; + + if (p[g] < 1) { + small.add(g); + } else { + large.add(g); + } + } + + while (!large.isEmpty()) { + int g = large.remove(large.size() - 1); + this.probabilities[g] = 1; + } + + while (!small.isEmpty()) { + int l = small.remove(small.size() - 1); + probabilities[l] = 1; + } + } + + int next() { + int n = alias.length; + + int i; + double f; + synchronized (lock) { + i = random.nextInt(n); + f = random.nextDouble(); + } + + if (f < probabilities[i]) { + return i; + } + + return alias[i]; + } +} + diff --git a/client/src/main/java/info/frostfs/sdk/pool/SessionCache.java b/client/src/main/java/info/frostfs/sdk/pool/SessionCache.java new file mode 100644 index 0000000..998beb6 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/SessionCache.java @@ -0,0 +1,37 @@ +package info.frostfs.sdk.pool; + +import info.frostfs.sdk.dto.session.SessionToken; +import org.apache.commons.lang3.StringUtils; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +public class SessionCache { + private final ConcurrentMap cache = new ConcurrentHashMap<>(); + private final long tokenDuration; + private long currentEpoch; + + public SessionCache(long sessionExpirationDuration) { + this.tokenDuration = sessionExpirationDuration; + } + + public boolean contains(String key) { + return cache.containsKey(key); + } + + public SessionToken tryGetValue(String key) { + return StringUtils.isBlank(key) ? null : cache.get(key); + } + + public void setValue(String key, SessionToken value) { + if (key != null) { + cache.put(key, value); + } + } + + public void deleteByPrefix(String prefix) { + cache.keySet().removeIf(key -> key.startsWith(prefix)); + } +} + + diff --git a/client/src/main/java/info/frostfs/sdk/pool/Statistic.java b/client/src/main/java/info/frostfs/sdk/pool/Statistic.java new file mode 100644 index 0000000..c7bc2ea --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/Statistic.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.pool; + +import lombok.Getter; +import lombok.Setter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class Statistic { + private long overallErrors; + private List nodes = new ArrayList<>(); + private String[] currentNodes; +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/StatusSnapshot.java b/client/src/main/java/info/frostfs/sdk/pool/StatusSnapshot.java new file mode 100644 index 0000000..99a797b --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/StatusSnapshot.java @@ -0,0 +1,13 @@ +package info.frostfs.sdk.pool; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class StatusSnapshot { + private long allTime; + private long allRequests; +} + + diff --git a/client/src/main/java/info/frostfs/sdk/pool/WorkList.java b/client/src/main/java/info/frostfs/sdk/pool/WorkList.java new file mode 100644 index 0000000..7e7a455 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/WorkList.java @@ -0,0 +1,22 @@ +package info.frostfs.sdk.pool; + +import java.util.ArrayList; +import java.util.List; + +class WorkList { + private final List elements = new ArrayList<>(); + + private int getLength() { + return elements.size(); + } + + private void add(int element) { + elements.add(element); + } + + private int remove() { + int last = elements.get(elements.size() - 1); + elements.remove(elements.size() - 1); + return last; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/pool/WrapperPrm.java b/client/src/main/java/info/frostfs/sdk/pool/WrapperPrm.java new file mode 100644 index 0000000..d03efc4 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/pool/WrapperPrm.java @@ -0,0 +1,26 @@ +package info.frostfs.sdk.pool; + +import info.frostfs.sdk.jdo.ECDsa; +import io.grpc.ClientInterceptors; +import io.grpc.ManagedChannelBuilder; +import lombok.Getter; +import lombok.Setter; +import org.slf4j.Logger; + +import java.util.Collection; + +@Getter +@Setter +public class WrapperPrm { + private Logger logger; + private String address; + private ECDsa key; + private long dialTimeout; + private long streamTimeout; + private int errorThreshold; + private Runnable responseInfoCallback; + private Runnable poolRequestInfoCallback; + private ManagedChannelBuilder grpcChannelOptions; + private long gracefulCloseOnSwitchTimeout; + private Collection interceptors; +} diff --git a/client/src/main/java/info/frostfs/sdk/services/AccountingClient.java b/client/src/main/java/info/frostfs/sdk/services/AccountingClient.java new file mode 100644 index 0000000..28c2516 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/AccountingClient.java @@ -0,0 +1,8 @@ +package info.frostfs.sdk.services; + +import frostfs.accounting.Types; +import info.frostfs.sdk.jdo.parameters.CallContext; + +public interface AccountingClient { + Types.Decimal getBalance(CallContext ctx); +} diff --git a/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java b/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java index 5c5263c..47d7827 100644 --- a/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/ApeManagerClient.java @@ -1,14 +1,17 @@ package info.frostfs.sdk.services; -import info.frostfs.sdk.dto.chain.Chain; -import info.frostfs.sdk.dto.chain.ChainTarget; +import info.frostfs.sdk.dto.ape.Chain; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainRemove; import java.util.List; public interface ApeManagerClient { - byte[] addChain(Chain chain, ChainTarget chainTarget); + byte[] addChain(PrmApeChainAdd args, CallContext ctx); - void removeChain(Chain chain, ChainTarget chainTarget); + void removeChain(PrmApeChainRemove args, CallContext ctx); - List listChains(ChainTarget chainTarget); + List listChains(PrmApeChainList args, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/CommonClient.java b/client/src/main/java/info/frostfs/sdk/services/CommonClient.java new file mode 100644 index 0000000..ce0dcba --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/CommonClient.java @@ -0,0 +1,9 @@ +package info.frostfs.sdk.services; + +import info.frostfs.sdk.jdo.parameters.CallContext; + +public interface CommonClient extends + AccountingClient, ApeManagerClient, ContainerClient, NetmapClient, ObjectClient, SessionClient, ToolsClient { + + String dial(CallContext ctx); +} diff --git a/client/src/main/java/info/frostfs/sdk/services/ContainerClient.java b/client/src/main/java/info/frostfs/sdk/services/ContainerClient.java index d3f7f9c..121bc80 100644 --- a/client/src/main/java/info/frostfs/sdk/services/ContainerClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/ContainerClient.java @@ -2,15 +2,20 @@ package info.frostfs.sdk.services; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerDelete; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGet; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGetAll; import java.util.List; public interface ContainerClient { - Container getContainer(ContainerId cid); + Container getContainer(PrmContainerGet args, CallContext ctx); - List listContainers(); + List listContainers(PrmContainerGetAll args, CallContext ctx); - ContainerId createContainer(Container container); + ContainerId createContainer(PrmContainerCreate args, CallContext ctx); - void deleteContainer(ContainerId cid); + void deleteContainer(PrmContainerDelete args, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/NetmapClient.java b/client/src/main/java/info/frostfs/sdk/services/NetmapClient.java index bc34978..067c836 100644 --- a/client/src/main/java/info/frostfs/sdk/services/NetmapClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/NetmapClient.java @@ -3,11 +3,12 @@ package info.frostfs.sdk.services; import info.frostfs.sdk.dto.netmap.NetmapSnapshot; import info.frostfs.sdk.dto.netmap.NodeInfo; import info.frostfs.sdk.jdo.NetworkSettings; +import info.frostfs.sdk.jdo.parameters.CallContext; public interface NetmapClient { - NetmapSnapshot getNetmapSnapshot(); + NetmapSnapshot getNetmapSnapshot(CallContext ctx); - NodeInfo getLocalNodeInfo(); + NodeInfo getLocalNodeInfo(CallContext ctx); - NetworkSettings getNetworkSettings(); + NetworkSettings getNetworkSettings(CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/ObjectClient.java b/client/src/main/java/info/frostfs/sdk/services/ObjectClient.java index 3a38295..5e8362c 100644 --- a/client/src/main/java/info/frostfs/sdk/services/ObjectClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/ObjectClient.java @@ -1,22 +1,34 @@ package info.frostfs.sdk.services; -import info.frostfs.sdk.dto.container.ContainerId; -import info.frostfs.sdk.dto.object.ObjectFilter; import info.frostfs.sdk.dto.object.ObjectFrostFS; -import info.frostfs.sdk.dto.object.ObjectHeader; import info.frostfs.sdk.dto.object.ObjectId; -import info.frostfs.sdk.jdo.PutObjectParameters; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.object.*; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmObjectPatch; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeGet; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeHashGet; +import info.frostfs.sdk.jdo.result.ObjectHeaderResult; +import info.frostfs.sdk.services.impl.rwhelper.ObjectWriter; +import info.frostfs.sdk.services.impl.rwhelper.RangeReader; public interface ObjectClient { - ObjectHeader getObjectHead(ContainerId containerId, ObjectId objectId); + ObjectHeaderResult getObjectHead(PrmObjectHeadGet args, CallContext ctx); - ObjectFrostFS getObject(ContainerId containerId, ObjectId objectId); + ObjectFrostFS getObject(PrmObjectGet args, CallContext ctx); - ObjectId putObject(PutObjectParameters parameters); + ObjectWriter putObject(PrmObjectPut args, CallContext ctx); - ObjectId putSingleObject(ObjectFrostFS objectFrostFS); + ObjectId putClientCutObject(PrmObjectClientCutPut args, CallContext ctx); - void deleteObject(ContainerId containerId, ObjectId objectId); + ObjectId putSingleObject(PrmObjectSinglePut args, CallContext ctx); - Iterable searchObjects(ContainerId cid, ObjectFilter... filters); + void deleteObject(PrmObjectDelete args, CallContext ctx); + + Iterable searchObjects(PrmObjectSearch args, CallContext ctx); + + RangeReader getRange(PrmRangeGet args, CallContext ctx); + + byte[][] getRangeHash(PrmRangeHashGet args, CallContext ctx); + + ObjectId patchObject(PrmObjectPatch args, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/SessionClient.java b/client/src/main/java/info/frostfs/sdk/services/SessionClient.java index e9b197c..4e5f0a9 100644 --- a/client/src/main/java/info/frostfs/sdk/services/SessionClient.java +++ b/client/src/main/java/info/frostfs/sdk/services/SessionClient.java @@ -1,7 +1,9 @@ package info.frostfs.sdk.services; import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.session.PrmSessionCreate; public interface SessionClient { - SessionToken createSession(long expiration); + SessionToken createSession(PrmSessionCreate args, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/SessionTools.java b/client/src/main/java/info/frostfs/sdk/services/SessionTools.java index 7976592..b389235 100644 --- a/client/src/main/java/info/frostfs/sdk/services/SessionTools.java +++ b/client/src/main/java/info/frostfs/sdk/services/SessionTools.java @@ -1,9 +1,9 @@ package info.frostfs.sdk.services; -import frostfs.session.Types; import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.CallContext; public interface SessionTools { - Types.SessionToken getOrCreateSession(SessionToken token, ClientEnvironment env); + SessionToken getOrCreateSession(ClientEnvironment env, CallContext ctx); } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/AccountingClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/AccountingClientImpl.java new file mode 100644 index 0000000..7aee62b --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/impl/AccountingClientImpl.java @@ -0,0 +1,48 @@ +package info.frostfs.sdk.services.impl; + +import frostfs.accounting.AccountingServiceGrpc; +import frostfs.accounting.Service; +import frostfs.accounting.Types; +import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.mappers.object.OwnerIdMapper; +import info.frostfs.sdk.services.AccountingClient; +import info.frostfs.sdk.services.ContextAccessor; +import info.frostfs.sdk.tools.RequestConstructor; +import info.frostfs.sdk.tools.RequestSigner; +import info.frostfs.sdk.tools.Verifier; + +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; + +public class AccountingClientImpl extends ContextAccessor implements AccountingClient { + private final AccountingServiceGrpc.AccountingServiceBlockingStub serviceBlockingStub; + + public AccountingClientImpl(ClientEnvironment clientEnvironment) { + super(clientEnvironment); + this.serviceBlockingStub = AccountingServiceGrpc.newBlockingStub(getContext().getChannel()); + } + + @Override + public Types.Decimal getBalance(CallContext ctx) { + var request = createGetRequest(); + + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.balance(request); + + Verifier.checkResponse(response); + return response.getBody().getBalance(); + } + + private Service.BalanceRequest createGetRequest() { + var body = Service.BalanceRequest.Body.newBuilder() + .setOwnerId(OwnerIdMapper.toGrpcMessage(getContext().getOwnerId())) + .build(); + var request = Service.BalanceRequest.newBuilder() + .setBody(body); + + RequestConstructor.addMetaHeader(request); + RequestSigner.sign(request, getContext().getKey()); + + return request.build(); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java index 0e5c9f1..6e10fe3 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ApeManagerClientImpl.java @@ -4,23 +4,26 @@ import com.google.protobuf.ByteString; import frostfs.ape.Types; import frostfs.apemanager.APEManagerServiceGrpc; import frostfs.apemanager.Service; -import info.frostfs.sdk.dto.chain.Chain; -import info.frostfs.sdk.dto.chain.ChainTarget; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import info.frostfs.sdk.dto.ape.Chain; import info.frostfs.sdk.jdo.ClientEnvironment; -import info.frostfs.sdk.mappers.chain.ChainMapper; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainRemove; import info.frostfs.sdk.mappers.chain.ChainTargetMapper; import info.frostfs.sdk.services.ApeManagerClient; import info.frostfs.sdk.services.ContextAccessor; import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.RequestSigner; import info.frostfs.sdk.tools.Verifier; +import info.frostfs.sdk.tools.ape.RuleDeserializer; +import info.frostfs.sdk.tools.ape.RuleSerializer; import java.util.List; -import java.util.Map; +import java.util.stream.Collectors; -import static info.frostfs.sdk.constants.ErrorConst.*; -import static java.util.Objects.isNull; +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; +import static info.frostfs.sdk.utils.Validator.validate; public class ApeManagerClientImpl extends ContextAccessor implements ApeManagerClient { private final APEManagerServiceGrpc.APEManagerServiceBlockingStub apeManagerServiceClient; @@ -31,19 +34,13 @@ public class ApeManagerClientImpl extends ContextAccessor implements ApeManagerC } @Override - public byte[] addChain(Chain chain, ChainTarget chainTarget) { - if (isNull(chain) || isNull(chainTarget)) { - throw new ValidationFrostFSException( - String.format( - PARAMS_ARE_MISSING_TEMPLATE, - String.join(FIELDS_DELIMITER_COMMA, Chain.class.getName(), ChainTarget.class.getName()) - ) - ); - } + public byte[] addChain(PrmApeChainAdd args, CallContext ctx) { + validate(args); - var request = createAddChainRequest(chain, chainTarget, null); + var request = createAddChainRequest(args); - var response = apeManagerServiceClient.addChain(request); + var service = deadLineAfter(apeManagerServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.addChain(request); Verifier.checkResponse(response); @@ -51,82 +48,74 @@ public class ApeManagerClientImpl extends ContextAccessor implements ApeManagerC } @Override - public void removeChain(Chain chain, ChainTarget chainTarget) { - if (isNull(chain) || isNull(chainTarget)) { - throw new ValidationFrostFSException( - String.format( - PARAMS_ARE_MISSING_TEMPLATE, - String.join(FIELDS_DELIMITER_COMMA, Chain.class.getName(), ChainTarget.class.getName()) - ) - ); - } + public void removeChain(PrmApeChainRemove args, CallContext ctx) { + validate(args); - var request = createRemoveChainRequest(chain, chainTarget, null); + var request = createRemoveChainRequest(args); - var response = apeManagerServiceClient.removeChain(request); + var service = deadLineAfter(apeManagerServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.removeChain(request); Verifier.checkResponse(response); } @Override - public List listChains(ChainTarget chainTarget) { - if (isNull(chainTarget)) { - throw new ValidationFrostFSException(String.format(PARAM_IS_MISSING_TEMPLATE, ChainTarget.class.getName())); - } + public List listChains(PrmApeChainList args, CallContext ctx) { + validate(args); - var request = createListChainsRequest(chainTarget, null); + var request = createListChainsRequest(args); - var response = apeManagerServiceClient.listChains(request); + var service = deadLineAfter(apeManagerServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.listChains(request); Verifier.checkResponse(response); - return ChainMapper.toModels(response.getBody().getChainsList()); + return response.getBody().getChainsList().stream() + .map(chain -> RuleDeserializer.deserialize(chain.getRaw().toByteArray())) + .collect(Collectors.toList()); } - private Service.AddChainRequest createAddChainRequest(Chain chain, - ChainTarget chainTarget, - Map xHeaders) { + private Service.AddChainRequest createAddChainRequest(PrmApeChainAdd args) { + var raw = RuleSerializer.serialize(args.getChain()); + var chainGrpc = Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(chain.getRaw())) + .setRaw(ByteString.copyFrom(raw)) .build(); var body = Service.AddChainRequest.Body.newBuilder() .setChain(chainGrpc) - .setTarget(ChainTargetMapper.toGrpcMessage(chainTarget)) + .setTarget(ChainTargetMapper.toGrpcMessage(args.getChainTarget())) .build(); var request = Service.AddChainRequest.newBuilder() .setBody(body); - RequestConstructor.addMetaHeader(request, xHeaders); + RequestConstructor.addMetaHeader(request, args.getXHeaders()); RequestSigner.sign(request, getContext().getKey()); return request.build(); } - private Service.RemoveChainRequest createRemoveChainRequest(Chain chain, - ChainTarget chainTarget, - Map xHeaders) { + private Service.RemoveChainRequest createRemoveChainRequest(PrmApeChainRemove args) { var body = Service.RemoveChainRequest.Body.newBuilder() - .setChainId(ByteString.copyFrom(chain.getRaw())) - .setTarget(ChainTargetMapper.toGrpcMessage(chainTarget)) + .setChainId(ByteString.copyFrom(args.getChainId())) + .setTarget(ChainTargetMapper.toGrpcMessage(args.getChainTarget())) .build(); var request = Service.RemoveChainRequest.newBuilder() .setBody(body); - RequestConstructor.addMetaHeader(request, xHeaders); + RequestConstructor.addMetaHeader(request, args.getXHeaders()); RequestSigner.sign(request, getContext().getKey()); return request.build(); } - private Service.ListChainsRequest createListChainsRequest(ChainTarget chainTarget, - Map xHeaders) { + private Service.ListChainsRequest createListChainsRequest(PrmApeChainList args) { var body = Service.ListChainsRequest.Body.newBuilder() - .setTarget(ChainTargetMapper.toGrpcMessage(chainTarget)) + .setTarget(ChainTargetMapper.toGrpcMessage(args.getChainTarget())) .build(); var request = Service.ListChainsRequest.newBuilder() .setBody(body); - RequestConstructor.addMetaHeader(request, xHeaders); + RequestConstructor.addMetaHeader(request, args.getXHeaders()); RequestSigner.sign(request, getContext().getKey()); return request.build(); diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ContainerClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ContainerClientImpl.java index 77468af..7e32ca1 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ContainerClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ContainerClientImpl.java @@ -10,9 +10,14 @@ import info.frostfs.sdk.enums.StatusCode; import info.frostfs.sdk.enums.WaitExpects; import info.frostfs.sdk.exceptions.ResponseFrostFSException; import info.frostfs.sdk.exceptions.TimeoutFrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; -import info.frostfs.sdk.jdo.WaitParameters; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.PrmWait; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerCreate; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerDelete; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGet; +import info.frostfs.sdk.jdo.parameters.container.PrmContainerGetAll; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; import info.frostfs.sdk.mappers.container.ContainerIdMapper; import info.frostfs.sdk.mappers.container.ContainerMapper; import info.frostfs.sdk.mappers.netmap.VersionMapper; @@ -29,8 +34,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static info.frostfs.sdk.constants.ErrorConst.PARAM_IS_MISSING_TEMPLATE; +import static info.frostfs.sdk.constants.AttributeConst.DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE; +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; +import static info.frostfs.sdk.utils.Validator.validate; import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; public class ContainerClientImpl extends ContextAccessor implements ContainerClient { private final ContainerServiceGrpc.ContainerServiceBlockingStub serviceBlockingStub; @@ -42,29 +50,33 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli this.sessionTools = new SessionToolsImpl(clientEnvironment); } - public frostfs.session.Types.SessionToken getOrCreateSession(SessionToken sessionToken) { - return sessionTools.getOrCreateSession(sessionToken, getContext()); + public SessionToken getOrCreateSession(SessionContext sessionContext, CallContext ctx) { + return isNull(sessionContext.getSessionToken()) + ? sessionTools.getOrCreateSession(getContext(), ctx) + : sessionContext.getSessionToken(); } @Override - public Container getContainer(ContainerId cid) { - if (isNull(cid)) { - throw new ValidationFrostFSException(String.format(PARAM_IS_MISSING_TEMPLATE, ContainerId.class.getName())); - } + public Container getContainer(PrmContainerGet args, CallContext ctx) { + validate(args); - var request = createGetRequest(ContainerIdMapper.toGrpcMessage(cid), null); + var request = createGetRequest(args); - var response = serviceBlockingStub.get(request); + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.get(request); Verifier.checkResponse(response); return ContainerMapper.toModel(response.getBody().getContainer()); } @Override - public List listContainers() { - var request = createListRequest(null); + public List listContainers(PrmContainerGetAll args, CallContext ctx) { + validate(args); - var response = serviceBlockingStub.list(request); + var request = createListRequest(args); + + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.list(request); Verifier.checkResponse(response); @@ -74,43 +86,39 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli } @Override - public ContainerId createContainer(Container container) { - if (isNull(container)) { - throw new ValidationFrostFSException(String.format(PARAM_IS_MISSING_TEMPLATE, Container.class.getName())); - } + public ContainerId createContainer(PrmContainerCreate args, CallContext ctx) { + validate(args); - var grpcContainer = ContainerMapper.toGrpcMessage(container); - var request = createPutRequest(grpcContainer, null); + var request = createPutRequest(args, ctx); - var response = serviceBlockingStub.put(request); + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.put(request); Verifier.checkResponse(response); - waitForContainer(WaitExpects.EXISTS, response.getBody().getContainerId(), null); + waitForContainer(WaitExpects.EXISTS, response.getBody().getContainerId(), args.getWaitParams()); return new ContainerId(response.getBody().getContainerId().getValue().toByteArray()); } @Override - public void deleteContainer(ContainerId cid) { - if (isNull(cid)) { - throw new ValidationFrostFSException(String.format(PARAM_IS_MISSING_TEMPLATE, ContainerId.class.getName())); - } + public void deleteContainer(PrmContainerDelete args, CallContext ctx) { + validate(args); - var grpcContainerId = ContainerIdMapper.toGrpcMessage(cid); - var request = createDeleteRequest(grpcContainerId, null); + var request = createDeleteRequest(args, ctx); - var response = serviceBlockingStub.delete(request); + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.delete(request); Verifier.checkResponse(response); - waitForContainer(WaitExpects.REMOVED, request.getBody().getContainerId(), null); + waitForContainer(WaitExpects.REMOVED, request.getBody().getContainerId(), args.getWaitParams()); } - private void waitForContainer(WaitExpects expect, Types.ContainerID id, WaitParameters waitParams) { - var request = createGetRequest(id, null); + private void waitForContainer(WaitExpects expect, Types.ContainerID cid, PrmWait waitParams) { + var request = createGetRequest(cid, null); - waitParams = isNull(waitParams) ? new WaitParameters() : waitParams; + waitParams = isNull(waitParams) ? new PrmWait() : waitParams; var deadLine = waitParams.getDeadline(); while (true) { @@ -145,9 +153,12 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli } } + private Service.GetRequest createGetRequest(PrmContainerGet args) { + var cid = ContainerIdMapper.toGrpcMessage(args.getContainerId()); + return createGetRequest(cid, args.getXHeaders()); + } - private Service.GetRequest createGetRequest(Types.ContainerID cid, - Map xHeaders) { + private Service.GetRequest createGetRequest(Types.ContainerID cid, Map xHeaders) { var body = Service.GetRequest.Body.newBuilder() .setContainerId(cid) .build(); @@ -160,26 +171,30 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli return request.build(); } - private Service.ListRequest createListRequest(Map xHeaders) { + private Service.ListRequest createListRequest(PrmContainerGetAll args) { var body = Service.ListRequest.Body.newBuilder() .setOwnerId(OwnerIdMapper.toGrpcMessage(getContext().getOwnerId())) .build(); var request = Service.ListRequest.newBuilder() .setBody(body); - RequestConstructor.addMetaHeader(request, xHeaders); + RequestConstructor.addMetaHeader(request, args.getXHeaders()); RequestSigner.sign(request, getContext().getKey()); return request.build(); } - private Service.PutRequest createPutRequest(frostfs.container.Types.Container container, - Map xHeaders) { - container = container.toBuilder() - .setOwnerId(OwnerIdMapper.toGrpcMessage(getContext().getOwnerId())) - .setVersion(VersionMapper.toGrpcMessage(getContext().getVersion())) - .build(); + private Service.PutRequest createPutRequest(PrmContainerCreate args, CallContext ctx) { + syncContainerWithNetwork(args.getContainer(), ctx); + var builder = ContainerMapper.toGrpcMessage(args.getContainer()); + if (!builder.hasOwnerId()) { + builder.setOwnerId(OwnerIdMapper.toGrpcMessage(getContext().getOwnerId())); + } + if (!builder.hasVersion()) { + builder.setVersion(VersionMapper.toGrpcMessage(getContext().getVersion())); + } + var container = builder.build(); var body = Service.PutRequest.Body.newBuilder() .setContainer(container) .setSignature(RequestSigner.signRFC6979(getContext().getKey(), container)) @@ -187,9 +202,8 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli var request = Service.PutRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createContainerTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createContainerTokenContext( sessionToken, null, frostfs.session.Types.ContainerSessionContext.Verb.PUT, @@ -197,14 +211,15 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli getContext().getKey() ); - RequestConstructor.addMetaHeader(request, xHeaders, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); RequestSigner.sign(request, getContext().getKey()); return request.build(); } - private Service.DeleteRequest createDeleteRequest(Types.ContainerID cid, - Map xHeaders) { + private Service.DeleteRequest createDeleteRequest(PrmContainerDelete args, CallContext ctx) { + var cid = ContainerIdMapper.toGrpcMessage(args.getContainerId()); + var body = Service.DeleteRequest.Body.newBuilder() .setContainerId(cid) .setSignature(RequestSigner.signRFC6979(getContext().getKey(), cid.getValue())) @@ -212,9 +227,8 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli var request = Service.DeleteRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createContainerTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createContainerTokenContext( sessionToken, null, frostfs.session.Types.ContainerSessionContext.Verb.DELETE, @@ -222,9 +236,18 @@ public class ContainerClientImpl extends ContextAccessor implements ContainerCli getContext().getKey() ); - RequestConstructor.addMetaHeader(request, xHeaders, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); RequestSigner.sign(request, getContext().getKey()); return request.build(); } + + private void syncContainerWithNetwork(Container container, CallContext callContext) { + var settings = getContext().getFrostFSClient().getNetworkSettings(callContext); + if (nonNull(settings.getHomomorphicHashingDisabled()) && settings.getHomomorphicHashingDisabled()) { + container.getAttributes().put(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, Boolean.TRUE.toString()); + } else { + container.getAttributes().remove(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, Boolean.TRUE.toString()); + } + } } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/NetmapClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/NetmapClientImpl.java index 04109ad..a32abac 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/NetmapClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/NetmapClientImpl.java @@ -7,6 +7,7 @@ import info.frostfs.sdk.dto.netmap.NetmapSnapshot; import info.frostfs.sdk.dto.netmap.NodeInfo; import info.frostfs.sdk.jdo.ClientEnvironment; import info.frostfs.sdk.jdo.NetworkSettings; +import info.frostfs.sdk.jdo.parameters.CallContext; import info.frostfs.sdk.mappers.netmap.NetmapSnapshotMapper; import info.frostfs.sdk.mappers.netmap.NodeInfoMapper; import info.frostfs.sdk.services.ContextAccessor; @@ -17,6 +18,7 @@ import info.frostfs.sdk.tools.Verifier; import java.nio.charset.StandardCharsets; import static info.frostfs.sdk.tools.RequestSigner.sign; +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; import static java.util.Objects.nonNull; public class NetmapClientImpl extends ContextAccessor implements NetmapClient { @@ -94,12 +96,12 @@ public class NetmapClientImpl extends ContextAccessor implements NetmapClient { } @Override - public NetworkSettings getNetworkSettings() { + public NetworkSettings getNetworkSettings(CallContext ctx) { if (nonNull(getContext().getNetworkSettings())) { return getContext().getNetworkSettings(); } - var info = getNetworkInfo(); + var info = getNetworkInfo(ctx); var settings = new NetworkSettings(); @@ -113,25 +115,28 @@ public class NetmapClientImpl extends ContextAccessor implements NetmapClient { } @Override - public NodeInfo getLocalNodeInfo() { + public NodeInfo getLocalNodeInfo(CallContext ctx) { var request = Service.LocalNodeInfoRequest.newBuilder(); RequestConstructor.addMetaHeader(request); sign(request, getContext().getKey()); - var response = netmapServiceClient.localNodeInfo(request.build()); + var service = deadLineAfter(netmapServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.localNodeInfo(request.build()); + Verifier.checkResponse(response); return NodeInfoMapper.toModel(response.getBody()); } - public Service.NetworkInfoResponse getNetworkInfo() { + public Service.NetworkInfoResponse getNetworkInfo(CallContext ctx) { var request = Service.NetworkInfoRequest.newBuilder(); RequestConstructor.addMetaHeader(request); sign(request, getContext().getKey()); - var response = netmapServiceClient.networkInfo(request.build()); + var service = deadLineAfter(netmapServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.networkInfo(request.build()); Verifier.checkResponse(response); @@ -139,13 +144,14 @@ public class NetmapClientImpl extends ContextAccessor implements NetmapClient { } @Override - public NetmapSnapshot getNetmapSnapshot() { + public NetmapSnapshot getNetmapSnapshot(CallContext ctx) { var request = Service.NetmapSnapshotRequest.newBuilder(); RequestConstructor.addMetaHeader(request); sign(request, getContext().getKey()); - var response = netmapServiceClient.netmapSnapshot(request.build()); + var service = deadLineAfter(netmapServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.netmapSnapshot(request.build()); Verifier.checkResponse(response); diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java index f85936c..71741e8 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectClientImpl.java @@ -6,29 +6,28 @@ import frostfs.object.ObjectServiceGrpc; import frostfs.object.Service; import frostfs.refs.Types; import info.frostfs.sdk.constants.AppConst; -import info.frostfs.sdk.dto.container.ContainerId; import info.frostfs.sdk.dto.object.*; import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.enums.ObjectType; import info.frostfs.sdk.exceptions.ProcessFrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; -import info.frostfs.sdk.jdo.PutObjectParameters; import info.frostfs.sdk.jdo.PutObjectResult; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.object.*; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmObjectPatch; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeGet; +import info.frostfs.sdk.jdo.parameters.object.patch.PrmRangeHashGet; +import info.frostfs.sdk.jdo.parameters.session.SessionContext; +import info.frostfs.sdk.jdo.result.ObjectHeaderResult; import info.frostfs.sdk.mappers.container.ContainerIdMapper; -import info.frostfs.sdk.mappers.object.ObjectFilterMapper; -import info.frostfs.sdk.mappers.object.ObjectFrostFSMapper; -import info.frostfs.sdk.mappers.object.ObjectHeaderMapper; -import info.frostfs.sdk.mappers.object.ObjectIdMapper; -import info.frostfs.sdk.mappers.session.SessionMapper; +import info.frostfs.sdk.mappers.object.*; +import info.frostfs.sdk.mappers.object.patch.AddressMapper; +import info.frostfs.sdk.mappers.object.patch.RangeMapper; import info.frostfs.sdk.services.ContextAccessor; import info.frostfs.sdk.services.ObjectClient; -import info.frostfs.sdk.services.impl.rwhelper.ObjectReaderImpl; -import info.frostfs.sdk.services.impl.rwhelper.ObjectWriter; -import info.frostfs.sdk.services.impl.rwhelper.SearchReader; +import info.frostfs.sdk.services.impl.rwhelper.*; import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.Verifier; -import info.frostfs.sdk.utils.Validator; import org.apache.commons.collections4.CollectionUtils; import java.io.IOException; @@ -36,10 +35,12 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import static info.frostfs.sdk.Helper.getSha256; -import static info.frostfs.sdk.constants.ErrorConst.*; +import static info.frostfs.sdk.constants.ErrorConst.PROTO_MESSAGE_IS_EMPTY_TEMPLATE; import static info.frostfs.sdk.tools.RequestSigner.sign; +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; +import static info.frostfs.sdk.utils.Validator.validate; import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; public class ObjectClientImpl extends ContextAccessor implements ObjectClient { private final ObjectServiceGrpc.ObjectServiceBlockingStub objectServiceBlockingClient; @@ -55,87 +56,234 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { this.sessionTools = new SessionToolsImpl(clientEnvironment); } - public frostfs.session.Types.SessionToken getOrCreateSession(SessionToken sessionToken) { - return sessionTools.getOrCreateSession(sessionToken, getContext()); + public SessionToken getOrCreateSession(SessionContext sessionContext, CallContext ctx) { + return isNull(sessionContext.getSessionToken()) + ? sessionTools.getOrCreateSession(getContext(), ctx) + : sessionContext.getSessionToken(); } @Override - public ObjectHeader getObjectHead(ContainerId cid, ObjectId oid) { - if (isNull(cid) || isNull(oid)) { - throw new ValidationFrostFSException( - String.format( - PARAMS_ARE_MISSING_TEMPLATE, - String.join(FIELDS_DELIMITER_COMMA, ContainerId.class.getName(), ObjectId.class.getName()) - ) - ); - } + public ObjectHeaderResult getObjectHead(PrmObjectHeadGet args, CallContext ctx) { + validate(args); - var request = createHeadRequest(ContainerIdMapper.toGrpcMessage(cid), ObjectIdMapper.toGrpcMessage(oid)); + var request = createHeadRequest(args, ctx); - var response = objectServiceBlockingClient.head(request); + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.head(request); Verifier.checkResponse(response); - return ObjectHeaderMapper.toModel(response.getBody().getHeader().getHeader()); + return ObjectHeaderResult.builder() + .headerInfo(ObjectHeaderMapper.toModel(response.getBody().getHeader().getHeader())) + .splitInfo(SplitInfoMapper.toModel(response.getBody().getSplitInfo())) + .build(); } @Override - public ObjectFrostFS getObject(ContainerId cid, ObjectId oid) { - var request = createGetRequest(ContainerIdMapper.toGrpcMessage(cid), ObjectIdMapper.toGrpcMessage(oid)); + public ObjectFrostFS getObject(PrmObjectGet args, CallContext ctx) { + validate(args); - return getObject(request); + var request = createGetRequest(args, ctx); + + return getObject(request, ctx); } @Override - public void deleteObject(ContainerId cid, ObjectId oid) { - var request = createDeleteRequest(ContainerIdMapper.toGrpcMessage(cid), ObjectIdMapper.toGrpcMessage(oid)); + public void deleteObject(PrmObjectDelete args, CallContext ctx) { + validate(args); - var response = objectServiceBlockingClient.delete(request); + var request = createDeleteRequest(args, ctx); + + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.delete(request); Verifier.checkResponse(response); } @Override - public Iterable searchObjects(ContainerId cid, ObjectFilter... filters) { - var request = createSearchRequest(ContainerIdMapper.toGrpcMessage(cid), filters); + public Iterable searchObjects(PrmObjectSearch args, CallContext ctx) { + validate(args); - var objectsIds = searchObjects(request); + var request = createSearchRequest(args, ctx); + + var objectsIds = searchObjects(request, ctx); return Iterables.transform(objectsIds, input -> new ObjectId(input.getValue().toByteArray())); } @Override - public ObjectId putObject(PutObjectParameters parameters) { - Validator.validate(parameters); + public ObjectWriter putObject(PrmObjectPut args, CallContext ctx) { + validate(args); - if (parameters.isClientCut()) { - return putClientCutObject(parameters); - } - - if (parameters.getHeader().getPayloadLength() > 0) { - parameters.setFullLength(parameters.getHeader().getPayloadLength()); - } else { - parameters.setFullLength(getStreamSize(parameters.getPayload())); - } - - return putStreamObject(parameters).getObjectId(); + return new ObjectWriter(getContext(), args, getUploadStream(args, ctx)); } @Override - public ObjectId putSingleObject(ObjectFrostFS modelObject) { - var grpcObject = objectToolsImpl.createObject(modelObject); + public ObjectId putClientCutObject(PrmObjectClientCutPut args, CallContext ctx) { + validate(args); - var request = createPutSingleRequest(grpcObject); + var header = args.getObjectHeader(); + var fullLength = header.getPayloadLength() == 0 ? getStreamSize(args.getPayload()) : header.getPayloadLength(); + args.getPutObjectContext().setFullLength(fullLength); - var response = objectServiceBlockingClient.putSingle(request); + if (args.getPutObjectContext().getMaxObjectSizeCache() == 0) { + var networkSettings = getContext().getFrostFSClient().getNetworkSettings(ctx); + args.getPutObjectContext().setMaxObjectSizeCache(networkSettings.getMaxObjectSize().intValue()); + } + + var restBytes = fullLength - args.getPutObjectContext().getCurrentStreamPosition(); + var objectSize = restBytes > 0 + ? Math.min(args.getPutObjectContext().getMaxObjectSizeCache(), restBytes) + : args.getPutObjectContext().getMaxObjectSizeCache(); + + //define collection capacity + var restPart = (restBytes % objectSize) > 0 ? 1 : 0; + var objectsCount = fullLength > 0 ? (int) (restBytes / objectSize) + restPart : 0; + + List sentObjectIds = new ArrayList<>(objectsCount); + + // keep attributes for the large object + var attributes = args.getObjectHeader().getAttributes(); + + Split split = new Split(); + args.getObjectHeader().setAttributes(new ArrayList<>()); + + // send all parts except the last one as separate Objects + while (restBytes > (long) args.getPutObjectContext().getMaxObjectSizeCache()) { + var previous = CollectionUtils.isNotEmpty(sentObjectIds) + ? sentObjectIds.get(sentObjectIds.size() - 1) + : null; + split.setPrevious(previous); + args.getObjectHeader().setSplit(split); + + var result = putMultipartStreamObject(args, ctx); + + sentObjectIds.add(result.getObjectId()); + + restBytes -= result.getObjectSize(); + } + + // send the last part and create linkObject + if (CollectionUtils.isNotEmpty(sentObjectIds)) { + var largeObjectHeader = new ObjectHeader( + header.getContainerId(), ObjectType.REGULAR, attributes, fullLength, header.getVersion() + ); + largeObjectHeader.setOwnerId(header.getOwnerId()); + + split.setParentHeader(largeObjectHeader); + + var result = putMultipartStreamObject(args, ctx); + + sentObjectIds.add(result.getObjectId()); + + var linkObject = new LinkObject(header.getContainerId(), split.getSplitId(), largeObjectHeader); + linkObject.addChildren(sentObjectIds); + + putSingleObject(new PrmObjectSinglePut(linkObject), ctx); + + return split.getParent(); + } + + // We are here if the payload is placed to one Object. It means no cut action, just simple PUT. + var singlePartResult = putMultipartStreamObject(args, ctx); + + return singlePartResult.getObjectId(); + } + + @Override + public ObjectId putSingleObject(PrmObjectSinglePut args, CallContext ctx) { + var grpcObject = objectToolsImpl.createObject(args.getObjectFrostFS()); + var request = createPutSingleRequest(grpcObject, args, ctx); + + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.putSingle(request); Verifier.checkResponse(response); return new ObjectId(grpcObject.getObjectId().getValue().toByteArray()); } - private ObjectFrostFS getObject(Service.GetRequest request) { - var reader = getObjectInit(request); + @Override + public RangeReader getRange(PrmRangeGet args, CallContext ctx) { + validate(args); + + var request = createGetRangeRequest(args, ctx); + + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + return new RangeReader(service.getRange(request)); + } + + @Override + public byte[][] getRangeHash(PrmRangeHashGet args, CallContext ctx) { + validate(args); + + var request = createGetRangeHashRequest(args, ctx); + + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.getRangeHash(request); + + Verifier.checkResponse(response); + + return response.getBody().getHashListList().stream().map(ByteString::toByteArray).toArray(byte[][]::new); + } + + @Override + public ObjectId patchObject(PrmObjectPatch args, CallContext ctx) { + validate(args); + + var service = deadLineAfter(objectServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + PatchStreamer writer = new PatchStreamer(service); + + var request = createInitPatchRequest(args, ctx); + writer.write(request.build()); + + if (nonNull(args.getPayload())) { + patchObjectPayload(request, args, writer); + } + + var response = writer.complete(); + + Verifier.checkResponse(response); + + return ObjectIdMapper.toModel(response.getBody().getObjectId()); + } + + private void patchObjectPayload(Service.PatchRequest.Builder request, PrmObjectPatch args, PatchStreamer writer) { + var currentPos = args.getRange().getOffset(); + + var chunkSize = args.getMaxChunkLength() > 0 ? args.getMaxChunkLength() : AppConst.OBJECT_CHUNK_SIZE; + byte[] chunkBuffer = new byte[chunkSize]; + + var bytesCount = readNBytes(args.getPayload(), chunkBuffer, chunkSize); + while (bytesCount > 0) { + var range = Service.Range.newBuilder() + .setOffset(currentPos) + .setLength(bytesCount) + .build(); + + var patch = Service.PatchRequest.Body.Patch.newBuilder() + .setChunk(ByteString.copyFrom(chunkBuffer, 0, bytesCount)) + .setSourceRange(range) + .build(); + + var body = Service.PatchRequest.Body.newBuilder() + .setAddress(request.getBody().getAddress()) + .setPatch(patch) + .build(); + request.setBody(body); + + RequestConstructor.addMetaHeader(request, args.getXHeaders(), request.getMetaHeader().getSessionToken()); + sign(request, getContext().getKey()); + + writer.write(request.build()); + + currentPos += bytesCount; + bytesCount = readNBytes(args.getPayload(), chunkBuffer, chunkSize); + } + } + + private ObjectFrostFS getObject(Service.GetRequest request, CallContext ctx) { + var reader = getObjectInit(request, ctx); var grpcObject = reader.readHeader(); var modelObject = ObjectFrostFSMapper.toModel(grpcObject); @@ -145,39 +293,41 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { return modelObject; } - private ObjectReaderImpl getObjectInit(Service.GetRequest initRequest) { + private ObjectReaderImpl getObjectInit(Service.GetRequest initRequest, CallContext ctx) { if (initRequest.getSerializedSize() == 0) { throw new ProcessFrostFSException( String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, initRequest.getClass().getName()) ); } - return new ObjectReaderImpl(objectServiceBlockingClient.get(initRequest)); + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + return new ObjectReaderImpl(service.get(initRequest)); } - private PutObjectResult putStreamObject(PutObjectParameters parameters) { - var chunkSize = parameters.getBufferMaxSize() > 0 ? parameters.getBufferMaxSize() : AppConst.OBJECT_CHUNK_SIZE; + private PutObjectResult putMultipartStreamObject(PrmObjectClientCutPut args, CallContext ctx) { + var chunkSize = args.getBufferMaxSize() > 0 ? args.getBufferMaxSize() : AppConst.OBJECT_CHUNK_SIZE; - var restBytes = parameters.getFullLength() - parameters.getCurrentStreamPosition(); + var restBytes = + args.getPutObjectContext().getFullLength() - args.getPutObjectContext().getCurrentStreamPosition(); chunkSize = (int) Math.min(restBytes, chunkSize); - byte[] chunkBuffer = parameters.getCustomerBuffer() != null - ? parameters.getCustomerBuffer() + byte[] chunkBuffer = args.getCustomerBuffer() != null + ? args.getCustomerBuffer() : new byte[chunkSize];//todo change to pool var sentBytes = 0; // 0 means no limit from client, so server side cut is performed - var objectLimitSize = parameters.isClientCut() ? parameters.getMaxObjectSizeCache() : 0; + var objectLimitSize = args.getPutObjectContext().getMaxObjectSizeCache(); - var stream = getUploadStream(parameters); + var stream = getUploadStream(args, ctx); while (objectLimitSize == 0 || sentBytes < objectLimitSize) { // send chunks limited to default or user's settings var bufferSize = objectLimitSize > 0 ? Math.min(objectLimitSize - sentBytes, chunkSize) : chunkSize; - var bytesCount = readNBytes(parameters.getPayload(), chunkBuffer, bufferSize); + var bytesCount = readNBytes(args.getPayload(), chunkBuffer, bufferSize); if (bytesCount == 0) { break; @@ -192,6 +342,7 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { .setBody(body) .clearVerifyHeader(); + RequestConstructor.addMetaHeader(chunkRequest, args.getXHeaders()); sign(chunkRequest, getContext().getKey()); stream.write(chunkRequest.build()); @@ -204,110 +355,38 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { return new PutObjectResult(objectId, sentBytes); } - private ObjectId putClientCutObject(PutObjectParameters parameters) { - var header = parameters.getHeader(); - var tokenRaw = getOrCreateSession(parameters.getSessionToken()); - var token = new SessionToken(SessionMapper.serialize(tokenRaw)); - parameters.setSessionToken(token); - - var fullLength = header.getPayloadLength() == 0 - ? getStreamSize(parameters.getPayload()) - : header.getPayloadLength(); - - parameters.setFullLength(fullLength); - - if (parameters.getMaxObjectSizeCache() == 0) { - var networkSettings = getContext().getFrostFSClient().getNetworkSettings(); - parameters.setMaxObjectSizeCache(networkSettings.getMaxObjectSize().intValue()); - } - - var restBytes = fullLength - parameters.getCurrentStreamPosition(); - var objectSize = restBytes > 0 - ? Math.min(parameters.getMaxObjectSizeCache(), restBytes) - : parameters.getMaxObjectSizeCache(); - - //define collection capacity - var restPart = (restBytes % objectSize) > 0 ? 1 : 0; - var objectsCount = fullLength > 0 ? (int) (restBytes / objectSize) + restPart : 0; - - List sentObjectIds = new ArrayList<>(objectsCount); - - // keep attributes for the large object - var attributes = parameters.getHeader().getAttributes(); - - Split split = new Split(); - parameters.getHeader().setSplit(split); - parameters.getHeader().setAttributes(new ArrayList<>()); - - // send all parts except the last one as separate Objects - while (restBytes > (long) parameters.getMaxObjectSizeCache()) { - var previous = CollectionUtils.isNotEmpty(sentObjectIds) - ? sentObjectIds.get(sentObjectIds.size() - 1) - : null; - split.setPrevious(previous); - - var result = putStreamObject(parameters); - - sentObjectIds.add(result.getObjectId()); - - restBytes -= result.getObjectSize(); - } - - // send the last part and create linkObject - if (CollectionUtils.isNotEmpty(sentObjectIds)) { - var largeObjectHeader = - new ObjectHeader(header.getContainerId(), ObjectType.REGULAR, attributes, fullLength, null); - - split.setParentHeader(largeObjectHeader); - - var result = putStreamObject(parameters); - - sentObjectIds.add(result.getObjectId()); - - var linkObject = new LinkObject(header.getContainerId(), split.getSplitId(), largeObjectHeader); - linkObject.addChildren(sentObjectIds); - - putSingleObject(linkObject); - - return split.getParent(); - } - - // We are here if the payload is placed to one Object. It means no cut action, just simple PUT. - var singlePartResult = putStreamObject(parameters); - - return singlePartResult.getObjectId(); - } - - private ObjectWriter getUploadStream(PutObjectParameters parameters) { - var header = parameters.getHeader(); + private ObjectStreamer getUploadStream(PrmObjectPutBase args, CallContext ctx) { + var header = args.getObjectHeader(); header.setOwnerId(getContext().getOwnerId()); header.setVersion(getContext().getVersion()); var grpcHeader = ObjectHeaderMapper.toGrpcMessage(header); - grpcHeader = objectToolsImpl.updateSplitValues(grpcHeader, header.getSplit()); - var oid = Types.ObjectID.newBuilder().setValue(getSha256(grpcHeader)).build(); + if (nonNull(header.getSplit())) { + grpcHeader = objectToolsImpl.updateSplitValues(grpcHeader, header.getSplit()); + } - var initRequest = createInitPutRequest(oid, grpcHeader); + var initRequest = createInitPutRequest(grpcHeader, args, ctx); - return putObjectInit(initRequest); + return putObjectInit(initRequest, ctx); } - private ObjectWriter putObjectInit(Service.PutRequest initRequest) { + private ObjectStreamer putObjectInit(Service.PutRequest initRequest, CallContext ctx) { if (initRequest.getSerializedSize() == 0) { throw new ProcessFrostFSException( String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, initRequest.getClass().getName()) ); } - ObjectWriter writer = new ObjectWriter(objectServiceClient); + var service = deadLineAfter(objectServiceClient, ctx.getTimeout(), ctx.getTimeUnit()); + ObjectStreamer writer = new ObjectStreamer(service); writer.write(initRequest); return writer; } - private Iterable searchObjects(Service.SearchRequest request) { - var reader = getSearchReader(request); + private Iterable searchObjects(Service.SearchRequest request, CallContext ctx) { + var reader = getSearchReader(request, ctx); var ids = reader.read(); List result = new ArrayList<>(); @@ -319,14 +398,15 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { return result;//todo return yield } - private SearchReader getSearchReader(Service.SearchRequest initRequest) { + private SearchReader getSearchReader(Service.SearchRequest initRequest, CallContext ctx) { if (initRequest.getSerializedSize() == 0) { throw new ProcessFrostFSException( String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, initRequest.getClass().getName()) ); } - return new SearchReader(objectServiceBlockingClient.search(initRequest)); + var service = deadLineAfter(objectServiceBlockingClient, ctx.getTimeout(), ctx.getTimeUnit()); + return new SearchReader(service.search(initRequest)); } private int readNBytes(InputStream inputStream, byte[] buffer, int size) { @@ -345,36 +425,36 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { } } - private Service.HeadRequest createHeadRequest(Types.ContainerID cid, Types.ObjectID oid) { + private Service.HeadRequest createHeadRequest(PrmObjectHeadGet args, CallContext ctx) { var address = Types.Address.newBuilder() - .setContainerId(cid) - .setObjectId(oid) + .setContainerId(ContainerIdMapper.toGrpcMessage(args.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(args.getObjectId())) .build(); var body = Service.HeadRequest.Body.newBuilder() .setAddress(address) + .setRaw(args.isRaw()) .build(); var request = Service.HeadRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.HEAD, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } - private Service.GetRequest createGetRequest(Types.ContainerID cid, Types.ObjectID oid) { + private Service.GetRequest createGetRequest(PrmObjectGet args, CallContext ctx) { var address = Types.Address.newBuilder() - .setContainerId(cid) - .setObjectId(oid) + .setContainerId(ContainerIdMapper.toGrpcMessage(args.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(args.getObjectId())) .build(); var body = Service.GetRequest.Body.newBuilder() .setAddress(address) @@ -382,25 +462,24 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { var request = Service.GetRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.GET, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } - private Service.DeleteRequest createDeleteRequest(Types.ContainerID cid, Types.ObjectID oid) { + private Service.DeleteRequest createDeleteRequest(PrmObjectDelete args, CallContext ctx) { var address = Types.Address.newBuilder() - .setContainerId(cid) - .setObjectId(oid) + .setContainerId(ContainerIdMapper.toGrpcMessage(args.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(args.getObjectId())) .build(); var body = Service.DeleteRequest.Body.newBuilder() .setAddress(address) @@ -408,22 +487,23 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { var request = Service.DeleteRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.DELETE, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } - private Service.SearchRequest createSearchRequest(Types.ContainerID cid, ObjectFilter... filters) { + private Service.SearchRequest createSearchRequest(PrmObjectSearch args, CallContext ctx) { + var cid = ContainerIdMapper.toGrpcMessage(args.getContainerId()); + var address = Types.Address.newBuilder() .setContainerId(cid) .build(); @@ -432,32 +512,32 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { .setContainerId(cid) .setVersion(1);// TODO: clarify this param - for (ObjectFilter filter : filters) { + for (ObjectFilter filter : args.getFilters()) { body.addFilters(ObjectFilterMapper.toGrpcMessage(filter)); } var request = Service.SearchRequest.newBuilder() .setBody(body.build()); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.SEARCH, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } - private Service.PutRequest createInitPutRequest(Types.ObjectID oid, frostfs.object.Types.Header header) { + private Service.PutRequest createInitPutRequest(frostfs.object.Types.Header header, + PrmObjectPutBase args, + CallContext ctx) { var address = Types.Address.newBuilder() .setContainerId(header.getContainerId()) - .setObjectId(oid) .build(); var init = Service.PutRequest.Body.Init.newBuilder() .setHeader(header) @@ -468,25 +548,25 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { var request = Service.PutRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.PUT, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } - private Service.PutSingleRequest createPutSingleRequest(frostfs.object.Types.Object grpcObject) { + private Service.PutSingleRequest createPutSingleRequest(frostfs.object.Types.Object grpcObject, + PrmObjectSinglePut args, + CallContext ctx) { var address = Types.Address.newBuilder() .setContainerId(grpcObject.getHeader().getContainerId()) - .setObjectId(grpcObject.getObjectId()) .build(); var body = Service.PutSingleRequest.Body.newBuilder() .setObject(grpcObject) @@ -494,18 +574,95 @@ public class ObjectClientImpl extends ContextAccessor implements ObjectClient { var request = Service.PutSingleRequest.newBuilder() .setBody(body); - var sessionToken = getOrCreateSession(null); - - sessionToken = RequestConstructor.createObjectTokenContext( + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( sessionToken, address, frostfs.session.Types.ObjectSessionContext.Verb.PUT, getContext().getKey() ); - RequestConstructor.addMetaHeader(request, null, sessionToken); + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); sign(request, getContext().getKey()); return request.build(); } + + private Service.GetRangeRequest createGetRangeRequest(PrmRangeGet args, CallContext ctx) { + var address = Types.Address.newBuilder() + .setContainerId(ContainerIdMapper.toGrpcMessage(args.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(args.getObjectId())) + .build(); + var body = Service.GetRangeRequest.Body.newBuilder() + .setAddress(address) + .setRange(RangeMapper.toGrpcMessage(args.getRange())) + .setRaw(args.isRaw()) + .build(); + var request = Service.GetRangeRequest.newBuilder() + .setBody(body); + + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( + sessionToken, + address, + frostfs.session.Types.ObjectSessionContext.Verb.RANGE, + getContext().getKey() + ); + + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); + sign(request, getContext().getKey()); + + return request.build(); + } + + private Service.GetRangeHashRequest createGetRangeHashRequest(PrmRangeHashGet args, CallContext ctx) { + var address = Types.Address.newBuilder() + .setContainerId(ContainerIdMapper.toGrpcMessage(args.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(args.getObjectId())) + .build(); + var body = Service.GetRangeHashRequest.Body.newBuilder() + .setAddress(address) + .setType(Types.ChecksumType.SHA256) + .setSalt(ByteString.copyFrom(args.getSalt())) + .addAllRanges(RangeMapper.toGrpcMessages(args.getRanges())) + .build(); + var request = Service.GetRangeHashRequest.newBuilder() + .setBody(body); + + var sessionToken = getOrCreateSession(args, ctx); + var protoToken = RequestConstructor.createObjectTokenContext( + sessionToken, + address, + frostfs.session.Types.ObjectSessionContext.Verb.RANGEHASH, + getContext().getKey() + ); + + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); + sign(request, getContext().getKey()); + + return request.build(); + } + + private Service.PatchRequest.Builder createInitPatchRequest(PrmObjectPatch args, CallContext ctx) { + var address = AddressMapper.toGrpcMessage(args.getAddress()); + var body = Service.PatchRequest.Body.newBuilder() + .setAddress(address) + .setReplaceAttributes(args.isReplaceAttributes()) + .addAllNewAttributes(ObjectAttributeMapper.toGrpcMessages(args.getNewAttributes())) + .build(); + var request = Service.PatchRequest.newBuilder() + .setBody(body); + + var protoToken = RequestConstructor.createObjectTokenContext( + getOrCreateSession(args, ctx), + request.getBody().getAddress(), + frostfs.session.Types.ObjectSessionContext.Verb.PATCH, + getContext().getKey() + ); + + RequestConstructor.addMetaHeader(request, args.getXHeaders(), protoToken); + sign(request, getContext().getKey()); + + return request; + } } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectToolsImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectToolsImpl.java index 9730f34..f0f4e1c 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/ObjectToolsImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/ObjectToolsImpl.java @@ -47,16 +47,16 @@ public class ObjectToolsImpl extends ContextAccessor implements ToolsClient { ); } - public Types.Object createObject(ObjectFrostFS objectFrostFs) { - objectFrostFs.getHeader().setOwnerId(getContext().getOwnerId()); - objectFrostFs.getHeader().setVersion(getContext().getVersion()); - objectFrostFs.getHeader().setPayloadLength(objectFrostFs.getPayload().length); + public Types.Object createObject(ObjectFrostFS objectFrostFS) { + objectFrostFS.getHeader().setOwnerId(getContext().getOwnerId()); + objectFrostFS.getHeader().setVersion(getContext().getVersion()); + objectFrostFS.getHeader().setPayloadLength(objectFrostFS.getPayload().length); - var grpcHeader = ObjectHeaderMapper.toGrpcMessage(objectFrostFs.getHeader()).toBuilder() - .setPayloadHash(sha256Checksum(objectFrostFs.getPayload())) + var grpcHeader = ObjectHeaderMapper.toGrpcMessage(objectFrostFS.getHeader()).toBuilder() + .setPayloadHash(sha256Checksum(objectFrostFS.getPayload())) .build(); - var split = objectFrostFs.getHeader().getSplit(); + var split = objectFrostFS.getHeader().getSplit(); if (nonNull(split)) { grpcHeader = updateSplitValues(grpcHeader, split); } @@ -68,7 +68,7 @@ public class ObjectToolsImpl extends ContextAccessor implements ToolsClient { return Types.Object.newBuilder() .setHeader(grpcHeader) .setObjectId(objectId) - .setPayload(ByteString.copyFrom(objectFrostFs.getPayload())) + .setPayload(ByteString.copyFrom(objectFrostFS.getPayload())) .setSignature(sig) .build(); } @@ -105,7 +105,10 @@ public class ObjectToolsImpl extends ContextAccessor implements ToolsClient { split.setParent(ObjectIdMapper.toModel(parentObjectId)); } - grpcSplit.setPrevious(ObjectIdMapper.toGrpcMessage(split.getPrevious())).build(); + if (nonNull(split.getPrevious())) { + grpcSplit.setPrevious(ObjectIdMapper.toGrpcMessage(split.getPrevious())).build(); + } + return grpcHeader.toBuilder().setSplit(grpcSplit.build()).build(); } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/SessionClientImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/SessionClientImpl.java index 8151fc9..7b999ce 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/SessionClientImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/SessionClientImpl.java @@ -5,6 +5,8 @@ import frostfs.session.SessionServiceGrpc; import frostfs.session.Types; import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.session.PrmSessionCreate; import info.frostfs.sdk.mappers.object.OwnerIdMapper; import info.frostfs.sdk.mappers.session.SessionMapper; import info.frostfs.sdk.services.ContextAccessor; @@ -13,6 +15,7 @@ import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.Verifier; import static info.frostfs.sdk.tools.RequestSigner.sign; +import static info.frostfs.sdk.utils.DeadLineUtil.deadLineAfter; public class SessionClientImpl extends ContextAccessor implements SessionClient { private final SessionServiceGrpc.SessionServiceBlockingStub serviceBlockingStub; @@ -23,16 +26,16 @@ public class SessionClientImpl extends ContextAccessor implements SessionClient } @Override - public SessionToken createSession(long expiration) { - var sessionToken = createSessionInternal(expiration); + public SessionToken createSession(PrmSessionCreate args, CallContext ctx) { + var sessionToken = createSessionInternal(args, ctx); var token = SessionMapper.serialize(sessionToken); return new SessionToken(token); } - public Types.SessionToken createSessionInternal(long expiration) { + public Types.SessionToken createSessionInternal(PrmSessionCreate args, CallContext ctx) { var body = Service.CreateRequest.Body.newBuilder() .setOwnerId(OwnerIdMapper.toGrpcMessage(getContext().getOwnerId())) - .setExpiration(expiration) + .setExpiration(args.getExpiration()) .build(); var request = Service.CreateRequest.newBuilder() .setBody(body); @@ -40,11 +43,12 @@ public class SessionClientImpl extends ContextAccessor implements SessionClient RequestConstructor.addMetaHeader(request); sign(request, getContext().getKey()); - return createSession(request.build()); + return createSession(request.build(), ctx); } - private Types.SessionToken createSession(Service.CreateRequest request) { - var response = serviceBlockingStub.create(request); + private Types.SessionToken createSession(Service.CreateRequest request, CallContext ctx) { + var service = deadLineAfter(serviceBlockingStub, ctx.getTimeout(), ctx.getTimeUnit()); + var response = service.create(request); Verifier.checkResponse(response); diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/SessionToolsImpl.java b/client/src/main/java/info/frostfs/sdk/services/impl/SessionToolsImpl.java index 1ee5cbd..c51b673 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/SessionToolsImpl.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/SessionToolsImpl.java @@ -1,12 +1,14 @@ package info.frostfs.sdk.services.impl; -import frostfs.session.Types; import info.frostfs.sdk.dto.session.SessionToken; +import info.frostfs.sdk.exceptions.FrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; -import info.frostfs.sdk.mappers.session.SessionMapper; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.session.PrmSessionCreate; import info.frostfs.sdk.services.ContextAccessor; import info.frostfs.sdk.services.SessionTools; +import static info.frostfs.sdk.constants.ErrorConst.SESSION_CREATE_FAILED; import static java.util.Objects.isNull; public class SessionToolsImpl extends ContextAccessor implements SessionTools { @@ -16,11 +18,18 @@ public class SessionToolsImpl extends ContextAccessor implements SessionTools { } @Override - public Types.SessionToken getOrCreateSession(SessionToken sessionToken, ClientEnvironment env) { - if (isNull(sessionToken)) { - return env.getFrostFSClient().createSessionInternal(-1); + public SessionToken getOrCreateSession(ClientEnvironment env, CallContext ctx) { + var token = env.getSessionCache().tryGetValue(env.getSessionKey()); + + if (isNull(token)) { + token = env.getFrostFSClient().createSession(new PrmSessionCreate(-1), ctx); + if (isNull(token)) { + throw new FrostFSException(SESSION_CREATE_FAILED); + } + + env.getSessionCache().setValue(env.getSessionKey(), token); } - return SessionMapper.deserializeSessionToken(sessionToken.getToken()); + return token; } } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/ClientMetrics.java b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/ClientMetrics.java index 149c145..656d19f 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/ClientMetrics.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/ClientMetrics.java @@ -15,10 +15,10 @@ import static info.frostfs.sdk.services.impl.interceptor.Labels.*; public class ClientMetrics { private static final List defaultRequestLabels = - Arrays.asList("grpc_type", "grpc_service", "grpc_method"); + Arrays.asList("grpc_target", "grpc_type", "grpc_service", "grpc_method"); private static final List defaultResponseLabels = - Arrays.asList("grpc_type", "grpc_service", "grpc_method", "code", "grpc_code"); + Arrays.asList("grpc_target", "grpc_type", "grpc_service", "grpc_method", "code", "grpc_code"); private static final Counter.Builder rpcStartedBuilder = Counter.build() @@ -113,7 +113,9 @@ public class ClientMetrics { .observe(latencySec); } - /** Knows how to produce {@link ClientMetrics} instances for individual methods. */ + /** + * Knows how to produce {@link ClientMetrics} instances for individual methods. + */ static class Factory { private final List> labelHeaderKeys; private final Counter rpcStarted; @@ -155,7 +157,9 @@ public class ClientMetrics { } } - /** Creates a {@link ClientMetrics} for the supplied gRPC method. */ + /** + * Creates a {@link ClientMetrics} for the supplied gRPC method. + */ ClientMetrics createMetricsForMethod(GrpcMethod grpcMethod) { return new ClientMetrics( labelHeaderKeys, diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Configuration.java b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Configuration.java index d4adcca..cfc9193 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Configuration.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Configuration.java @@ -2,18 +2,21 @@ package info.frostfs.sdk.services.impl.interceptor; import io.prometheus.client.CollectorRegistry; + import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; + public class Configuration { private static final double[] DEFAULT_LATENCY_BUCKETS = - new double[] {.001, .005, .01, .05, 0.075, .1, .25, .5, 1, 2, 5, 10}; + new double[]{.001, .005, .01, .05, 0.075, .1, .25, .5, 1, 2, 5, 10}; private final boolean isIncludeLatencyHistograms; private final CollectorRegistry collectorRegistry; private final double[] latencyBuckets; private final List labelHeaders; private final boolean isAddCodeLabelToHistograms; + private Configuration( boolean isIncludeLatencyHistograms, CollectorRegistry collectorRegistry, @@ -28,7 +31,9 @@ public class Configuration { } - /** Returns a {@link Configuration} for recording all cheap metrics about the rpcs. */ + /** + * Returns a {@link Configuration} for recording all cheap metrics about the rpcs. + */ public static Configuration cheapMetricsOnly() { return new Configuration( false /* isIncludeLatencyHistograms */, @@ -119,27 +124,37 @@ public class Configuration { true /* isAddCodeLabelToHistograms */); } - /** Returns whether or not latency histograms for calls should be included. */ + /** + * Returns whether or not latency histograms for calls should be included. + */ public boolean isIncludeLatencyHistograms() { return isIncludeLatencyHistograms; } - /** Returns the {@link CollectorRegistry} used to record stats. */ + /** + * Returns the {@link CollectorRegistry} used to record stats. + */ public CollectorRegistry getCollectorRegistry() { return collectorRegistry; } - /** Returns the histogram buckets to use for latency metrics. */ + /** + * Returns the histogram buckets to use for latency metrics. + */ public double[] getLatencyBuckets() { return latencyBuckets; } - /** Returns the configured list of headers to be used as labels. */ + /** + * Returns the configured list of headers to be used as labels. + */ public List getLabelHeaders() { return labelHeaders; } - /** Returns whether or not status code label should be added to latency histogram. */ + /** + * Returns whether or not status code label should be added to latency histogram. + */ public boolean isAddCodeLabelToHistograms() { return isAddCodeLabelToHistograms; } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/GrpcMethod.java b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/GrpcMethod.java index 8f9aa4c..41e4ef1 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/GrpcMethod.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/GrpcMethod.java @@ -1,27 +1,33 @@ package info.frostfs.sdk.services.impl.interceptor; +import io.grpc.Channel; import io.grpc.MethodDescriptor; import io.grpc.MethodDescriptor.MethodType; public class GrpcMethod { + private final String targetName; private final String serviceName; private final String methodName; private final MethodType type; - private GrpcMethod(String serviceName, String methodName, MethodType type) { + private GrpcMethod(String targetName, String serviceName, String methodName, MethodType type) { + this.targetName = targetName; this.serviceName = serviceName; this.methodName = methodName; this.type = type; } - static GrpcMethod of(MethodDescriptor method) { + static GrpcMethod of(MethodDescriptor method, Channel channel) { String serviceName = MethodDescriptor.extractFullServiceName(method.getFullMethodName()); // Full method names are of the form: "full.serviceName/MethodName". We extract the last part. String methodName = method.getFullMethodName().substring(serviceName.length() + 1); - return new GrpcMethod(serviceName, methodName, method.getType()); + return new GrpcMethod(channel.authority(), serviceName, methodName, method.getType()); } + String targetName() { + return targetName; + } String serviceName() { return serviceName; diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Labels.java b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Labels.java index ba44eaf..e728a79 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Labels.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/Labels.java @@ -53,6 +53,7 @@ public class Labels { */ static T addLabels(SimpleCollector collector, List labels, GrpcMethod method) { List allLabels = new ArrayList<>(); + allLabels.add(method.targetName()); allLabels.add(method.type()); allLabels.add(method.serviceName()); allLabels.add(method.methodName()); diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/MonitoringClientInterceptor.java b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/MonitoringClientInterceptor.java index ada4761..4654487 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/MonitoringClientInterceptor.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/interceptor/MonitoringClientInterceptor.java @@ -1,11 +1,8 @@ package info.frostfs.sdk.services.impl.interceptor; +import io.grpc.*; + import java.time.Clock; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; -import io.grpc.MethodDescriptor; public class MonitoringClientInterceptor implements ClientInterceptor { @@ -19,6 +16,7 @@ public class MonitoringClientInterceptor implements ClientInterceptor { this.configuration = configuration; this.clientMetricsFactory = clientMetricsFactory; } + public static MonitoringClientInterceptor create(Configuration configuration) { return new MonitoringClientInterceptor( Clock.systemDefaultZone(), configuration, new ClientMetrics.Factory(configuration)); @@ -27,7 +25,7 @@ public class MonitoringClientInterceptor implements ClientInterceptor { @Override public ClientCall interceptCall( MethodDescriptor methodDescriptor, CallOptions callOptions, Channel channel) { - GrpcMethod grpcMethod = GrpcMethod.of(methodDescriptor); + GrpcMethod grpcMethod = GrpcMethod.of(methodDescriptor, channel); ClientMetrics metrics = clientMetricsFactory.createMetricsForMethod(grpcMethod); return new MonitoringClientCall<>( channel.newCall(methodDescriptor, callOptions), metrics, grpcMethod, configuration, clock); diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectStreamer.java b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectStreamer.java new file mode 100644 index 0000000..cadfbf1 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectStreamer.java @@ -0,0 +1,62 @@ +package info.frostfs.sdk.services.impl.rwhelper; + +import frostfs.object.ObjectServiceGrpc; +import frostfs.object.Service; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.utils.WaitUtil; +import io.grpc.stub.StreamObserver; +import lombok.Getter; + +import static info.frostfs.sdk.constants.AppConst.DEFAULT_POLL_INTERVAL; +import static info.frostfs.sdk.constants.ErrorConst.PROTO_MESSAGE_IS_EMPTY_TEMPLATE; +import static java.util.Objects.isNull; + +public class ObjectStreamer { + private final StreamObserver requestObserver; + private final PutResponseCallback responseObserver; + + public ObjectStreamer(ObjectServiceGrpc.ObjectServiceStub objectServiceStub) { + PutResponseCallback responseObserver = new PutResponseCallback(); + + this.responseObserver = responseObserver; + this.requestObserver = objectServiceStub.put(responseObserver); + } + + public void write(Service.PutRequest request) { + if (isNull(request)) { + throw new ProcessFrostFSException( + String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, Service.PutRequest.class.getName()) + ); + } + requestObserver.onNext(request); + } + + public Service.PutResponse complete() { + requestObserver.onCompleted(); + + while (isNull(responseObserver.getResponse())) { + WaitUtil.sleep(DEFAULT_POLL_INTERVAL); + } + + return responseObserver.getResponse(); + } + + @Getter + private static class PutResponseCallback implements StreamObserver { + private Service.PutResponse response; + + @Override + public void onNext(Service.PutResponse putResponse) { + this.response = putResponse; + } + + @Override + public void onError(Throwable throwable) { + throw new ProcessFrostFSException(throwable); + } + + @Override + public void onCompleted() { + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectWriter.java b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectWriter.java index f082507..9650dbf 100644 --- a/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectWriter.java +++ b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/ObjectWriter.java @@ -1,63 +1,43 @@ package info.frostfs.sdk.services.impl.rwhelper; -import frostfs.object.ObjectServiceGrpc; +import com.google.protobuf.ByteString; import frostfs.object.Service; -import info.frostfs.sdk.exceptions.ProcessFrostFSException; -import info.frostfs.sdk.utils.WaitUtil; -import io.grpc.stub.StreamObserver; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.object.PrmObjectPutBase; +import info.frostfs.sdk.tools.RequestConstructor; +import info.frostfs.sdk.tools.Verifier; +import lombok.AllArgsConstructor; import lombok.Getter; -import static info.frostfs.sdk.constants.ErrorConst.PROTO_MESSAGE_IS_EMPTY_TEMPLATE; -import static java.util.Objects.isNull; +import static info.frostfs.sdk.tools.RequestSigner.sign; +//todo specify a deadline for each stream request, not for the entire stream +@Getter +@AllArgsConstructor public class ObjectWriter { - private static final long POLL_INTERVAL = 10; - private final StreamObserver requestObserver; - private final PutResponseCallback responseObserver; + private final ClientEnvironment environment; + private final PrmObjectPutBase args; + private final ObjectStreamer streamer; - public ObjectWriter(ObjectServiceGrpc.ObjectServiceStub objectServiceStub) { - PutResponseCallback responseObserver = new PutResponseCallback(); + public void write(byte[] buffer) { + var body = Service.PutRequest.Body.newBuilder() + .setChunk(ByteString.copyFrom(buffer)) + .build(); + var chunkRequest = Service.PutRequest.newBuilder() + .setBody(body) + .clearVerifyHeader(); - this.responseObserver = responseObserver; - this.requestObserver = objectServiceStub.put(responseObserver); + RequestConstructor.addMetaHeader(chunkRequest, args.getXHeaders()); + sign(chunkRequest, environment.getKey()); + + streamer.write(chunkRequest.build()); } - public void write(Service.PutRequest request) { - if (isNull(request)) { - throw new ProcessFrostFSException( - String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, Service.PutRequest.class.getName()) - ); - } + public ObjectId complete() { + var response = streamer.complete(); + Verifier.checkResponse(response); - requestObserver.onNext(request); - } - - public Service.PutResponse complete() { - requestObserver.onCompleted(); - - while (isNull(responseObserver.getResponse())) { - WaitUtil.sleep(POLL_INTERVAL); - } - - return responseObserver.getResponse(); - } - - @Getter - private static class PutResponseCallback implements StreamObserver { - private Service.PutResponse response; - - @Override - public void onNext(Service.PutResponse putResponse) { - this.response = putResponse; - } - - @Override - public void onError(Throwable throwable) { - throw new ProcessFrostFSException(throwable); - } - - @Override - public void onCompleted() { - } + return new ObjectId(response.getBody().getObjectId().getValue().toByteArray()); } } diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/PatchStreamer.java b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/PatchStreamer.java new file mode 100644 index 0000000..4cdee35 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/PatchStreamer.java @@ -0,0 +1,62 @@ +package info.frostfs.sdk.services.impl.rwhelper; + +import frostfs.object.ObjectServiceGrpc; +import frostfs.object.Service; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.utils.WaitUtil; +import io.grpc.stub.StreamObserver; +import lombok.Getter; + +import static info.frostfs.sdk.constants.AppConst.DEFAULT_POLL_INTERVAL; +import static info.frostfs.sdk.constants.ErrorConst.PROTO_MESSAGE_IS_EMPTY_TEMPLATE; +import static java.util.Objects.isNull; + +public class PatchStreamer { + private final StreamObserver requestObserver; + private final PatchResponseCallback responseObserver; + + public PatchStreamer(ObjectServiceGrpc.ObjectServiceStub objectServiceStub) { + PatchResponseCallback responseObserver = new PatchResponseCallback(); + + this.responseObserver = responseObserver; + this.requestObserver = objectServiceStub.patch(responseObserver); + } + + public void write(Service.PatchRequest request) { + if (isNull(request)) { + throw new ProcessFrostFSException( + String.format(PROTO_MESSAGE_IS_EMPTY_TEMPLATE, Service.PutRequest.class.getName()) + ); + } + requestObserver.onNext(request); + } + + public Service.PatchResponse complete() { + requestObserver.onCompleted(); + + while (isNull(responseObserver.getResponse())) { + WaitUtil.sleep(DEFAULT_POLL_INTERVAL); + } + + return responseObserver.getResponse(); + } + + @Getter + private static class PatchResponseCallback implements StreamObserver { + private Service.PatchResponse response; + + @Override + public void onNext(Service.PatchResponse patchResponse) { + this.response = patchResponse; + } + + @Override + public void onError(Throwable throwable) { + throw new ProcessFrostFSException(throwable); + } + + @Override + public void onCompleted() { + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/RangeReader.java b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/RangeReader.java new file mode 100644 index 0000000..75c4c64 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/services/impl/rwhelper/RangeReader.java @@ -0,0 +1,25 @@ +package info.frostfs.sdk.services.impl.rwhelper; + +import frostfs.object.Service; +import info.frostfs.sdk.tools.Verifier; + +import java.util.Iterator; + +public class RangeReader { + public Iterator call; + + public RangeReader(Iterator call) { + this.call = call; + } + + public byte[] readChunk() { + if (!call.hasNext()) { + return null; + } + + var response = call.next(); + Verifier.checkResponse(response); + + return response.getBody().getChunk().toByteArray(); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/tools/GrpcClient.java b/client/src/main/java/info/frostfs/sdk/tools/GrpcClient.java index 13cdfe5..9692e3a 100644 --- a/client/src/main/java/info/frostfs/sdk/tools/GrpcClient.java +++ b/client/src/main/java/info/frostfs/sdk/tools/GrpcClient.java @@ -3,7 +3,7 @@ package info.frostfs.sdk.tools; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import info.frostfs.sdk.jdo.ClientSettings; import info.frostfs.sdk.utils.Validator; -import io.grpc.Channel; +import io.grpc.ManagedChannel; import io.grpc.netty.NettyChannelBuilder; import java.net.URI; @@ -16,7 +16,7 @@ public class GrpcClient { private GrpcClient() { } - public static Channel initGrpcChannel(ClientSettings clientSettings) { + public static ManagedChannel initGrpcChannel(ClientSettings clientSettings) { Validator.validate(clientSettings); try { @@ -32,4 +32,15 @@ public class GrpcClient { ); } } + + public static ManagedChannel initGrpcChannel(String address) { + try { + URI uri = new URI(address); + return NettyChannelBuilder.forAddress(uri.getHost(), uri.getPort()).usePlaintext().build(); + } catch (URISyntaxException exp) { + throw new ValidationFrostFSException( + String.format(INVALID_HOST_TEMPLATE, address, exp.getMessage()) + ); + } + } } diff --git a/client/src/main/java/info/frostfs/sdk/tools/RequestConstructor.java b/client/src/main/java/info/frostfs/sdk/tools/RequestConstructor.java index 9ad7457..17a011c 100644 --- a/client/src/main/java/info/frostfs/sdk/tools/RequestConstructor.java +++ b/client/src/main/java/info/frostfs/sdk/tools/RequestConstructor.java @@ -4,9 +4,11 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Message; import frostfs.session.Types; import info.frostfs.sdk.dto.response.MetaHeader; +import info.frostfs.sdk.dto.session.SessionToken; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import info.frostfs.sdk.jdo.ECDsa; import info.frostfs.sdk.mappers.response.MetaHeaderMapper; +import info.frostfs.sdk.mappers.session.SessionMapper; import org.apache.commons.collections4.MapUtils; import java.util.Map; @@ -65,13 +67,11 @@ public class RequestConstructor { setField(request, META_HEADER_FIELD_NAME, metaHeader.build()); } - public static Types.SessionToken createObjectTokenContext(Types.SessionToken sessionToken, + public static Types.SessionToken createObjectTokenContext(SessionToken sessionToken, frostfs.refs.Types.Address address, Types.ObjectSessionContext.Verb verb, ECDsa key) { - if (isNull(sessionToken) || sessionToken.getBody().getObject().getTarget().getSerializedSize() > 0) { - return sessionToken; - } + var protoToken = SessionMapper.deserializeSessionToken(sessionToken.getToken()); var target = Types.ObjectSessionContext.Target.newBuilder() .setContainer(address.getContainerId()); @@ -84,25 +84,22 @@ public class RequestConstructor { .setTarget(target.build()) .setVerb(verb) .build(); - var body = sessionToken.getBody().toBuilder() + var body = protoToken.getBody().toBuilder() .setObject(ctx) - .setSessionKey(ByteString.copyFrom(key.getPublicKeyByte())) .build(); - return sessionToken.toBuilder() + return protoToken.toBuilder() .setSignature(signMessagePart(key, body)) .setBody(body) .build(); } - public static Types.SessionToken createContainerTokenContext(Types.SessionToken sessionToken, + public static Types.SessionToken createContainerTokenContext(SessionToken sessionToken, frostfs.refs.Types.ContainerID containerId, Types.ContainerSessionContext.Verb verb, frostfs.refs.Types.OwnerID ownerId, ECDsa key) { - if (isNull(sessionToken) || sessionToken.getBody().getContainer().getContainerId().getSerializedSize() > 0) { - return sessionToken; - } + var protoToken = SessionMapper.deserializeSessionToken(sessionToken.getToken()); var containerSession = Types.ContainerSessionContext.newBuilder().setVerb(verb); @@ -112,7 +109,7 @@ public class RequestConstructor { containerSession.setContainerId(containerId); } - var bodyBuilder = sessionToken.getBody().toBuilder() + var bodyBuilder = protoToken.getBody().toBuilder() .setContainer(containerSession) .setSessionKey(ByteString.copyFrom(key.getPublicKeyByte())); @@ -122,7 +119,7 @@ public class RequestConstructor { var body = bodyBuilder.build(); - return sessionToken.toBuilder() + return protoToken.toBuilder() .setSignature(signMessagePart(key, body)) .setBody(body) .build(); diff --git a/client/src/main/java/info/frostfs/sdk/tools/Verifier.java b/client/src/main/java/info/frostfs/sdk/tools/Verifier.java index f4dbde3..8a5e968 100644 --- a/client/src/main/java/info/frostfs/sdk/tools/Verifier.java +++ b/client/src/main/java/info/frostfs/sdk/tools/Verifier.java @@ -78,6 +78,10 @@ public class Verifier { } var metaHeader = (Types.ResponseMetaHeader) MessageHelper.getField(response, META_HEADER_FIELD_NAME); + if (isNull(metaHeader) || metaHeader.getSerializedSize() == 0) { + return; + } + var status = ResponseStatusMapper.toModel(metaHeader.getStatus()); if (!status.isSuccess()) { throw new ResponseFrostFSException(status); diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java b/client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java new file mode 100644 index 0000000..1b08d5c --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/tools/ape/MarshalFunction.java @@ -0,0 +1,5 @@ +package info.frostfs.sdk.tools.ape; + +public interface MarshalFunction { + int marshal(byte[] buf, int offset, T t); +} diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java b/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java new file mode 100644 index 0000000..cbba2e2 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/tools/ape/RuleDeserializer.java @@ -0,0 +1,198 @@ +package info.frostfs.sdk.tools.ape; + +import info.frostfs.sdk.dto.ape.*; +import info.frostfs.sdk.enums.ConditionKindType; +import info.frostfs.sdk.enums.ConditionType; +import info.frostfs.sdk.enums.RuleMatchType; +import info.frostfs.sdk.enums.RuleStatus; +import info.frostfs.sdk.exceptions.FrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.lang3.ArrayUtils; + +import java.lang.reflect.Array; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +import static info.frostfs.sdk.constants.ErrorConst.*; +import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; +import static info.frostfs.sdk.constants.RuleConst.*; + +public class RuleDeserializer { + private RuleDeserializer() { + } + + public static Chain deserialize(byte[] data) { + if (ArrayUtils.isEmpty(data)) { + throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); + } + + AtomicInteger offset = new AtomicInteger(0); + Chain chain = new Chain(); + + var version = uInt8Unmarshal(data, offset); + if (version != VERSION) { + throw new FrostFSException(String.format(UNSUPPORTED_MARSHALLER_VERSION_TEMPLATE, version)); + } + + var chainVersion = uInt8Unmarshal(data, offset); + if (chainVersion != CHAIN_MARSHAL_VERSION) { + throw new FrostFSException(String.format(UNSUPPORTED_CHAIN_VERSION_TEMPLATE, chainVersion)); + } + + chain.setId(sliceUnmarshal(data, offset, Byte.class, RuleDeserializer::uInt8Unmarshal)); + chain.setRules(sliceUnmarshal(data, offset, Rule.class, RuleDeserializer::unmarshalRule)); + chain.setMatchType(RuleMatchType.get(uInt8Unmarshal(data, offset))); + + verifyUnmarshal(data, offset); + + return chain; + } + + private static Byte uInt8Unmarshal(byte[] buf, AtomicInteger offset) { + if (buf.length - offset.get() < 1) { + throw new FrostFSException( + String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, Byte.class.getName(), offset.get())); + } + + return buf[offset.getAndAdd(1)]; + } + + public static long varInt(byte[] buf, AtomicInteger offset) { + long ux = uVarInt(buf, offset); // ok to continue in presence of error + long x = ux >> 1; + if ((ux & 1) != 0) { + x = ~x; + } + return x; + } + + + public static long uVarInt(byte[] buf, AtomicInteger offset) { + long x = 0; + int s = 0; + + for (int i = offset.get(); i < buf.length; i++) { + long b = buf[i]; + if (i == MAX_VAR_INT_LENGTH) { + offset.set(-(i + 1)); + return 0; // overflow + } + if (b >= 0) { + if (i == MAX_VAR_INT_LENGTH - 1 && b > 1) { + offset.set(-(i + 1)); + return 0; // overflow + } + offset.set(i + 1); + return x | (b << s); + } + x |= (b & OFFSET127) << s; + s += UNSIGNED_SERIALIZE_SIZE; + } + offset.set(0); + return 0; + } + + private static T[] sliceUnmarshal(byte[] buf, + AtomicInteger offset, + Class clazz, + UnmarshalFunction unmarshalT) { + var size = (int) varInt(buf, offset); + if (size == NULL_SLICE) { + return null; + } + + if (size > MAX_SLICE_LENGTH) { + throw new ValidationFrostFSException(String.format(SLICE_IS_TOO_BIG_TEMPLATE, size)); + } + + if (size < 0) { + throw new ValidationFrostFSException(String.format(SLICE_SIZE_IS_INVALID_TEMPLATE, size)); + } + + T[] result = (T[]) Array.newInstance(clazz, size); + for (int i = 0; i < result.length; i++) { + result[i] = unmarshalT.unmarshal(buf, offset); + } + + return result; + } + + private static boolean boolUnmarshal(byte[] buf, AtomicInteger offset) { + return uInt8Unmarshal(buf, offset) == BYTE_TRUE; + } + + private static long int64Unmarshal(byte[] buf, AtomicInteger offset) { + if (buf.length - offset.get() < Long.BYTES) { + throw new ValidationFrostFSException( + String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, Long.class.getName(), offset.get()) + ); + } + + return varInt(buf, offset); + } + + private static String stringUnmarshal(byte[] buf, AtomicInteger offset) { + int size = (int) int64Unmarshal(buf, offset); + if (size == 0) { + return EMPTY_STRING; + } + + if (size > MAX_SLICE_LENGTH) { + throw new ValidationFrostFSException(String.format(STRING_IS_TOO_BIG_TEMPLATE, size)); + } + + if (size < 0) { + throw new ValidationFrostFSException(String.format(STRING_SIZE_IS_INVALID_TEMPLATE, size)); + } + + if (buf.length - offset.get() < size) { + throw new ValidationFrostFSException( + String.format(BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE, String.class.getName(), offset.get()) + ); + } + + return new String(buf, offset.getAndAdd(size), size, StandardCharsets.UTF_8); + } + + private static Actions unmarshalActions(byte[] buf, AtomicInteger offset) { + Actions actions = new Actions(); + actions.setInverted(boolUnmarshal(buf, offset)); + actions.setNames(sliceUnmarshal(buf, offset, String.class, RuleDeserializer::stringUnmarshal)); + return actions; + } + + private static Condition unmarshalCondition(byte[] buf, AtomicInteger offset) { + Condition condition = new Condition(); + condition.setOp(ConditionType.get(uInt8Unmarshal(buf, offset))); + condition.setKind(ConditionKindType.get(uInt8Unmarshal(buf, offset))); + condition.setKey(stringUnmarshal(buf, offset)); + condition.setValue(stringUnmarshal(buf, offset)); + + return condition; + } + + private static Rule unmarshalRule(byte[] buf, AtomicInteger offset) { + Rule rule = new Rule(); + rule.setStatus(RuleStatus.get(uInt8Unmarshal(buf, offset))); + rule.setActions(unmarshalActions(buf, offset)); + rule.setResources(unmarshalResources(buf, offset)); + rule.setAny(boolUnmarshal(buf, offset)); + rule.setConditions(sliceUnmarshal(buf, offset, Condition.class, RuleDeserializer::unmarshalCondition)); + + return rule; + } + + private static Resources unmarshalResources(byte[] buf, AtomicInteger offset) { + Resources resources = new Resources(); + resources.setInverted(boolUnmarshal(buf, offset)); + resources.setNames(sliceUnmarshal(buf, offset, String.class, RuleDeserializer::stringUnmarshal)); + + return resources; + } + + private static void verifyUnmarshal(byte[] buf, AtomicInteger offset) { + if (buf.length != offset.get()) { + throw new ValidationFrostFSException(UNMARSHAL_SIZE_DIFFERS); + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java b/client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java new file mode 100644 index 0000000..43e7210 --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/tools/ape/RuleSerializer.java @@ -0,0 +1,260 @@ +package info.frostfs.sdk.tools.ape; + +import info.frostfs.sdk.dto.ape.*; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.lang3.StringUtils; + +import java.nio.charset.StandardCharsets; +import java.util.function.Function; + +import static info.frostfs.sdk.constants.ErrorConst.*; +import static info.frostfs.sdk.constants.RuleConst.*; +import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; + +public class RuleSerializer { + private RuleSerializer() { + } + + public static byte[] serialize(Chain chain) { + if (isNull(chain)) { + throw new ValidationFrostFSException(String.format(INPUT_PARAM_IS_MISSING_TEMPLATE, Chain.class.getName())); + } + + int s = U_INT_8_SIZE // Marshaller version + + U_INT_8_SIZE // Chain version + + sliceSize(chain.getId(), b -> BYTE_SIZE) + + sliceSize(chain.getRules(), RuleSerializer::ruleSize) + + U_INT_8_SIZE; // MatchType + + byte[] buf = new byte[s]; + + int offset = uInt8Marshal(buf, 0, VERSION); + offset = uInt8Marshal(buf, offset, (byte) CHAIN_MARSHAL_VERSION); + offset = sliceMarshal(buf, offset, chain.getId(), RuleSerializer::byteMarshal); + offset = sliceMarshal(buf, offset, chain.getRules(), RuleSerializer::marshalRule); + offset = uInt8Marshal(buf, offset, (byte) chain.getMatchType().value); + + verifyMarshal(buf, offset); + + return buf; + } + + private static int sliceSize(T[] slice, Function sizeOf) { + if (isNull(slice)) { + return NULL_SLICE_SIZE; + } + + // Assuming int64Size is the size of the slice + int size = int64Size(slice.length); + for (T v : slice) { + size += sizeOf.apply(v); + } + + return size; + } + + /* + * https://cs.opensource.google/go/go/+/master:src/encoding/binary/varint.go;l=92;drc=dac9b9ddbd5160c5f4552410f5f828 + * 1bd5eed38c + * + * and + * + * https://cs.opensource.google/go/go/+/master:src/encoding/binary/varint.go;l=41;drc=dac9b9ddbd5160c5f4552410f5f828 + * 1bd5eed38c + * */ + private static int int64Size(long value) { + long ux = value << 1; + if (value < 0) { + ux = ~ux; + } + + int size = 0; + while (ux >= OFFSET128) { + size++; + ux >>>= UNSIGNED_SERIALIZE_SIZE; + } + + return size + 1; + } + + private static int stringSize(String s) { + int len = nonNull(s) ? s.length() : 0; + return int64Size(len) + len; + } + + private static int actionsSize(Actions action) { + return BOOL_SIZE // Inverted + + (nonNull(action) ? sliceSize(action.getNames(), RuleSerializer::stringSize) : 0); + } + + private static int resourcesSize(Resources resource) { + return BOOL_SIZE // Inverted + + (nonNull(resource) ? sliceSize(resource.getNames(), RuleSerializer::stringSize) : 0); + } + + private static int conditionSize(Condition condition) { + if (isNull(condition)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Rule.class.getName())); + } + + return BYTE_SIZE // Op + + BYTE_SIZE // Object + + stringSize(condition.getKey()) + + stringSize(condition.getValue()); + } + + private static int ruleSize(Rule rule) { + if (isNull(rule)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Rule.class.getName())); + } + + return BYTE_SIZE // Status + + actionsSize(rule.getActions()) + + resourcesSize(rule.getResources()) + + BOOL_SIZE // Any + + sliceSize(rule.getConditions(), RuleSerializer::conditionSize); + } + + private static int uInt8Marshal(byte[] buf, int offset, byte value) { + if (buf.length - offset < 1) { + throw new ValidationFrostFSException( + String.format(BYTES_ARE_OVER_FOR_SERIALIZE_TEMPLATE, Byte.class.getName(), 1) + ); + } + + buf[offset] = value; + + return offset + 1; + } + + private static int byteMarshal(byte[] buf, int offset, byte value) { + return uInt8Marshal(buf, offset, value); + } + + // putVarInt encodes an int64 into buf and returns the number of bytes written. + private static int putVarInt(byte[] buf, int offset, long x) { + long ux = x << 1; + if (x < 0) { + ux = ~ux; + } + + return putUVarInt(buf, offset, ux); + } + + private static int putUVarInt(byte[] buf, int offset, long x) { + while (x >= OFFSET128) { + buf[offset] = (byte) (x | OFFSET128); + x >>>= UNSIGNED_SERIALIZE_SIZE; + offset++; + } + buf[offset] = (byte) x; + return offset + 1; + } + + private static int int64Marshal(byte[] buf, int offset, long v) { + var size = int64Size(v); + if (buf.length - offset < size) { + throw new ValidationFrostFSException( + String.format(BYTES_ARE_OVER_FOR_SERIALIZE_TEMPLATE, Long.class.getName(), size) + ); + } + + return putVarInt(buf, offset, v); + } + + private static int sliceMarshal(byte[] buf, int offset, T[] slice, MarshalFunction marshalT) { + if (isNull(slice)) { + return int64Marshal(buf, offset, NULL_SLICE); + } + + if (slice.length > MAX_SLICE_LENGTH) { + throw new ValidationFrostFSException(String.format(SLICE_IS_TOO_BIG_TEMPLATE, slice.length)); + } + + offset = int64Marshal(buf, offset, slice.length); + for (T v : slice) { + offset = marshalT.marshal(buf, offset, v); + } + + return offset; + } + + private static int boolMarshal(byte[] buf, int offset, boolean value) { + return uInt8Marshal(buf, offset, value ? BYTE_TRUE : BYTE_FALSE); + } + + private static int stringMarshal(byte[] buf, int offset, String value) { + if (StringUtils.isBlank(value)) { + throw new ValidationFrostFSException(STRING_IS_BLANK); + } + + if (value.length() > MAX_SLICE_LENGTH) { + throw new ValidationFrostFSException(String.format(STRING_IS_TOO_BIG_TEMPLATE, value.length())); + } + + if (buf.length - offset < int64Size(value.length()) + value.length()) { + throw new ValidationFrostFSException( + String.format(BYTES_ARE_OVER_FOR_SERIALIZE_TEMPLATE, String.class.getName(), value.length()) + ); + } + + offset = int64Marshal(buf, offset, value.length()); + if (value.isEmpty()) { + return offset; + } + + byte[] stringBytes = value.getBytes(StandardCharsets.UTF_8); + + // Copy exactly value.length() bytes as in the original code + System.arraycopy(stringBytes, 0, buf, offset, value.length()); + return offset + value.length(); + } + + private static int marshalActions(byte[] buf, int offset, Actions action) { + if (isNull(action)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Actions.class.getName())); + } + + offset = boolMarshal(buf, offset, action.isInverted()); + return sliceMarshal(buf, offset, action.getNames(), RuleSerializer::stringMarshal); + } + + private static int marshalCondition(byte[] buf, int offset, Condition condition) { + if (isNull(condition)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Condition.class.getName())); + } + + offset = byteMarshal(buf, offset, (byte) condition.getOp().value); + offset = byteMarshal(buf, offset, (byte) condition.getKind().value); + offset = stringMarshal(buf, offset, condition.getKey()); + return stringMarshal(buf, offset, condition.getValue()); + } + + private static int marshalRule(byte[] buf, int offset, Rule rule) { + if (isNull(rule)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Rule.class.getName())); + } + + offset = byteMarshal(buf, offset, (byte) rule.getStatus().value); + offset = marshalActions(buf, offset, rule.getActions()); + offset = marshalResources(buf, offset, rule.getResources()); + offset = boolMarshal(buf, offset, rule.isAny()); + return sliceMarshal(buf, offset, rule.getConditions(), RuleSerializer::marshalCondition); + } + + private static int marshalResources(byte[] buf, int offset, Resources resources) { + if (isNull(resources)) { + throw new ValidationFrostFSException(String.format(REQUIRED_FIELD_TEMPLATE, Resources.class.getName())); + } + + offset = boolMarshal(buf, offset, resources.isInverted()); + return sliceMarshal(buf, offset, resources.getNames(), RuleSerializer::stringMarshal); + } + + private static void verifyMarshal(byte[] buf, int lastOffset) { + if (buf.length != lastOffset) { + throw new ValidationFrostFSException(MARSHAL_SIZE_DIFFERS); + } + } +} diff --git a/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java b/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java new file mode 100644 index 0000000..ce0977e --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/tools/ape/UnmarshalFunction.java @@ -0,0 +1,7 @@ +package info.frostfs.sdk.tools.ape; + +import java.util.concurrent.atomic.AtomicInteger; + +public interface UnmarshalFunction { + T unmarshal(byte[] buf, AtomicInteger offset); +} diff --git a/client/src/main/java/info/frostfs/sdk/utils/DeadLineUtil.java b/client/src/main/java/info/frostfs/sdk/utils/DeadLineUtil.java new file mode 100644 index 0000000..96ea46f --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/utils/DeadLineUtil.java @@ -0,0 +1,25 @@ +package info.frostfs.sdk.utils; + +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import io.grpc.stub.AbstractStub; + +import java.util.concurrent.TimeUnit; + +import static info.frostfs.sdk.constants.ErrorConst.PARAM_IS_MISSING_TEMPLATE; +import static java.util.Objects.isNull; + +public class DeadLineUtil { + private DeadLineUtil() { + } + + public static > T deadLineAfter(T stub, long deadLine, TimeUnit timeUnit) { + if (isNull(stub)) { + throw new ValidationFrostFSException( + String.format(PARAM_IS_MISSING_TEMPLATE, AbstractStub.class.getName()) + ); + } + + timeUnit = isNull(timeUnit) ? TimeUnit.MILLISECONDS : timeUnit; + return deadLine > 0 ? stub.withDeadlineAfter(deadLine, timeUnit) : stub; + } +} diff --git a/client/src/main/java/info/frostfs/sdk/utils/FrostFSMessages.java b/client/src/main/java/info/frostfs/sdk/utils/FrostFSMessages.java new file mode 100644 index 0000000..04b7dab --- /dev/null +++ b/client/src/main/java/info/frostfs/sdk/utils/FrostFSMessages.java @@ -0,0 +1,20 @@ +package info.frostfs.sdk.utils; + +import org.slf4j.Logger; + +public class FrostFSMessages { + private FrostFSMessages() { + } + + public static void sessionCreationError(Logger logger, String address, String error) { + logger.warn("Failed to create frostfs session token for client. Address {}, {}", address, error); + } + + public static void errorThresholdReached(Logger logger, String address, long threshold) { + logger.warn("Error threshold reached. Address {}, threshold {}", address, threshold); + } + + public static void healthChanged(Logger logger, String address, boolean healthy, String error) { + logger.warn("Health has changed: {} healthy {}, reason {}", address, healthy, error); + } +} diff --git a/client/src/main/java/info/frostfs/sdk/utils/Validator.java b/client/src/main/java/info/frostfs/sdk/utils/Validator.java index 316f4c8..5d61493 100644 --- a/client/src/main/java/info/frostfs/sdk/utils/Validator.java +++ b/client/src/main/java/info/frostfs/sdk/utils/Validator.java @@ -1,9 +1,6 @@ package info.frostfs.sdk.utils; -import info.frostfs.sdk.annotations.AtLeastOneIsFilled; -import info.frostfs.sdk.annotations.NotBlank; -import info.frostfs.sdk.annotations.NotNull; -import info.frostfs.sdk.annotations.Validate; +import info.frostfs.sdk.annotations.*; import info.frostfs.sdk.constants.ErrorConst; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; @@ -37,6 +34,10 @@ public class Validator { Class clazz = object.getClass(); + if (clazz.isAnnotationPresent(ComplexAtLeastOneIsFilled.class)) { + processComplexAtLeastOneIsFilled(object, clazz, errorMessage); + } + if (clazz.isAnnotationPresent(AtLeastOneIsFilled.class)) { processAtLeastOneIsFilled(object, clazz, errorMessage); } @@ -83,8 +84,22 @@ public class Validator { process(getFieldValue(object, field), errorMessage); } + private static void processComplexAtLeastOneIsFilled(T object, Class clazz, StringBuilder errorMessage) { + var annotation = clazz.getAnnotation(ComplexAtLeastOneIsFilled.class); + for (AtLeastOneIsFilled value : annotation.value()) { + processAtLeastOneIsFilled(object, clazz, errorMessage, value); + } + } + private static void processAtLeastOneIsFilled(T object, Class clazz, StringBuilder errorMessage) { var annotation = clazz.getAnnotation(AtLeastOneIsFilled.class); + processAtLeastOneIsFilled(object, clazz, errorMessage, annotation); + } + + private static void processAtLeastOneIsFilled(T object, + Class clazz, + StringBuilder errorMessage, + AtLeastOneIsFilled annotation) { var emptyFieldsCount = 0; for (String fieldName : annotation.fields()) { var field = getField(clazz, fieldName); @@ -106,6 +121,7 @@ public class Validator { } } + private static Object getFieldValue(T object, Field field) { try { return field.get(object); diff --git a/client/src/test/java/info/frostfs/sdk/FileUtils.java b/client/src/test/java/info/frostfs/sdk/FileUtils.java deleted file mode 100644 index 0eb9e80..0000000 --- a/client/src/test/java/info/frostfs/sdk/FileUtils.java +++ /dev/null @@ -1,31 +0,0 @@ -package info.frostfs.sdk; - -import com.google.common.io.ByteStreams; -import lombok.SneakyThrows; -import org.apache.commons.lang3.StringUtils; - -import java.io.InputStream; - -import static java.util.Objects.isNull; - -public class FileUtils { - - @SneakyThrows - public static byte[] resourceToBytes(String resourcePath) { - if (StringUtils.isBlank(resourcePath)) { - throw new IllegalArgumentException("Blank filename!"); - } - - ClassLoader loader = FileUtils.class.getClassLoader(); - if (isNull(loader)) { - throw new RuntimeException("Class loader is null!"); - } - - InputStream certStream = loader.getResourceAsStream(resourcePath); - if (isNull(certStream)) { - throw new RuntimeException("Resource could not be found!"); - } - - return ByteStreams.toByteArray(certStream); - } -} diff --git a/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java b/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java new file mode 100644 index 0000000..8ae8d48 --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/placement/PlacementVectorTest.java @@ -0,0 +1,238 @@ +package info.frostfs.sdk.placement; + +import info.frostfs.sdk.dto.netmap.*; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.enums.netmap.NodeState; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import lombok.Getter; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; +import org.yaml.snakeyaml.Yaml; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +public class PlacementVectorTest { + private static final Yaml YAML = new Yaml(); + + private static void compareNodes(Map attrs, NodeInfo nodeInfo) { + assertEquals(attrs.size(), nodeInfo.getAttributes().size()); + assertEquals( + attrs.entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()), + nodeInfo.getAttributes().entrySet().stream().sorted(Map.Entry.comparingByKey()).collect(Collectors.toList()) + ); + } + + @SneakyThrows + @Test + public void placementTest() { + Path resourceDirYaml = Paths.get(Objects.requireNonNull(getClass().getClassLoader() + .getResource("placement")).toURI()); + + List yamlFiles; + try (Stream paths = Files.walk(resourceDirYaml)) { + yamlFiles = paths.filter(Files::isRegularFile).collect(Collectors.toList()); + } + + Version v = new Version(2, 13); + String[] addresses = {"localhost", "server1"}; + + for (Path file : yamlFiles) { + TestCase testCase = YAML.loadAs(Files.newInputStream(file), TestCase.class); + + assertNotNull(testCase); + assertNotNull(testCase.nodes); + assertTrue(testCase.nodes.length > 0); + + List nodes = Arrays.stream(testCase.nodes) + .map(n -> new NodeInfo( + n.state, + v, + List.of(addresses), + n.attributes != null ? + Arrays.stream(n.attributes) + .collect(Collectors.toMap(KeyValuePair::getKey, KeyValuePair::getValue)) : + Collections.emptyMap(), + n.getPublicKeyBytes() + )) + .collect(Collectors.toList()); + + NetmapSnapshot netmap = new NetmapSnapshot(100L, nodes); + + assertNotNull(testCase.tests); + + for (var entry : testCase.tests.entrySet()) { + var test = entry.getValue(); + PlacementPolicy policy = new PlacementPolicy( + test.policy.replicas != null ? + Arrays.stream(test.policy.replicas) + .map(r -> new Replica(r.count, r.selector)) + .toArray(Replica[]::new) : + new Replica[0], + test.policy.unique, + test.policy.containerBackupFactor, + test.policy.filters != null + ? Arrays.stream(test.policy.filters) + .map(FilterDto::getFilter) + .toArray(Filter[]::new) + : new Filter[]{}, + test.policy.selectors != null + ? Arrays.stream(test.policy.selectors) + .map(SelectorDto::getSelector) + .toArray(Selector[]::new) + : new Selector[]{} + ); + + try { + var vector = new PlacementVector(netmap); + NodeInfo[][] result = vector.containerNodes(policy, test.getPivotBytes()); + + if (test.result == null) { + if (test.error != null && !test.error.isEmpty()) { + fail("Error is expected but has not been thrown"); + } else { + assertNotNull(test.policy.replicas); + assertEquals(result.length, test.policy.replicas.length); + + for (NodeInfo[] nodesArr : result) { + assertEquals(0, nodesArr.length); + } + } + } else { + assertEquals(test.result.length, result.length); + + for (int i = 0; i < test.result.length; i++) { + assertEquals(test.result[i].length, result[i].length); + for (int j = 0; j < test.result[i].length; j++) { + compareNodes(nodes.get(test.result[i][j]).getAttributes(), result[i][j]); + } + } + + if (test.placement != null + && test.placement.result != null + && test.placement.getPivotBytes() != null) { + NodeInfo[][] placementResult = vector.placementVectors( + result, test.placement.getPivotBytes() + ); + + assertEquals(test.placement.result.length, placementResult.length); + + for (int i = 0; i < placementResult.length; i++) { + assertEquals(test.placement.result[i].length, placementResult[i].length); + for (int j = 0; j < placementResult[i].length; j++) { + compareNodes( + nodes.get(test.placement.result[i][j]).getAttributes(), + placementResult[i][j] + ); + } + } + } + } + } catch (Exception ex) { + if (test.error != null && !test.error.isEmpty()) { + assertTrue(ex.getMessage().contains(test.error)); + } else { + throw ex; + } + } + } + } + } + + + public static class TestCase { + public String name; + public String comment; + public Node[] nodes; + public Map tests; + } + + public static class Node { + public KeyValuePair[] attributes; + public String publicKey; + public String[] addresses; + public NodeState state = NodeState.ONLINE; + + public byte[] getPublicKeyBytes() { + return publicKey == null || publicKey.isEmpty() ? new byte[0] : Base64.getDecoder().decode(publicKey); + } + } + + @Getter + public static class KeyValuePair { + public String key; + public String value; + } + + public static class TestData { + public PolicyDto policy; + public String pivot; + public int[][] result; + public String error; + public ResultData placement; + + public byte[] getPivotBytes() { + return pivot == null ? null : Base64.getDecoder().decode(pivot); + } + } + + public static class PolicyDto { + public boolean unique; + public int containerBackupFactor; + public FilterDto[] filters; + public ReplicaDto[] replicas; + public SelectorDto[] selectors; + } + + public static class SelectorDto { + public int count; + public String name; + public SelectorClause clause; + public String attribute; + public String filter; + + public Selector getSelector() { + return new Selector(name != null ? name : "", count, clause, attribute, filter); + } + } + + public static class FilterDto { + public String name; + public String key; + public FilterOperation op; + public String value; + public FilterDto[] filters; + + public Filter getFilter() { + return new Filter( + name != null ? name : "", + key != null ? key : "", + op, + value != null ? value : "", + filters != null + ? Arrays.stream(filters).map(FilterDto::getFilter).toArray(Filter[]::new) + : new Filter[0] + ); + } + } + + public static class ReplicaDto { + public int count; + public String selector; + } + + public static class ResultData { + public String pivot; + public int[][] result; + + public byte[] getPivotBytes() { + return pivot == null ? null : Base64.getDecoder().decode(pivot); + } + } +} diff --git a/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java b/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java new file mode 100644 index 0000000..741475c --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/services/AccountingClientTest.java @@ -0,0 +1,103 @@ +package info.frostfs.sdk.services; + +import frostfs.accounting.AccountingServiceGrpc; +import frostfs.accounting.Service; +import info.frostfs.sdk.dto.object.OwnerId; +import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.services.impl.AccountingClientImpl; +import info.frostfs.sdk.testgenerator.AccountingGenerator; +import info.frostfs.sdk.tools.RequestConstructor; +import info.frostfs.sdk.tools.RequestSigner; +import info.frostfs.sdk.tools.Verifier; +import io.grpc.Channel; +import io.neow3j.crypto.Base58; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.platform.commons.util.ReflectionUtils; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class AccountingClientTest { + private static final String OWNER_ID = "NVxUSpEEJzYXZZtUs18PrJTD9QZkLLNQ8S"; + + private AccountingClientImpl accountingClient; + + @Mock + private AccountingServiceGrpc.AccountingServiceBlockingStub AccountingServiceClient; + @Mock + private ClientEnvironment clientEnvironment; + @Mock + private Channel channel; + + private MockedStatic verifierMock; + private MockedStatic requestConstructorMock; + private MockedStatic requestSignerMock; + + @BeforeEach + void setUp() throws IllegalAccessException { + when(clientEnvironment.getChannel()).thenReturn(channel); + when(clientEnvironment.getOwnerId()).thenReturn(new OwnerId(OWNER_ID)); + + accountingClient = new AccountingClientImpl(clientEnvironment); + + Field field = ReflectionUtils.findFields(AccountingClientImpl.class, + f -> f.getName().equals("serviceBlockingStub"), + ReflectionUtils.HierarchyTraversalMode.TOP_DOWN) + .get(0); + + field.setAccessible(true); + field.set(accountingClient, AccountingServiceClient); + + verifierMock = Mockito.mockStatic(Verifier.class); + requestConstructorMock = Mockito.mockStatic(RequestConstructor.class); + requestSignerMock = Mockito.mockStatic(RequestSigner.class); + } + + @AfterEach + void cleanUp() { + verifierMock.close(); + requestConstructorMock.close(); + requestSignerMock.close(); + } + + @Test + void getBalance_success() { + //Given + var response = AccountingGenerator.generateBalanceResponse(); + + var captor = ArgumentCaptor.forClass(Service.BalanceRequest.class); + + when(AccountingServiceClient.balance(captor.capture())).thenReturn(response); + + //When + var result = accountingClient.getBalance(new CallContext(0, null)); + + //Then + requestConstructorMock.verify( + () -> RequestConstructor.addMetaHeader(any(Service.BalanceRequest.Builder.class)), + times(1) + ); + requestSignerMock.verify( + () -> RequestSigner.sign(any(Service.BalanceRequest.Builder.class), eq(null)), + times(1) + ); + verifierMock.verify(() -> Verifier.checkResponse(response), times(1)); + + assertEquals(response.getBody().getBalance(), result); + + var request = captor.getValue(); + assertEquals(OWNER_ID, Base58.encode(request.getBody().getOwnerId().getValue().toByteArray())); + } +} diff --git a/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java b/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java index 7cdfcff..d6a4e4b 100644 --- a/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java +++ b/client/src/test/java/info/frostfs/sdk/services/ApeManagerClientTest.java @@ -2,18 +2,22 @@ package info.frostfs.sdk.services; import frostfs.apemanager.APEManagerServiceGrpc; import frostfs.apemanager.Service; -import info.frostfs.sdk.FileUtils; -import info.frostfs.sdk.dto.chain.Chain; +import info.frostfs.sdk.dto.ape.*; import info.frostfs.sdk.dto.chain.ChainTarget; -import info.frostfs.sdk.enums.TargetType; +import info.frostfs.sdk.enums.*; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import info.frostfs.sdk.jdo.ClientEnvironment; +import info.frostfs.sdk.jdo.parameters.CallContext; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainAdd; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainList; +import info.frostfs.sdk.jdo.parameters.ape.PrmApeChainRemove; import info.frostfs.sdk.services.impl.ApeManagerClientImpl; import info.frostfs.sdk.testgenerator.ApeManagerGenerator; import info.frostfs.sdk.tools.RequestConstructor; import info.frostfs.sdk.tools.RequestSigner; import info.frostfs.sdk.tools.Verifier; import io.grpc.Channel; +import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -26,7 +30,8 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import java.lang.reflect.Field; -import java.util.stream.Collectors; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -35,6 +40,9 @@ import static org.mockito.Mockito.*; @ExtendWith(MockitoExtension.class) class ApeManagerClientTest { + private static final String CHAIN_BASE64 = + "AAAaY2hhaW4taWQtdGVzdAIAAAISR2V0T2JqZWN0AAIebmF0aXZlOm9iamVjdC8qAAIAABREZXBhcnRtZW50BEhSAA=="; + private ApeManagerClientImpl apeManagerClient; @Mock @@ -78,6 +86,7 @@ class ApeManagerClientTest { //Given Chain chain = generateChain(); ChainTarget chainTarget = generateChainTarget(); + PrmApeChainAdd params = new PrmApeChainAdd(chain, chainTarget); var response = ApeManagerGenerator.generateAddChainResponse(); @@ -86,7 +95,7 @@ class ApeManagerClientTest { when(apeManagerServiceClient.addChain(captor.capture())).thenReturn(response); //When - var result = apeManagerClient.addChain(chain, chainTarget); + var result = apeManagerClient.addChain(params, new CallContext(0, null)); //Then requestConstructorMock.verify( @@ -102,7 +111,9 @@ class ApeManagerClientTest { assertThat(result).containsOnly(response.getBody().getChainId().toByteArray()); var request = captor.getValue(); - assertThat(request.getBody().getChain().getRaw().toByteArray()).containsOnly(chain.getRaw()); + assertEquals( + Base64.getEncoder().encodeToString(request.getBody().getChain().getRaw().toByteArray()), CHAIN_BASE64) + ; assertEquals(chainTarget.getName(), request.getBody().getTarget().getName()); assertEquals(chainTarget.getType().value, request.getBody().getTarget().getType().getNumber()); } @@ -112,11 +123,15 @@ class ApeManagerClientTest { //Given Chain chain = generateChain(); ChainTarget chainTarget = generateChainTarget(); + PrmApeChainAdd params1 = new PrmApeChainAdd(null, chainTarget); + PrmApeChainAdd params2 = new PrmApeChainAdd(chain, null); + PrmApeChainAdd params3 = new PrmApeChainAdd(null, null); + //When + Then - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(null, chainTarget)); - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(chain, null)); - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(null, null)); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(params1, new CallContext())); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(params2, new CallContext())); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.addChain(params3, new CallContext())); } @Test @@ -124,6 +139,7 @@ class ApeManagerClientTest { //Given Chain chain = generateChain(); ChainTarget chainTarget = generateChainTarget(); + PrmApeChainRemove params = new PrmApeChainRemove(Base64.getDecoder().decode(CHAIN_BASE64), chainTarget); var response = ApeManagerGenerator.generateRemoveChainResponse(); @@ -132,7 +148,7 @@ class ApeManagerClientTest { when(apeManagerServiceClient.removeChain(captor.capture())).thenReturn(response); //When - apeManagerClient.removeChain(chain, chainTarget); + apeManagerClient.removeChain(params, new CallContext(0, null)); //Then requestConstructorMock.verify( @@ -146,7 +162,7 @@ class ApeManagerClientTest { verifierMock.verify(() -> Verifier.checkResponse(response), times(1)); var request = captor.getValue(); - assertThat(request.getBody().getChainId().toByteArray()).containsOnly(chain.getRaw()); + assertThat(request.getBody().getChainId().toByteArray()).containsOnly(Base64.getDecoder().decode(CHAIN_BASE64)); assertEquals(chainTarget.getName(), request.getBody().getTarget().getName()); assertEquals(chainTarget.getType().value, request.getBody().getTarget().getType().getNumber()); } @@ -156,17 +172,21 @@ class ApeManagerClientTest { //Given Chain chain = generateChain(); ChainTarget chainTarget = generateChainTarget(); + PrmApeChainRemove params1 = new PrmApeChainRemove(null, chainTarget); + PrmApeChainRemove params2 = new PrmApeChainRemove(Base64.getDecoder().decode(CHAIN_BASE64), null); + PrmApeChainRemove params3 = new PrmApeChainRemove(null, null); //When + Then - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(null, chainTarget)); - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(chain, null)); - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(null, null)); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(params1, new CallContext())); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(params2, new CallContext())); + assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.removeChain(params3, new CallContext())); } @Test void listChain_success() { //Given ChainTarget chainTarget = generateChainTarget(); + PrmApeChainList params = new PrmApeChainList(chainTarget); var response = ApeManagerGenerator.generateListChainsResponse(); @@ -175,7 +195,7 @@ class ApeManagerClientTest { when(apeManagerServiceClient.listChains(captor.capture())).thenReturn(response); //When - var result = apeManagerClient.listChains(chainTarget); + var result = apeManagerClient.listChains(params, new CallContext(0, null)); //Then requestConstructorMock.verify( @@ -188,11 +208,7 @@ class ApeManagerClientTest { ); verifierMock.verify(() -> Verifier.checkResponse(response), times(1)); - var actual = result.stream().map(Chain::getRaw).collect(Collectors.toList()); - var expected = response.getBody().getChainsList().stream() - .map(chain -> chain.getRaw().toByteArray()) - .collect(Collectors.toList()); - assertThat(actual).hasSize(10).containsAll(expected); + assertThat(result).hasSize(10); var request = captor.getValue(); assertEquals(chainTarget.getName(), request.getBody().getTarget().getName()); @@ -202,12 +218,29 @@ class ApeManagerClientTest { @Test void listChain_wrongParams() { //When + Then - assertThrows(ValidationFrostFSException.class, () -> apeManagerClient.listChains(null)); + assertThrows(ValidationFrostFSException.class, + () -> apeManagerClient.listChains(new PrmApeChainList(null), new CallContext())); } private Chain generateChain() { - byte[] chainRaw = FileUtils.resourceToBytes("test_chain_raw.json"); - return new Chain(chainRaw); + var resources = new Resources(false, new String[]{"native:object/*"}); + var actions = new Actions(false, new String[]{"GetObject"}); + var condition = new Condition( + ConditionType.COND_STRING_EQUALS, ConditionKindType.RESOURCE, "Department", "HR" + ); + + var rule = new Rule(); + rule.setStatus(RuleStatus.ALLOW); + rule.setResources(resources); + rule.setActions(actions); + rule.setAny(false); + rule.setConditions(new Condition[]{condition}); + + var chain = new Chain(); + chain.setId(ArrayUtils.toObject("chain-id-test".getBytes(StandardCharsets.UTF_8))); + chain.setRules(new Rule[]{rule}); + chain.setMatchType(RuleMatchType.DENY_PRIORITY); + return chain; } private ChainTarget generateChainTarget() { diff --git a/client/src/test/java/info/frostfs/sdk/testgenerator/AccountingGenerator.java b/client/src/test/java/info/frostfs/sdk/testgenerator/AccountingGenerator.java new file mode 100644 index 0000000..13a59d2 --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/testgenerator/AccountingGenerator.java @@ -0,0 +1,42 @@ +package info.frostfs.sdk.testgenerator; + +import frostfs.accounting.Service; +import frostfs.accounting.Types; + +public class AccountingGenerator { + + public static Service.BalanceRequest generateBalanceRequest() { + return Service.BalanceRequest.newBuilder() + .setBody(generateBalanceRequestBody()) + .setMetaHeader(SessionGenerator.generateRequestMetaHeader()) + .setVerifyHeader(SessionGenerator.generateRequestVerificationHeader()) + .build(); + } + + public static Service.BalanceRequest.Body generateBalanceRequestBody() { + return Service.BalanceRequest.Body.newBuilder() + .setOwnerId(RefsGenerator.generateOwnerID()) + .build(); + } + + public static Service.BalanceResponse generateBalanceResponse() { + return Service.BalanceResponse.newBuilder() + .setBody(generateBalanceResponseBody()) + .setMetaHeader(SessionGenerator.generateResponseMetaHeader()) + .setVerifyHeader(SessionGenerator.generateResponseVerificationHeader()) + .build(); + } + + public static Service.BalanceResponse.Body generateBalanceResponseBody() { + return Service.BalanceResponse.Body.newBuilder() + .setBalance(generateDecimal()) + .build(); + } + + public static Types.Decimal generateDecimal() { + return Types.Decimal.newBuilder() + .setValue(1) + .setPrecision(2) + .build(); + } +} diff --git a/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java b/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java index 1a9e497..a050114 100644 --- a/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java +++ b/client/src/test/java/info/frostfs/sdk/testgenerator/ApeManagerGenerator.java @@ -3,22 +3,21 @@ package info.frostfs.sdk.testgenerator; import com.google.protobuf.ByteString; import frostfs.ape.Types; import frostfs.apemanager.Service; -import info.frostfs.sdk.FileUtils; import info.frostfs.sdk.Helper; +import org.bouncycastle.util.encoders.Base64; import java.util.ArrayList; public class ApeManagerGenerator { + private static final String BASE64_CHAIN = "AAAaY2hhaW4taWQtdGVzdAIAAAICKgACHm5hdGl2ZTpvYmplY3QvKgAAAA=="; private static ByteString generateChainID() { return ByteString.copyFrom(Helper.getByteArrayFromHex("616c6c6f774f626a476574436e72")); } private static Types.Chain generateRawChain() { - byte[] chainRaw = FileUtils.resourceToBytes("test_chain_raw.json"); - return Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(chainRaw)) + .setRaw(ByteString.copyFrom(Base64.decode(BASE64_CHAIN))) .build(); } diff --git a/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java b/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java new file mode 100644 index 0000000..e3df60c --- /dev/null +++ b/client/src/test/java/info/frostfs/sdk/tools/ape/ApeRuleTest.java @@ -0,0 +1,179 @@ +package info.frostfs.sdk.tools.ape; + +import info.frostfs.sdk.dto.ape.*; +import info.frostfs.sdk.enums.ConditionKindType; +import info.frostfs.sdk.enums.ConditionType; +import info.frostfs.sdk.enums.RuleMatchType; +import info.frostfs.sdk.enums.RuleStatus; +import info.frostfs.sdk.exceptions.FrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.lang3.ArrayUtils; +import org.bouncycastle.util.encoders.Base64; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static java.util.Objects.isNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +public class ApeRuleTest { + + @Test + void apeRuleTest() { + //Given + var resources = new Resources(false, new String[]{"native:object/*"}); + var actions = new Actions(false, new String[]{"*"}); + var rule = new Rule(); + rule.setStatus(RuleStatus.ALLOW); + rule.setResources(resources); + rule.setActions(actions); + rule.setAny(false); + rule.setConditions(new Condition[]{}); + + + var chain = new Chain(); + chain.setId(ArrayUtils.toObject("chain-id-test".getBytes(StandardCharsets.UTF_8))); + chain.setRules(new Rule[]{rule}); + chain.setMatchType(RuleMatchType.DENY_PRIORITY); + + //When + var serialized = RuleSerializer.serialize(chain); + var t = Base64.encode(serialized); + var restoredChain = RuleDeserializer.deserialize(serialized); + + //Then + assertThat(restoredChain.getId()).isNotEmpty().containsOnly(chain.getId()); + assertEquals(chain.getMatchType(), restoredChain.getMatchType()); + compareRules(chain.getRules(), restoredChain.getRules()); + } + + @Test + void apeRuleTest2() { + //Given + var resources = new Resources(true, new String[]{"native:object/*", "U.S.S. ENTERPRISE"}); + var actions = new Actions(true, new String[]{"put", "get"}); + var cond1 = new Condition( + ConditionType.COND_STRING_EQUALS, ConditionKindType.RESOURCE, "key1", "value1" + ); + var cond2 = new Condition( + ConditionType.COND_NUMERIC_GREATER_THAN, ConditionKindType.REQUEST, "key2", "value2" + ); + var rule = new Rule(); + rule.setStatus(RuleStatus.ACCESS_DENIED); + rule.setResources(resources); + rule.setActions(actions); + rule.setAny(true); + rule.setConditions(new Condition[]{cond1, cond2}); + + + var chain = new Chain(); + chain.setId(ArrayUtils.toObject("dumptext".getBytes(StandardCharsets.UTF_8))); + chain.setRules(new Rule[]{rule}); + chain.setMatchType(RuleMatchType.FIRST_MATCH); + + //When + var serialized = RuleSerializer.serialize(chain); + var restoredChain = RuleDeserializer.deserialize(serialized); + + //Then + assertThat(restoredChain.getId()).isNotEmpty().containsOnly(chain.getId()); + assertEquals(chain.getMatchType(), restoredChain.getMatchType()); + compareRules(chain.getRules(), restoredChain.getRules()); + } + + @Test + void apeRuleTest3() { + //Given + var chain = new Chain(); + chain.setMatchType(RuleMatchType.DENY_PRIORITY); + + //When + var serialized = RuleSerializer.serialize(chain); + var restoredChain = RuleDeserializer.deserialize(serialized); + + //Then + assertNull(restoredChain.getId()); + assertEquals(chain.getMatchType(), restoredChain.getMatchType()); + assertNull(restoredChain.getRules()); + } + + @Test + void apeRule_deserialize_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(null)); + assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{})); + assertThrows(FrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{1, 2, 3})); + assertThrows(ValidationFrostFSException.class, () -> RuleDeserializer.deserialize(new byte[]{ + 0x00, 0x00, 0x3A, 0x77, 0x73, 0x3A, 0x69, 0x61, 0x6D, 0x3A, 0x3A, 0x6E, 0x61, 0x6D, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x3A, 0x67, 0x72, 0x6F, 0x75, 0x70, 0x2F, 0x73, 0x6F, (byte) 0x82, (byte) 0x82, + (byte) 0x82, (byte) 0x82, (byte) 0x82, (byte) 0x82, 0x75, (byte) 0x82 + })); + } + + @Test + void apeRule_serialize_wrong() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> RuleSerializer.serialize(null)); + } + + private void compareRules(Rule[] rules1, Rule[] rules2) { + assertThat(rules1).isNotEmpty(); + assertThat(rules2).isNotEmpty(); + + assertEquals(rules1.length, rules2.length); + + for (int ri = 0; ri < rules1.length; ri++) { + var rule1 = rules1[ri]; + var rule2 = rules2[ri]; + + assertEquals(rule1.getStatus(), rule2.getStatus()); + assertEquals(rule1.isAny(), rule2.isAny()); + + compareActions(rule1.getActions(), rule2.getActions()); + compareResources(rule1.getResources(), rule2.getResources()); + compareConditions(rule1.getConditions(), rule2.getConditions()); + } + } + + private void compareActions(Actions actions1, Actions actions2) { + if (isNull(actions1) && isNull(actions2)) { + return; + } + + assertEquals(actions1.isInverted(), actions2.isInverted()); + if (ArrayUtils.isEmpty(actions1.getNames()) && ArrayUtils.isEmpty(actions2.getNames())) { + return; + } + + assertThat(actions2.getNames()).hasSize(actions1.getNames().length).containsOnly(actions1.getNames()); + } + + private void compareResources(Resources resources1, Resources resources2) { + if (isNull(resources1) && isNull(resources2)) { + return; + } + + assertEquals(resources1.isInverted(), resources2.isInverted()); + if (ArrayUtils.isEmpty(resources1.getNames()) && ArrayUtils.isEmpty(resources2.getNames())) { + return; + } + + assertThat(resources2.getNames()).hasSize(resources1.getNames().length).containsOnly(resources1.getNames()); + } + + private void compareConditions(Condition[] conditions1, Condition[] conditions2) { + if (ArrayUtils.isEmpty(conditions1) && ArrayUtils.isEmpty(conditions2)) { + return; + } + + assertEquals(conditions1.length, conditions2.length); + for (int i = 0; i < conditions1.length; i++) { + assertEquals(conditions1[i].getOp(), conditions2[i].getOp()); + assertEquals(conditions1[i].getKind(), conditions2[i].getKind()); + assertEquals(conditions1[i].getKey(), conditions2[i].getKey()); + assertEquals(conditions1[i].getValue(), conditions2[i].getValue()); + } + } + +} diff --git a/client/src/test/resources/placement/cbf_default.yml b/client/src/test/resources/placement/cbf_default.yml new file mode 100644 index 0000000..c43a703 --- /dev/null +++ b/client/src/test/resources/placement/cbf_default.yml @@ -0,0 +1,48 @@ +name: default CBF is 3 +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: St.Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: RU + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: DE + - key: City + value: Berlin + - attributes: + - key: Location + value: Europe + - key: Country + value: FR + - key: City + value: Paris +tests: + set default CBF: + policy: + replicas: + - count: 1 + selector: EU + containerBackupFactor: 0 + selectors: + - name: EU + count: 1 + clause: SAME + attribute: Location + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/client/src/test/resources/placement/cbf_minimal.yml b/client/src/test/resources/placement/cbf_minimal.yml new file mode 100644 index 0000000..2fe2642 --- /dev/null +++ b/client/src/test/resources/placement/cbf_minimal.yml @@ -0,0 +1,52 @@ +name: Real node count multiplier is in range [1, specified CBF] +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: DE + - attributes: + - key: ID + value: '2' + - key: Country + value: DE + - attributes: + - key: ID + value: '3' + - key: Country + value: DE +tests: + select 2, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + select 3, CBF is 2: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 2 + selectors: + - name: X + count: 3 + clause: SAME + attribute: Country + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 diff --git a/client/src/test/resources/placement/cbf_requirements.yml b/client/src/test/resources/placement/cbf_requirements.yml new file mode 100644 index 0000000..ccd58d4 --- /dev/null +++ b/client/src/test/resources/placement/cbf_requirements.yml @@ -0,0 +1,82 @@ +name: CBF requirements +nodes: + - attributes: + - key: ID + value: '1' + - key: Attr + value: Same + - attributes: + - key: ID + value: '2' + - key: Attr + value: Same + - attributes: + - key: ID + value: '3' + - key: Attr + value: Same + - attributes: + - key: ID + value: '4' + - key: Attr + value: Same +tests: + default CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + explicit CBF, no selector: + policy: + replicas: + - count: 2 + containerBackupFactor: 3 + selectors: [] + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select distinct, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 2 + - 1 + - 3 + select same, weak CBF: + policy: + replicas: + - count: 2 + selector: X + containerBackupFactor: 3 + selectors: + - name: X + count: 2 + clause: SAME + attribute: Attr + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/filter_complex.yml b/client/src/test/resources/placement/filter_complex.yml new file mode 100644 index 0000000..1abc46a --- /dev/null +++ b/client/src/test/resources/placement/filter_complex.yml @@ -0,0 +1,207 @@ +name: compound filter +nodes: + - attributes: + - key: Storage + value: SSD + - key: Rating + value: '10' + - key: IntField + value: '100' + - key: Param + value: Value1 +tests: + good: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - op: OR + filters: + - key: Param + op: EQ + value: Value1 + filters: [] + - key: Param + op: EQ + value: Value2 + filters: [] + result: + - - 0 + bad storage type: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: HDD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + bad rating: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '15' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value1 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] + bad param: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: StorageSSD + key: Storage + op: EQ + value: SSD + filters: [] + - name: GoodRating + key: Rating + op: GE + value: '4' + filters: [] + - name: Main + op: AND + filters: + - name: StorageSSD + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + key: IntField + op: LT + value: '123' + filters: [] + - name: GoodRating + op: OPERATION_UNSPECIFIED + filters: [] + - name: '' + op: OR + filters: + - name: '' + key: Param + op: EQ + value: Value0 + filters: [] + - name: '' + key: Param + op: EQ + value: Value2 + filters: [] diff --git a/client/src/test/resources/placement/filter_invalid_integer.yml b/client/src/test/resources/placement/filter_invalid_integer.yml new file mode 100644 index 0000000..6674246 --- /dev/null +++ b/client/src/test/resources/placement/filter_invalid_integer.yml @@ -0,0 +1,43 @@ +name: invalid integer field +nodes: + - attributes: + - key: IntegerField + value: 'true' + - attributes: + - key: IntegerField + value: str +tests: + empty string is not casted to 0: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: LE + value: '8' + filters: [] + non-empty string is not casted to a number: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: IntegerField + op: GE + value: '0' + filters: [] diff --git a/client/src/test/resources/placement/filter_simple.yml b/client/src/test/resources/placement/filter_simple.yml new file mode 100644 index 0000000..7fdd84a --- /dev/null +++ b/client/src/test/resources/placement/filter_simple.yml @@ -0,0 +1,224 @@ +name: single-op filters +nodes: + - attributes: + - key: Rating + value: '4' + - key: Country + value: Germany +tests: + GE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '4' + filters: [] + result: + - - 0 + GE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GE + value: '5' + filters: [] + GT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '3' + filters: [] + result: + - - 0 + GT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: GT + value: '4' + filters: [] + LE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '4' + filters: [] + result: + - - 0 + LE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LE + value: '3' + filters: [] + LT true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '5' + filters: [] + result: + - - 0 + LT false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Rating + op: LT + value: '4' + filters: [] + EQ true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: Germany + filters: [] + result: + - - 0 + EQ false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: EQ + value: China + filters: [] + NE true: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: France + filters: [] + result: + - - 0 + NE false: + policy: + replicas: + - count: 1 + selector: S + containerBackupFactor: 1 + selectors: + - name: S + count: 1 + clause: DISTINCT + filter: Main + filters: + - name: Main + key: Country + op: NE + value: Germany + filters: [] diff --git a/client/src/test/resources/placement/hrw_sort.yml b/client/src/test/resources/placement/hrw_sort.yml new file mode 100644 index 0000000..c84f7c9 --- /dev/null +++ b/client/src/test/resources/placement/hrw_sort.yml @@ -0,0 +1,118 @@ +name: HRW ordering +nodes: + - attributes: + - key: Country + value: Germany + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Germany + - key: Price + value: '4' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '3' + - key: Capacity + value: '10' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '1' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: France + - key: Price + value: '100' + - key: Capacity + value: '1' + - attributes: + - key: Country + value: France + - key: Price + value: '7' + - key: Capacity + value: '10000' + - attributes: + - key: Country + value: Russia + - key: Price + value: '2' + - key: Capacity + value: '1' +tests: + select 3 nodes in 3 distinct countries, same placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 1 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 0 + - 7 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 0 + - 7 + select 6 nodes in 3 distinct countries, different placement: + policy: + replicas: + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: [] + pivot: Y29udGFpbmVySUQ= + result: + - - 5 + - 4 + - 0 + - 1 + - 7 + - 2 + placement: + pivot: b2JqZWN0SUQ= + result: + - - 5 + - 4 + - 0 + - 7 + - 2 + - 1 diff --git a/client/src/test/resources/placement/issue213.yml b/client/src/test/resources/placement/issue213.yml new file mode 100644 index 0000000..8e8aea4 --- /dev/null +++ b/client/src/test/resources/placement/issue213.yml @@ -0,0 +1,52 @@ +name: unnamed selector (nspcc-dev/neofs-api-go#213) +nodes: + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Moscow + - attributes: + - key: Location + value: Europe + - key: Country + value: Russia + - key: City + value: Saint-Petersburg + - attributes: + - key: Location + value: Europe + - key: Country + value: Sweden + - key: City + value: Stockholm + - attributes: + - key: Location + value: Europe + - key: Country + value: Finalnd + - key: City + value: Helsinki +tests: + test: + policy: + replicas: + - count: 4 + containerBackupFactor: 1 + selectors: + - name: '' + count: 4 + clause: DISTINCT + filter: LOC_EU + filters: + - name: LOC_EU + key: Location + op: EQ + value: Europe + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/many_selects.yml b/client/src/test/resources/placement/many_selects.yml new file mode 100644 index 0000000..29efd43 --- /dev/null +++ b/client/src/test/resources/placement/many_selects.yml @@ -0,0 +1,141 @@ +name: single-op filters +nodes: + - attributes: + - key: Country + value: Russia + - key: Rating + value: '1' + - key: City + value: SPB + - attributes: + - key: Country + value: Germany + - key: Rating + value: '5' + - key: City + value: Berlin + - attributes: + - key: Country + value: Russia + - key: Rating + value: '6' + - key: City + value: Moscow + - attributes: + - key: Country + value: France + - key: Rating + value: '4' + - key: City + value: Paris + - attributes: + - key: Country + value: France + - key: Rating + value: '1' + - key: City + value: Lyon + - attributes: + - key: Country + value: Russia + - key: Rating + value: '5' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '7' + - key: City + value: Moscow + - attributes: + - key: Country + value: Germany + - key: Rating + value: '3' + - key: City + value: Darmstadt + - attributes: + - key: Country + value: Germany + - key: Rating + value: '7' + - key: City + value: Frankfurt + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB + - attributes: + - key: Country + value: Russia + - key: Rating + value: '9' + - key: City + value: SPB +tests: + Select: + policy: + replicas: + - count: 1 + selector: SameRU + - count: 1 + selector: DistinctRU + - count: 1 + selector: Good + - count: 1 + selector: Main + containerBackupFactor: 2 + selectors: + - name: SameRU + count: 2 + clause: SAME + attribute: City + filter: FromRU + - name: DistinctRU + count: 2 + clause: DISTINCT + attribute: City + filter: FromRU + - name: Good + count: 2 + clause: DISTINCT + attribute: Country + filter: Good + - name: Main + count: 3 + clause: DISTINCT + attribute: Country + filter: '*' + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + - name: Good + key: Rating + op: GE + value: '4' + result: + - - 0 + - 5 + - 9 + - 10 + - - 2 + - 6 + - 0 + - 5 + - - 1 + - 8 + - 2 + - 5 + - - 3 + - 4 + - 1 + - 7 + - 0 + - 2 diff --git a/client/src/test/resources/placement/multiple_rep.yml b/client/src/test/resources/placement/multiple_rep.yml new file mode 100644 index 0000000..448214f --- /dev/null +++ b/client/src/test/resources/placement/multiple_rep.yml @@ -0,0 +1,46 @@ +name: multiple replicas (#215) +nodes: + - attributes: + - key: City + value: Saint-Petersburg + - attributes: + - key: City + value: Moscow + - attributes: + - key: City + value: Berlin + - attributes: + - key: City + value: Paris +tests: + test: + policy: + replicas: + - count: 1 + selector: LOC_SPB_PLACE + - count: 1 + selector: LOC_MSK_PLACE + containerBackupFactor: 1 + selectors: + - name: LOC_SPB_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_SPB + - name: LOC_MSK_PLACE + count: 1 + clause: CLAUSE_UNSPECIFIED + filter: LOC_MSK + filters: + - name: LOC_SPB + key: City + op: EQ + value: Saint-Petersburg + filters: [] + - name: LOC_MSK + key: City + op: EQ + value: Moscow + filters: [] + result: + - - 0 + - - 1 diff --git a/client/src/test/resources/placement/multiple_rep_asymmetric.yml b/client/src/test/resources/placement/multiple_rep_asymmetric.yml new file mode 100644 index 0000000..61f8f76 --- /dev/null +++ b/client/src/test/resources/placement/multiple_rep_asymmetric.yml @@ -0,0 +1,162 @@ +name: multiple REP, asymmetric +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '5' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '6' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '7' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '8' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '9' + - key: Continent + value: SA + - key: City + value: Lima + - attributes: + - key: ID + value: '10' + - key: Continent + value: AF + - key: City + value: Cairo + - attributes: + - key: ID + value: '11' + - key: Continent + value: NA + - key: City + value: NewYork + - attributes: + - key: ID + value: '12' + - key: Continent + value: NA + - key: City + value: LosAngeles + - attributes: + - key: ID + value: '13' + - key: Continent + value: SA + - key: City + value: Lima +tests: + test: + policy: + replicas: + - count: 1 + selector: SPB + - count: 2 + selector: Americas + containerBackupFactor: 2 + selectors: + - name: SPB + count: 1 + clause: SAME + attribute: City + filter: SPBSSD + - name: Americas + count: 2 + clause: DISTINCT + attribute: City + filter: Americas + filters: + - name: SPBSSD + op: AND + filters: + - name: '' + key: Country + op: EQ + value: RU + filters: [] + - name: '' + key: City + op: EQ + value: St.Petersburg + filters: [] + - name: '' + key: SSD + op: EQ + value: '1' + filters: [] + - name: Americas + op: OR + filters: + - name: '' + key: Continent + op: EQ + value: NA + filters: [] + - name: '' + key: Continent + op: EQ + value: SA + filters: [] + result: + - - 1 + - 4 + - - 8 + - 12 + - 5 + - 10 diff --git a/client/src/test/resources/placement/non_strict.yml b/client/src/test/resources/placement/non_strict.yml new file mode 100644 index 0000000..a01986d --- /dev/null +++ b/client/src/test/resources/placement/non_strict.yml @@ -0,0 +1,52 @@ +name: non-strict selections +comment: These test specify loose selection behaviour, to allow fetching already PUT + objects even when there is not enough nodes to select from. +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + not enough nodes (backup factor): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 + not enough nodes (buckets): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromRU + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + result: + - - 0 diff --git a/client/src/test/resources/placement/rep_only.yml b/client/src/test/resources/placement/rep_only.yml new file mode 100644 index 0000000..b354591 --- /dev/null +++ b/client/src/test/resources/placement/rep_only.yml @@ -0,0 +1,62 @@ +name: REP X +nodes: + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Saint-Petersburg + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Moscow + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Berlin + state: UNSPECIFIED + - publicKey: '' + addresses: [] + attributes: + - key: City + value: Paris + state: UNSPECIFIED +tests: + REP 1: + policy: + replicas: + - count: 1 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + REP 3: + policy: + replicas: + - count: 3 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 3 + - 1 + - 2 + REP 5: + policy: + replicas: + - count: 5 + containerBackupFactor: 0 + selectors: [] + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/select_no_attribute.yml b/client/src/test/resources/placement/select_no_attribute.yml new file mode 100644 index 0000000..02046f3 --- /dev/null +++ b/client/src/test/resources/placement/select_no_attribute.yml @@ -0,0 +1,56 @@ +name: select with unspecified attribute +nodes: + - attributes: + - key: ID + value: '1' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '0' + - attributes: + - key: ID + value: '2' + - key: Country + value: RU + - key: City + value: St.Petersburg + - key: SSD + value: '1' + - attributes: + - key: ID + value: '3' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' + - attributes: + - key: ID + value: '4' + - key: Country + value: RU + - key: City + value: Moscow + - key: SSD + value: '1' +tests: + test: + policy: + replicas: + - count: 1 + selector: X + containerBackupFactor: 1 + selectors: + - name: X + count: 4 + clause: DISTINCT + filter: '*' + filters: [] + result: + - - 0 + - 1 + - 2 + - 3 diff --git a/client/src/test/resources/placement/selector_invalid.yml b/client/src/test/resources/placement/selector_invalid.yml new file mode 100644 index 0000000..9b0a539 --- /dev/null +++ b/client/src/test/resources/placement/selector_invalid.yml @@ -0,0 +1,47 @@ +name: invalid selections +nodes: + - attributes: + - key: Country + value: Russia + - attributes: + - key: Country + value: Germany + - attributes: [] +tests: + missing filter: + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 1 + selectors: + - name: MyStore + count: 1 + clause: DISTINCT + attribute: Country + filter: FromNL + filters: + - name: FromRU + key: Country + op: EQ + value: Russia + filters: [] + error: filter not found + not enough nodes (filter results in empty set): + policy: + replicas: + - count: 1 + selector: MyStore + containerBackupFactor: 2 + selectors: + - name: MyStore + count: 2 + clause: DISTINCT + attribute: Country + filter: FromMoon + filters: + - name: FromMoon + key: Country + op: EQ + value: Moon + filters: [] diff --git a/client/src/test/resources/test_chain_raw.json b/client/src/test/resources/test_chain_raw.json deleted file mode 100644 index 60dfc3b..0000000 --- a/client/src/test/resources/test_chain_raw.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "ID": "", - "Rules": [ - { - "Status": "Allow", - "Actions": { - "Inverted": false, - "Names": [ - "GetObject" - ] - }, - "Resources": { - "Inverted": false, - "Names": [ - "native:object/*" - ] - }, - "Any": false, - "Condition": [ - { - "Op": "StringEquals", - "Object": "Resource", - "Key": "Department", - "Value": "HR" - } - ] - } - ], - "MatchType": "DenyPriority" -} \ No newline at end of file diff --git a/cryptography/pom.xml b/cryptography/pom.xml index 33bf8fb..e9e2c6b 100644 --- a/cryptography/pom.xml +++ b/cryptography/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} cryptography @@ -21,7 +21,7 @@ info.frostfs.sdk exceptions - 0.1.0 + ${revision} com.google.protobuf diff --git a/cryptography/src/main/java/info/frostfs/sdk/Base58.java b/cryptography/src/main/java/info/frostfs/sdk/Base58.java deleted file mode 100644 index d302763..0000000 --- a/cryptography/src/main/java/info/frostfs/sdk/Base58.java +++ /dev/null @@ -1,141 +0,0 @@ -package info.frostfs.sdk; - -import info.frostfs.sdk.exceptions.ProcessFrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import org.apache.commons.lang3.StringUtils; - -import java.util.Arrays; - -import static info.frostfs.sdk.ArrayHelper.concat; -import static info.frostfs.sdk.Helper.getSha256; -import static info.frostfs.sdk.constants.ErrorConst.*; -import static java.util.Objects.isNull; - -public class Base58 { - public static final char[] ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); - public static final int BASE58_SYMBOL_COUNT = 58; - public static final int BASE256_SYMBOL_COUNT = 256; - private static final int BYTE_DIVISION = 0xFF; - private static final char ENCODED_ZERO = ALPHABET[0]; - private static final char BASE58_ASCII_MAX_VALUE = 128; - private static final int[] INDEXES = new int[BASE58_ASCII_MAX_VALUE]; - - static { - Arrays.fill(INDEXES, -1); - for (int i = 0; i < ALPHABET.length; i++) { - INDEXES[ALPHABET[i]] = i; - } - } - - private Base58() { - } - - public static byte[] base58CheckDecode(String input) { - if (StringUtils.isEmpty(input)) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - byte[] buffer = decode(input); - if (buffer.length < 4) { - throw new ProcessFrostFSException(String.format(DECODE_LENGTH_VALUE_TEMPLATE, buffer.length)); - } - - byte[] decode = Arrays.copyOfRange(buffer, 0, buffer.length - 4); - byte[] checksum = getSha256(getSha256(decode)); - var bufferEnd = Arrays.copyOfRange(buffer, buffer.length - 4, buffer.length); - var checksumStart = Arrays.copyOfRange(checksum, 0, 4); - if (!Arrays.equals(bufferEnd, checksumStart)) { - throw new ProcessFrostFSException(INVALID_CHECKSUM); - } - - return decode; - } - - public static String base58CheckEncode(byte[] data) { - if (isNull(data)) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - byte[] checksum = getSha256(getSha256(data)); - var buffer = concat(data, Arrays.copyOfRange(checksum, 0, 4)); - return encode(buffer); - } - - public static String encode(byte[] input) { - if (input.length == 0) { - return ""; - } - // Count leading zeros. - int zeros = 0; - while (zeros < input.length && input[zeros] == 0) { - ++zeros; - } - // Convert base-256 digits to base-58 digits (plus conversion to ASCII characters) - input = Arrays.copyOf(input, input.length); // since we modify it in-place - char[] encoded = new char[input.length * 2]; // upper bound - int outputStart = encoded.length; - for (int inputStart = zeros; inputStart < input.length; ) { - encoded[--outputStart] = ALPHABET[divmod(input, inputStart, BASE256_SYMBOL_COUNT, BASE58_SYMBOL_COUNT)]; - if (input[inputStart] == 0) { - ++inputStart; // optimization - skip leading zeros - } - } - // Preserve exactly as many leading encoded zeros in output as there were leading zeros in input. - while (outputStart < encoded.length && encoded[outputStart] == ENCODED_ZERO) { - ++outputStart; - } - while (--zeros >= 0) { - encoded[--outputStart] = ENCODED_ZERO; - } - // Return encoded string (including encoded leading zeros). - return new String(encoded, outputStart, encoded.length - outputStart); - } - - public static byte[] decode(String input) { - if (input.isEmpty()) { - return new byte[0]; - } - // Convert the base58-encoded ASCII chars to a base58 byte sequence (base58 digits). - byte[] input58 = new byte[input.length()]; - for (int i = 0; i < input.length(); ++i) { - char c = input.charAt(i); - int digit = c < BASE58_ASCII_MAX_VALUE ? INDEXES[c] : -1; - if (digit < 0) { - throw new ValidationFrostFSException(String.format(INVALID_BASE58_CHARACTER_TEMPLATE, (int) c)); - } - input58[i] = (byte) digit; - } - // Count leading zeros. - int zeros = 0; - while (zeros < input58.length && input58[zeros] == 0) { - ++zeros; - } - // Convert base-58 digits to base-256 digits. - byte[] decoded = new byte[input.length()]; - int outputStart = decoded.length; - for (int inputStart = zeros; inputStart < input58.length; ) { - decoded[--outputStart] = divmod(input58, inputStart, BASE58_SYMBOL_COUNT, BASE256_SYMBOL_COUNT); - if (input58[inputStart] == 0) { - ++inputStart; // optimization - skip leading zeros - } - } - // Ignore extra leading zeroes that were added during the calculation. - while (outputStart < decoded.length && decoded[outputStart] == 0) { - ++outputStart; - } - // Return decoded data (including original number of leading zeros). - return Arrays.copyOfRange(decoded, outputStart - zeros, decoded.length); - } - - private static byte divmod(byte[] number, int firstDigit, int base, int divisor) { - // this is just long division which accounts for the base of the input digits - int remainder = 0; - for (int i = firstDigit; i < number.length; i++) { - int digit = (int) number[i] & BYTE_DIVISION; - int temp = remainder * base + digit; - number[i] = (byte) (temp / divisor); - remainder = temp % divisor; - } - return (byte) remainder; - } -} diff --git a/cryptography/src/main/java/info/frostfs/sdk/Helper.java b/cryptography/src/main/java/info/frostfs/sdk/Helper.java index 7ee18d2..0ca1dab 100644 --- a/cryptography/src/main/java/info/frostfs/sdk/Helper.java +++ b/cryptography/src/main/java/info/frostfs/sdk/Helper.java @@ -4,7 +4,6 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Message; import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.crypto.digests.RIPEMD160Digest; import java.math.BigInteger; import java.security.MessageDigest; @@ -15,24 +14,11 @@ import static java.util.Objects.isNull; public class Helper { private static final String SHA256 = "SHA-256"; - private static final int RIPEMD_160_HASH_BYTE_LENGTH = 20; private static final int HEX_RADIX = 16; private Helper() { } - public static byte[] getRipemd160(byte[] value) { - if (isNull(value)) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - var hash = new byte[RIPEMD_160_HASH_BYTE_LENGTH]; - var digest = new RIPEMD160Digest(); - digest.update(value, 0, value.length); - digest.doFinal(hash, 0); - return hash; - } - public static MessageDigest getSha256Instance() { try { return MessageDigest.getInstance(SHA256); diff --git a/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java b/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java index f939c08..248b717 100644 --- a/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java +++ b/cryptography/src/main/java/info/frostfs/sdk/KeyExtension.java @@ -2,7 +2,6 @@ package info.frostfs.sdk; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import org.apache.commons.lang3.StringUtils; import org.bouncycastle.asn1.sec.SECNamedCurves; import org.bouncycastle.asn1.sec.SECObjectIdentifiers; import org.bouncycastle.asn1.x9.X9ECParameters; @@ -10,12 +9,7 @@ import org.bouncycastle.crypto.params.ECDomainParameters; import org.bouncycastle.crypto.params.ECPrivateKeyParameters; import org.bouncycastle.crypto.params.ECPublicKeyParameters; import org.bouncycastle.jce.spec.ECNamedCurveSpec; -import org.bouncycastle.math.ec.ECPoint; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.security.KeyFactory; import java.security.NoSuchAlgorithmException; import java.security.PrivateKey; @@ -24,63 +18,20 @@ import java.security.spec.ECParameterSpec; import java.security.spec.ECPrivateKeySpec; import java.security.spec.ECPublicKeySpec; import java.security.spec.InvalidKeySpecException; -import java.util.Arrays; -import static info.frostfs.sdk.Helper.getRipemd160; -import static info.frostfs.sdk.Helper.getSha256; -import static info.frostfs.sdk.constants.ErrorConst.*; +import static info.frostfs.sdk.constants.ErrorConst.COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE; +import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING; import static java.util.Objects.isNull; import static org.bouncycastle.util.BigIntegers.fromUnsignedByteArray; public class KeyExtension { - public static final byte NEO_ADDRESS_VERSION = 0x35; private static final String CURVE_NAME = "secp256r1"; private static final String SECURITY_ALGORITHM = "EC"; - private static final int PS_IN_HASH160 = 0x0C; - private static final int DECODE_ADDRESS_LENGTH = 21; private static final int COMPRESSED_PUBLIC_KEY_LENGTH = 33; - private static final int UNCOMPRESSED_PUBLIC_KEY_LENGTH = 65; - private static final int CHECK_SIG_DESCRIPTOR = ByteBuffer - .wrap(getSha256("System.Crypto.CheckSig".getBytes(StandardCharsets.US_ASCII))) - .order(ByteOrder.LITTLE_ENDIAN).getInt(); private KeyExtension() { } - public static byte[] compress(byte[] publicKey) { - checkInputValue(publicKey); - if (publicKey.length != UNCOMPRESSED_PUBLIC_KEY_LENGTH) { - throw new ValidationFrostFSException(String.format( - UNCOMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, UNCOMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length - )); - } - - var secp256R1 = SECNamedCurves.getByOID(SECObjectIdentifiers.secp256r1); - var point = secp256R1.getCurve().decodePoint(publicKey); - return point.getEncoded(true); - } - - public static byte[] getPrivateKeyFromWIF(String wif) { - if (StringUtils.isEmpty(wif)) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - var data = Base58.base58CheckDecode(wif); - return Arrays.copyOfRange(data, 1, data.length - 1); - } - - public static byte[] loadPublicKey(byte[] privateKey) { - checkInputValue(privateKey); - - X9ECParameters params = SECNamedCurves.getByOID(SECObjectIdentifiers.secp256r1); - ECDomainParameters domain = new ECDomainParameters( - params.getCurve(), params.getG(), params.getN(), params.getH() - ); - ECPoint q = domain.getG().multiply(new BigInteger(1, privateKey)); - ECPublicKeyParameters publicParams = new ECPublicKeyParameters(q, domain); - return publicParams.getQ().getEncoded(true); - } - public static PrivateKey loadPrivateKey(byte[] privateKey) { checkInputValue(privateKey); @@ -134,58 +85,6 @@ public class KeyExtension { } } - public static byte[] getScriptHash(byte[] publicKey) { - checkInputValue(publicKey); - - var script = createSignatureRedeemScript(publicKey); - return getRipemd160(getSha256(script)); - } - - public static String publicKeyToAddress(byte[] publicKey) { - checkInputValue(publicKey); - if (publicKey.length != COMPRESSED_PUBLIC_KEY_LENGTH) { - throw new ValidationFrostFSException(String.format( - ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, COMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length - )); - } - - return toAddress(getScriptHash(publicKey)); - } - - private static String toAddress(byte[] scriptHash) { - checkInputValue(scriptHash); - byte[] data = new byte[DECODE_ADDRESS_LENGTH]; - data[0] = NEO_ADDRESS_VERSION; - System.arraycopy(scriptHash, 0, data, 1, scriptHash.length); - return Base58.base58CheckEncode(data); - } - - private static byte[] getBytes(int value) { - byte[] buffer = new byte[4]; - - for (int i = 0; i < buffer.length; i++) { - buffer[i] = (byte) (value >> i * Byte.SIZE); - } - - return buffer; - } - - private static byte[] createSignatureRedeemScript(byte[] publicKey) { - checkInputValue(publicKey); - if (publicKey.length != COMPRESSED_PUBLIC_KEY_LENGTH) { - throw new ValidationFrostFSException(String.format( - ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE, COMPRESSED_PUBLIC_KEY_LENGTH, publicKey.length - )); - } - - var script = new byte[]{PS_IN_HASH160, COMPRESSED_PUBLIC_KEY_LENGTH}; //PUSHDATA1 33 - - script = ArrayHelper.concat(script, publicKey); - script = ArrayHelper.concat(script, new byte[]{UNCOMPRESSED_PUBLIC_KEY_LENGTH}); //SYSCALL - script = ArrayHelper.concat(script, getBytes(CHECK_SIG_DESCRIPTOR)); //Neo_Crypto_CheckSig - return script; - } - private static void checkInputValue(byte[] data) { if (isNull(data) || data.length == 0) { throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); diff --git a/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java b/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java deleted file mode 100644 index bebcae4..0000000 --- a/cryptography/src/test/java/info/frostfs/sdk/Base58Test.java +++ /dev/null @@ -1,56 +0,0 @@ -package info.frostfs.sdk; - -import info.frostfs.sdk.exceptions.ProcessFrostFSException; -import info.frostfs.sdk.exceptions.ValidationFrostFSException; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -public class Base58Test { - private static final String WIF = "L1YS4myg3xHPvi3FHeLaEt7G8upwJaWL5YLV7huviuUtXFpzBMqZ"; - private static final byte[] DECODE = new byte[]{ - -128, -128, -5, 30, -36, -118, 85, -67, -6, 81, 43, 93, -38, 106, 21, -88, 127, 15, 125, -79, -17, -40, 77, - -15, 122, -88, 72, 109, -47, 125, -80, -40, -38, 1 - }; - - @Test - void base58DecodeEncode() { - //When + Then - assertEquals(WIF, Base58.base58CheckEncode(Base58.base58CheckDecode(WIF))); - } - - @Test - void base58Decode_success() { - //When - var decode = Base58.base58CheckDecode(WIF); - - //Then - assertThat(decode).hasSize(34).containsExactly(DECODE); - } - - @Test - void base58Decode_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode(null)); - assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode("")); - assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckDecode("WIF")); - assertThrows(ProcessFrostFSException.class, () -> Base58.base58CheckDecode("fh")); - } - - @Test - void base58Encode_success() { - //When - var encode = Base58.base58CheckEncode(DECODE); - - //Then - assertEquals(WIF, encode); - } - - @Test - void base58Encode_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> Base58.base58CheckEncode(null)); - } -} diff --git a/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java b/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java index 536e881..00a9121 100644 --- a/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java +++ b/cryptography/src/test/java/info/frostfs/sdk/HelperTest.java @@ -10,38 +10,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; public class HelperTest { - @Test - void getRipemd160_success() { - //Given - var value = new byte[]{1, 2, 3, 4, 5}; - var expected = new byte[] - {-21, -126, 92, 75, 36, -12, 37, 7, 122, 6, 124, -61, -66, -12, 87, 120, 63, 90, -41, 5}; - //When - var result = Helper.getRipemd160(value); - - //Then - assertThat(result).hasSize(20).containsExactly(expected); - } - - @Test - void getRipemd160_givenParamIsNull() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> Helper.getRipemd160(null)); - } - - @Test - void getRipemd160_givenParamsIsEmpty() { - //Given - var value = new byte[]{}; - var expected = new byte[] - {-100, 17, -123, -91, -59, -23, -4, 84, 97, 40, 8, -105, 126, -24, -11, 72, -78, 37, -115, 49}; - //When - var result = Helper.getRipemd160(value); - - //Then - assertThat(result).hasSize(20).containsExactly(expected); - } - @Test void getSha256Instance() { //When diff --git a/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java b/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java index 2f9b1b9..a99fe4d 100644 --- a/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java +++ b/cryptography/src/test/java/info/frostfs/sdk/KeyExtensionTest.java @@ -8,8 +8,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; public class KeyExtensionTest { - private static final String WIF = "L1YS4myg3xHPvi3FHeLaEt7G8upwJaWL5YLV7huviuUtXFpzBMqZ"; - private static final String OWNER_ID = "NVxUSpEEJzYXZZtUs18PrJTD9QZkLLNQ8S"; private static final byte[] PRIVATE_KEY = new byte[]{ -128, -5, 30, -36, -118, 85, -67, -6, 81, 43, 93, -38, 106, 21, -88, 127, 15, 125, -79, -17, -40, 77, -15, 122, -88, 72, 109, -47, 125, -80, -40, -38 @@ -24,38 +22,6 @@ public class KeyExtensionTest { -68, -73, 65, -57, -26, 75, 4, -51, -40, -20, 75, 89, -59, 111, 96, -80, 56, 13 }; - @Test - void getPrivateKeyFromWIF_success() { - //When - var privateKey = KeyExtension.getPrivateKeyFromWIF(WIF); - - //Then - assertThat(privateKey).hasSize(32).containsExactly(PRIVATE_KEY); - } - - @Test - void getPrivateKeyFromWIF_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPrivateKeyFromWIF("")); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPrivateKeyFromWIF(null)); - } - - @Test - void loadPublicKey_success() { - //When - var publicKey = KeyExtension.loadPublicKey(PRIVATE_KEY); - - //Then - assertThat(publicKey).hasSize(33).containsExactly(PUBLIC_KEY); - } - - @Test - void loadPublicKey_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.loadPublicKey(null)); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.loadPublicKey(new byte[]{})); - } - @Test void loadPrivateKey_success() { //When @@ -92,61 +58,4 @@ public class KeyExtensionTest { assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPublicKeyFromBytes(new byte[]{})); assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getPublicKeyFromBytes(PRIVATE_KEY)); } - - @Test - void getScriptHash_success() { - //Given - var expected = new byte[]{ - 110, 42, -125, -76, -25, -44, -94, 22, -98, 117, -100, -5, 103, 74, -128, -51, 37, -116, -102, 71 - }; - - //When - var hash = KeyExtension.getScriptHash(PUBLIC_KEY); - - //Then - assertThat(hash).hasSize(20).containsExactly(expected); - } - - @Test - void getScriptHash_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(null)); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(new byte[]{})); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.getScriptHash(PRIVATE_KEY)); - } - - @Test - void publicKeyToAddress_success() { - //When - var address = KeyExtension.publicKeyToAddress(PUBLIC_KEY); - - //Then - assertEquals(OWNER_ID, address); - } - - @Test - void publicKeyToAddress_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(null)); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(new byte[]{})); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.publicKeyToAddress(PRIVATE_KEY)); - } - - @Test - void compress_success() { - //When - var publicKey = KeyExtension.compress(UNCOMPRESSED_PUBLIC_KEY); - - //Then - assertThat(publicKey).hasSize(33).containsExactly(PUBLIC_KEY); - } - - @Test - void compress_wrong() { - //When + Then - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(null)); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(new byte[]{})); - assertThrows(ValidationFrostFSException.class, () -> KeyExtension.compress(PUBLIC_KEY)); - } - } diff --git a/exceptions/pom.xml b/exceptions/pom.xml index 2c80de2..c3b4546 100644 --- a/exceptions/pom.xml +++ b/exceptions/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} exceptions diff --git a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java index 1bf25d4..8c248c9 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java +++ b/exceptions/src/main/java/info/frostfs/sdk/constants/ErrorConst.java @@ -2,6 +2,7 @@ package info.frostfs.sdk.constants; public class ErrorConst { public static final String OBJECT_IS_NULL = "object must not be null"; + public static final String STRING_IS_BLANK = "string must not be blank"; public static final String INPUT_PARAM_IS_MISSING = "input parameter is not present"; public static final String SOME_PARAM_IS_MISSING = "one of the input parameters is not present"; public static final String PARAM_IS_MISSING_TEMPLATE = "param %s is not present"; @@ -19,6 +20,7 @@ public class ErrorConst { public static final String UNEXPECTED_MESSAGE_TYPE_TEMPLATE = "unexpected message type, expected %s, actually %s"; public static final String WIF_IS_INVALID = "WIF is invalid"; + public static final String WALLET_IS_INVALID = "wallet is not present"; public static final String UNEXPECTED_STREAM = "unexpected end of stream"; public static final String INVALID_HOST_TEMPLATE = "host %s has invalid format. Error: %s"; public static final String INVALID_RESPONSE = "invalid response"; @@ -27,23 +29,48 @@ public class ErrorConst { public static final String UNKNOWN_ENUM_VALUE_TEMPLATE = "unknown %s value: %s"; public static final String INPUT_PARAM_IS_NOT_SHA256 = "%s must be a sha256 hash"; - public static final String DECODE_LENGTH_VALUE_TEMPLATE = "decode array length must be >= 4, but %s"; - public static final String INVALID_BASE58_CHARACTER_TEMPLATE = "invalid character in Base58: 0x%04x"; - public static final String INVALID_CHECKSUM = "checksum does not match"; public static final String WRONG_SIGNATURE_SIZE_TEMPLATE = "wrong signature size. Expected length=%s, actual=%s"; - public static final String ENCODED_COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = - "publicKey isn't encoded compressed public key. Expected length=%s, actual=%s"; - public static final String UNCOMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = - "compress argument isn't uncompressed public key. Expected length=%s, actual=%s"; public static final String COMPRESSED_PUBLIC_KEY_WRONG_LENGTH_TEMPLATE = "decompress argument isn't compressed public key. Expected length=%s, actual=%s"; public static final String WRONG_UUID_SIZE_TEMPLATE = "uuid byte array length must be %s"; + public static final String POOL_CLIENT_UNHEALTHY = "pool client unhealthy"; + public static final String POOL_PEERS_IS_MISSING = "no FrostFS peers configured"; + public static final String POOL_NODES_UNHEALTHY = "at least one node must be healthy"; + public static final String POOL_CLIENTS_UNHEALTHY = "cannot find alive client"; + public static final String POOL_NOT_DIALED = "pool not dialed"; + + public static final String SESSION_CREATE_FAILED = "cannot create session"; public static final String FIELDS_DELIMITER_COMMA = ", "; public static final String FIELDS_DELIMITER_OR = " or "; + public static final String UNSUPPORTED_MARSHALLER_VERSION_TEMPLATE = "unsupported marshaller version %s"; + public static final String UNSUPPORTED_CHAIN_VERSION_TEMPLATE = "unsupported chain version %s"; + public static final String MARSHAL_SIZE_DIFFERS = "actual data size differs from expected"; + public static final String UNMARSHAL_SIZE_DIFFERS = "unmarshalled bytes left"; + public static final String BYTES_ARE_OVER_FOR_SERIALIZE_TEMPLATE = + "not enough bytes left to serialize value of type %s with length=%s"; + public static final String BYTES_ARE_OVER_FOR_DESERIALIZE_TEMPLATE = + "not enough bytes left to deserialize a value of type %s from offset=%s"; + public static final String SLICE_IS_TOO_BIG_TEMPLATE = "slice size is too big=%s"; + public static final String SLICE_SIZE_IS_INVALID_TEMPLATE = "invalid slice size=%s"; + public static final String STRING_IS_TOO_BIG_TEMPLATE = "string size is too big=%s"; + public static final String STRING_SIZE_IS_INVALID_TEMPLATE = "invalid string size=%s"; + + public static final String FILTER_NAME_IS_EMPTY = "Filter name for selector is empty"; + public static final String INVALID_FILTER_NAME_TEMPLATE = "filter name is invalid: '%s' is reserved"; + public static final String INVALID_FILTER_OPERATION_TEMPLATE = "invalid filter operation: %s"; + public static final String FILTER_NOT_FOUND = "filter not found"; + public static final String FILTER_NOT_FOUND_TEMPLATE = "filter not found: SELECT FROM '%s'"; + public static final String NON_EMPTY_FILTERS = "simple filter contains sub-filters"; + public static final String NOT_ENOUGH_NODES = "not enough nodes"; + public static final String NOT_ENOUGH_NODES_TEMPLATE = "not enough nodes to SELECT from '%s'"; + public static final String UNNAMED_TOP_FILTER = "unnamed top-level filter"; + public static final String VECTORS_IS_NULL = "vectors cannot be null"; + public static final String SELECTOR_NOT_FOUND_TEMPLATE = "selector not found: %s"; + private ErrorConst() { } } diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/FrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/FrostFSException.java new file mode 100644 index 0000000..fc41bb9 --- /dev/null +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/FrostFSException.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.exceptions; + +public class FrostFSException extends RuntimeException { + + public FrostFSException() { + } + + public FrostFSException(String message) { + super(message); + } + + public FrostFSException(Throwable cause) { + super(cause); + } +} diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/ProcessFrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/ProcessFrostFSException.java index 47e06b3..6cd3560 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/exceptions/ProcessFrostFSException.java +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/ProcessFrostFSException.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.exceptions; -public class ProcessFrostFSException extends RuntimeException { +public class ProcessFrostFSException extends FrostFSException { public ProcessFrostFSException(String message) { super(message); } diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionExpiredFrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionExpiredFrostFSException.java new file mode 100644 index 0000000..1a3a1e8 --- /dev/null +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionExpiredFrostFSException.java @@ -0,0 +1,8 @@ +package info.frostfs.sdk.exceptions; + +public class SessionExpiredFrostFSException extends FrostFSException { + + public SessionExpiredFrostFSException(String message) { + super(message); + } +} diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionNotFoundFrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionNotFoundFrostFSException.java new file mode 100644 index 0000000..0e8522b --- /dev/null +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/SessionNotFoundFrostFSException.java @@ -0,0 +1,8 @@ +package info.frostfs.sdk.exceptions; + +public class SessionNotFoundFrostFSException extends FrostFSException { + + public SessionNotFoundFrostFSException(String message) { + super(message); + } +} diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/TimeoutFrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/TimeoutFrostFSException.java index fca9263..5e1ee20 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/exceptions/TimeoutFrostFSException.java +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/TimeoutFrostFSException.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.exceptions; -public class TimeoutFrostFSException extends RuntimeException { +public class TimeoutFrostFSException extends FrostFSException { public TimeoutFrostFSException() { } } diff --git a/exceptions/src/main/java/info/frostfs/sdk/exceptions/ValidationFrostFSException.java b/exceptions/src/main/java/info/frostfs/sdk/exceptions/ValidationFrostFSException.java index d294ef8..74e5259 100644 --- a/exceptions/src/main/java/info/frostfs/sdk/exceptions/ValidationFrostFSException.java +++ b/exceptions/src/main/java/info/frostfs/sdk/exceptions/ValidationFrostFSException.java @@ -1,6 +1,6 @@ package info.frostfs.sdk.exceptions; -public class ValidationFrostFSException extends RuntimeException { +public class ValidationFrostFSException extends FrostFSException { public ValidationFrostFSException(String message) { super(message); } diff --git a/models/pom.xml b/models/pom.xml index a5dfe24..e913463 100644 --- a/models/pom.xml +++ b/models/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} models @@ -21,17 +21,17 @@ info.frostfs.sdk cryptography - 0.1.0 + ${revision} info.frostfs.sdk protos - 0.1.0 + ${revision} info.frostfs.sdk exceptions - 0.1.0 + ${revision} diff --git a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java index 9942aa7..58b0041 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/AppConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/AppConst.java @@ -1,6 +1,10 @@ package info.frostfs.sdk.constants; +import java.math.BigInteger; + public class AppConst { + public static final String RESERVED_PREFIX = "__SYSTEM__"; + public static final int DEFAULT_MAJOR_VERSION = 2; public static final int DEFAULT_MINOR_VERSION = 13; public static final int BYTE_SHIFT = 10; @@ -10,6 +14,10 @@ public class AppConst { public static final int OBJECT_CHUNK_SIZE = 3 * MIB; public static final int SHA256_HASH_LENGTH = 32; public static final int UUID_BYTE_ARRAY_LENGTH = 16; + public static final int DEFAULT_GRPC_TIMEOUT = 5; + public static final long DEFAULT_POLL_INTERVAL = 10; + + public static final BigInteger UNSIGNED_LONG_MASK = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); private AppConst() { } diff --git a/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java new file mode 100644 index 0000000..964b3d4 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/constants/AttributeConst.java @@ -0,0 +1,21 @@ +package info.frostfs.sdk.constants; + +import static info.frostfs.sdk.constants.AppConst.RESERVED_PREFIX; + +public class AttributeConst { + public static final String DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE = RESERVED_PREFIX + "DISABLE_HOMOMORPHIC_HASHING"; + + /* + * ATTRIBUTE_PRICE is a key to the node attribute that indicates + * the price in GAS tokens for storing one GB of data during one Epoch. + * */ + public static final String ATTRIBUTE_PRICE = "Price"; + + /* + * ATTRIBUTE_CAPACITY is a key to the node attribute that indicates the total available disk space in Gigabytes. + * */ + public static final String ATTRIBUTE_CAPACITY = "Capacity"; + + private AttributeConst() { + } +} diff --git a/models/src/main/java/info/frostfs/sdk/constants/XHeaderConst.java b/models/src/main/java/info/frostfs/sdk/constants/XHeaderConst.java index e8b2c72..805762a 100644 --- a/models/src/main/java/info/frostfs/sdk/constants/XHeaderConst.java +++ b/models/src/main/java/info/frostfs/sdk/constants/XHeaderConst.java @@ -1,9 +1,10 @@ package info.frostfs.sdk.constants; +import static info.frostfs.sdk.constants.AppConst.RESERVED_PREFIX; + public class XHeaderConst { - public static final String RESERVED_XHEADER_PREFIX = "__SYSTEM__"; - public static final String XHEADER_NETMAP_EPOCH = RESERVED_XHEADER_PREFIX + "NETMAP_EPOCH"; - public static final String XHEADER_NETMAP_LOOKUP_DEPTH = RESERVED_XHEADER_PREFIX + "NETMAP_LOOKUP_DEPTH"; + public static final String XHEADER_NETMAP_EPOCH = RESERVED_PREFIX + "NETMAP_EPOCH"; + public static final String XHEADER_NETMAP_LOOKUP_DEPTH = RESERVED_PREFIX + "NETMAP_LOOKUP_DEPTH"; private XHeaderConst() { } diff --git a/models/src/main/java/info/frostfs/sdk/dto/ape/Actions.java b/models/src/main/java/info/frostfs/sdk/dto/ape/Actions.java new file mode 100644 index 0000000..83415a4 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/ape/Actions.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.dto.ape; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Actions { + private boolean inverted; + private String[] names; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/ape/Chain.java b/models/src/main/java/info/frostfs/sdk/dto/ape/Chain.java new file mode 100644 index 0000000..0e8385f --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/ape/Chain.java @@ -0,0 +1,17 @@ +package info.frostfs.sdk.dto.ape; + +import info.frostfs.sdk.enums.RuleMatchType; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Chain { + private Byte[] id; + private Rule[] rules; + private RuleMatchType matchType; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/ape/Condition.java b/models/src/main/java/info/frostfs/sdk/dto/ape/Condition.java new file mode 100644 index 0000000..8334f94 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/ape/Condition.java @@ -0,0 +1,19 @@ +package info.frostfs.sdk.dto.ape; + +import info.frostfs.sdk.enums.ConditionKindType; +import info.frostfs.sdk.enums.ConditionType; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Condition { + private ConditionType op; + private ConditionKindType kind; + private String key; + private String value; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/ape/Resources.java b/models/src/main/java/info/frostfs/sdk/dto/ape/Resources.java new file mode 100644 index 0000000..017ed4a --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/ape/Resources.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.dto.ape; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Resources { + private boolean inverted; + private String[] names; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/ape/Rule.java b/models/src/main/java/info/frostfs/sdk/dto/ape/Rule.java new file mode 100644 index 0000000..14e462f --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/ape/Rule.java @@ -0,0 +1,27 @@ +package info.frostfs.sdk.dto.ape; + +import info.frostfs.sdk.enums.RuleStatus; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class Rule { + private RuleStatus status; + + // Actions the operation is applied to. + private Actions actions; + + // List of the resources the operation is applied to. + private Resources resources; + + // True if individual conditions must be combined with the logical OR. + // By default, AND is used, so _each_ condition must pass. + private boolean any; + + private Condition[] conditions; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/chain/Chain.java b/models/src/main/java/info/frostfs/sdk/dto/chain/Chain.java deleted file mode 100644 index 984b163..0000000 --- a/models/src/main/java/info/frostfs/sdk/dto/chain/Chain.java +++ /dev/null @@ -1,10 +0,0 @@ -package info.frostfs.sdk.dto.chain; - -import lombok.AllArgsConstructor; -import lombok.Getter; - -@Getter -@AllArgsConstructor -public class Chain { - private final byte[] raw; -} diff --git a/models/src/main/java/info/frostfs/sdk/dto/container/Container.java b/models/src/main/java/info/frostfs/sdk/dto/container/Container.java index 2d0816e..b9a0ea4 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/container/Container.java +++ b/models/src/main/java/info/frostfs/sdk/dto/container/Container.java @@ -2,23 +2,27 @@ package info.frostfs.sdk.dto.container; import info.frostfs.sdk.dto.netmap.PlacementPolicy; import info.frostfs.sdk.dto.netmap.Version; -import info.frostfs.sdk.enums.BasicAcl; +import info.frostfs.sdk.dto.object.OwnerId; +import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; @Getter @Setter +@AllArgsConstructor public class Container { private UUID nonce; - private BasicAcl basicAcl; private PlacementPolicy placementPolicy; private Version version; + private OwnerId ownerId; + private Map attributes = new HashMap<>(); - public Container(BasicAcl basicAcl, PlacementPolicy placementPolicy) { + public Container(PlacementPolicy placementPolicy) { this.nonce = UUID.randomUUID(); - this.basicAcl = basicAcl; this.placementPolicy = placementPolicy; } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java b/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java index 393e3d0..def97fd 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/container/ContainerId.java @@ -1,8 +1,8 @@ package info.frostfs.sdk.dto.container; -import info.frostfs.sdk.Base58; import info.frostfs.sdk.constants.AppConst; import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java new file mode 100644 index 0000000..a957dce --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Filter.java @@ -0,0 +1,15 @@ +package info.frostfs.sdk.dto.netmap; + +import info.frostfs.sdk.enums.netmap.FilterOperation; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class Filter { + private final String name; + private final String key; + private final FilterOperation operation; + private final String value; + private final Filter[] filters; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java new file mode 100644 index 0000000..286f77f --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Hasher.java @@ -0,0 +1,5 @@ +package info.frostfs.sdk.dto.netmap; + +public interface Hasher { + long getHash(); +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java index a3d28f8..a46cf2a 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/NodeInfo.java @@ -1,20 +1,31 @@ package info.frostfs.sdk.dto.netmap; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import lombok.Getter; +import org.apache.commons.codec.digest.MurmurHash3; +import java.math.BigInteger; import java.util.Collections; import java.util.List; import java.util.Map; +import static info.frostfs.sdk.constants.AppConst.UNSIGNED_LONG_MASK; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_CAPACITY; +import static info.frostfs.sdk.constants.AttributeConst.ATTRIBUTE_PRICE; +import static java.util.Objects.isNull; + @Getter -public class NodeInfo { +public class NodeInfo implements Hasher { private final NodeState state; private final Version version; private final List addresses; private final Map attributes; private final byte[] publicKey; + private long hash; + private BigInteger price = UNSIGNED_LONG_MASK; + + public NodeInfo(NodeState state, Version version, List addresses, Map attributes, byte[] publicKey) { this.state = state; @@ -23,4 +34,26 @@ public class NodeInfo { this.attributes = Collections.unmodifiableMap(attributes); this.publicKey = publicKey; } + + public long getHash() { + if (hash == 0) { + hash = MurmurHash3.hash128x64(publicKey, 0, publicKey.length, 0)[0]; + } + + return hash; + } + + public BigInteger getCapacity() { + var capacity = attributes.get(ATTRIBUTE_CAPACITY); + return isNull(capacity) ? BigInteger.valueOf(0) : new BigInteger(capacity); + } + + public BigInteger getPrice() { + if (price.equals(UNSIGNED_LONG_MASK)) { + var priceString = attributes.get(ATTRIBUTE_PRICE); + price = isNull(priceString) ? BigInteger.valueOf(0) : new BigInteger(priceString); + } + + return price; + } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/PlacementPolicy.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/PlacementPolicy.java index 0356dea..d25808b 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/PlacementPolicy.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/PlacementPolicy.java @@ -8,4 +8,15 @@ import lombok.Getter; public class PlacementPolicy { private final Replica[] replicas; private final boolean unique; + private final int backupFactory; + private final Filter[] filters; + private final Selector[] selectors; + + public PlacementPolicy(Replica[] replicas, boolean unique, int backupFactory) { + this.replicas = replicas; + this.unique = unique; + this.backupFactory = backupFactory; + this.filters = null; + this.selectors = null; + } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java index 5c28462..09d059d 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Replica.java @@ -13,6 +13,14 @@ import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; public class Replica { private final int count; private final String selector; + private long ecDataCount; + private long ecParityCount; + + public Replica(int count, String selector, int ecDataCount, int ecParityCount) { + this(count, selector); + this.ecDataCount = Integer.toUnsignedLong(ecDataCount); + this.ecParityCount = Integer.toUnsignedLong(ecParityCount); + } public Replica(int count, String selector) { if (count <= 0) { @@ -32,8 +40,11 @@ public class Replica { ); } - this.count = count; this.selector = EMPTY_STRING; } + + public int getCountNodes() { + return count != 0 ? count : (int) (ecDataCount + ecParityCount); + } } diff --git a/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java new file mode 100644 index 0000000..4dee15c --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/netmap/Selector.java @@ -0,0 +1,17 @@ +package info.frostfs.sdk.dto.netmap; + +import info.frostfs.sdk.enums.netmap.SelectorClause; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +@AllArgsConstructor +public class Selector { + private final String name; + private int count; + private SelectorClause clause; + private String attribute; + private String filter; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java b/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java index 7f34f87..35e9ab2 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/object/ObjectId.java @@ -1,8 +1,8 @@ package info.frostfs.sdk.dto.object; -import info.frostfs.sdk.Base58; import info.frostfs.sdk.constants.AppConst; import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java b/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java index ba65652..82886e1 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java +++ b/models/src/main/java/info/frostfs/sdk/dto/object/OwnerId.java @@ -1,14 +1,11 @@ package info.frostfs.sdk.dto.object; -import info.frostfs.sdk.Base58; import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import io.neow3j.crypto.Base58; import lombok.Getter; import org.apache.commons.lang3.StringUtils; -import static info.frostfs.sdk.KeyExtension.publicKeyToAddress; -import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING; import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING_TEMPLATE; -import static java.util.Objects.isNull; @Getter public class OwnerId { @@ -24,14 +21,6 @@ public class OwnerId { this.value = value; } - public OwnerId(byte[] publicKey) { - if (isNull(publicKey) || publicKey.length == 0) { - throw new ValidationFrostFSException(INPUT_PARAM_IS_MISSING); - } - - this.value = publicKeyToAddress(publicKey); - } - public byte[] toHash() { return Base58.decode(value); } diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/SplitInfo.java b/models/src/main/java/info/frostfs/sdk/dto/object/SplitInfo.java new file mode 100644 index 0000000..cbd9e17 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/object/SplitInfo.java @@ -0,0 +1,17 @@ +package info.frostfs.sdk.dto.object; + +import frostfs.object.Types; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class SplitInfo { + private final Types.SplitInfo splitInfo; + + private final SplitId splitId; + + private final ObjectId link; + + private final ObjectId lastPart; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/patch/Address.java b/models/src/main/java/info/frostfs/sdk/dto/object/patch/Address.java new file mode 100644 index 0000000..ae30880 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/object/patch/Address.java @@ -0,0 +1,13 @@ +package info.frostfs.sdk.dto.object.patch; + +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class Address { + private final ObjectId objectId; + private final ContainerId containerId; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/object/patch/Range.java b/models/src/main/java/info/frostfs/sdk/dto/object/patch/Range.java new file mode 100644 index 0000000..67a56aa --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/dto/object/patch/Range.java @@ -0,0 +1,11 @@ +package info.frostfs.sdk.dto.object.patch; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public class Range { + private long offset; + private long length; +} diff --git a/models/src/main/java/info/frostfs/sdk/dto/response/ResponseStatus.java b/models/src/main/java/info/frostfs/sdk/dto/response/ResponseStatus.java index af69f98..02602f1 100644 --- a/models/src/main/java/info/frostfs/sdk/dto/response/ResponseStatus.java +++ b/models/src/main/java/info/frostfs/sdk/dto/response/ResponseStatus.java @@ -3,19 +3,21 @@ package info.frostfs.sdk.dto.response; import info.frostfs.sdk.enums.StatusCode; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; import static info.frostfs.sdk.constants.FieldConst.EMPTY_STRING; -import static java.util.Objects.isNull; @Getter @Setter public class ResponseStatus { private StatusCode code; private String message; + private String details; - public ResponseStatus(StatusCode code, String message) { + public ResponseStatus(StatusCode code, String message, String details) { this.code = code; - this.message = isNull(message) ? EMPTY_STRING : message; + this.message = StringUtils.isBlank(message) ? EMPTY_STRING : message; + this.details = StringUtils.isBlank(details) ? EMPTY_STRING : details; } public ResponseStatus(StatusCode code) { @@ -25,7 +27,7 @@ public class ResponseStatus { @Override public String toString() { - return String.format("Response status: %s. Message: %s.", code, message); + return String.format("Response status: %s. Message: %s. Details: %s", code, message, details); } public boolean isSuccess() { diff --git a/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java b/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java deleted file mode 100644 index 52bab99..0000000 --- a/models/src/main/java/info/frostfs/sdk/enums/BasicAcl.java +++ /dev/null @@ -1,33 +0,0 @@ -package info.frostfs.sdk.enums; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -public enum BasicAcl { - PRIVATE(0x1C8C8CCC), - PUBLIC_RO(0x1FBF8CFF), - PUBLIC_RW(0x1FBFBFFF), - PUBLIC_APPEND(0x1FBF9FFF), - ; - - private static final Map ENUM_MAP_BY_VALUE; - - static { - Map map = new HashMap<>(); - for (BasicAcl basicAcl : BasicAcl.values()) { - map.put(basicAcl.value, basicAcl); - } - ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); - } - - public final int value; - - BasicAcl(int value) { - this.value = value; - } - - public static BasicAcl get(int value) { - return ENUM_MAP_BY_VALUE.get(value); - } -} diff --git a/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java b/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java new file mode 100644 index 0000000..9f1bc0e --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/ConditionKindType.java @@ -0,0 +1,31 @@ +package info.frostfs.sdk.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum ConditionKindType { + RESOURCE(0), + REQUEST(1), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (ConditionKindType conditionKindType : ConditionKindType.values()) { + map.put(conditionKindType.value, conditionKindType); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + ConditionKindType(int value) { + this.value = value; + } + + public static ConditionKindType get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java b/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java new file mode 100644 index 0000000..dca427d --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/ConditionType.java @@ -0,0 +1,54 @@ +package info.frostfs.sdk.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum ConditionType { + COND_STRING_EQUALS(0), + COND_STRING_NOT_EQUALS(1), + COND_STRING_EQUALS_IGNORE_CASE(2), + COND_STRING_NOT_EQUALS_IGNORE_CASE(3), + + COND_STRING_LIKE(4), + COND_STRING_NOT_LIKE(5), + + COND_STRING_LESS_THAN(6), + COND_STRING_LESS_THAN_EQUALS(7), + COND_STRING_GREATER_THAN(8), + COND_STRING_GREATER_THAN_EQUALS(9), + + COND_NUMERIC_EQUALS(10), + COND_NUMERIC_NOT_EQUALS(11), + + COND_NUMERIC_LESS_THAN(12), + COND_NUMERIC_LESS_THAN_EQUALS(13), + COND_NUMERIC_GREATER_THAN(14), + COND_NUMERIC_GREATER_THAN_EQUALS(15), + + COND_SLICE_CONTAINS(16), + + COND_IP_ADDRESS(17), + COND_NOT_IP_ADDRESS(18), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (ConditionType conditionType : ConditionType.values()) { + map.put(conditionType.value, conditionType); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + ConditionType(int value) { + this.value = value; + } + + public static ConditionType get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java b/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java new file mode 100644 index 0000000..6bbbe82 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/RuleMatchType.java @@ -0,0 +1,34 @@ +package info.frostfs.sdk.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum RuleMatchType { + // DENY_PRIORITY rejects the request if any `Deny` is specified. + DENY_PRIORITY(0), + + // FIRST_MATCH returns the first rule action matched to the request. + FIRST_MATCH(1), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (RuleMatchType ruleMatchType : RuleMatchType.values()) { + map.put(ruleMatchType.value, ruleMatchType); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + RuleMatchType(int value) { + this.value = value; + } + + public static RuleMatchType get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java b/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java new file mode 100644 index 0000000..45289d0 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/RuleStatus.java @@ -0,0 +1,33 @@ +package info.frostfs.sdk.enums; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum RuleStatus { + ALLOW(0), + NO_RULE_FOUND(1), + ACCESS_DENIED(2), + QUOTA_LIMIT_REACHED(3), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (RuleStatus ruleStatus : RuleStatus.values()) { + map.put(ruleStatus.value, ruleStatus); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + RuleStatus(int value) { + this.value = value; + } + + public static RuleStatus get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java new file mode 100644 index 0000000..69513c8 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/FilterOperation.java @@ -0,0 +1,40 @@ +package info.frostfs.sdk.enums.netmap; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum FilterOperation { + OPERATION_UNSPECIFIED(0), + EQ(1), + NE(2), + GT(3), + GE(4), + LT(5), + LE(6), + OR(7), + AND(8), + NOT(9), + LIKE(10), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (FilterOperation nodeState : FilterOperation.values()) { + map.put(nodeState.value, nodeState); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + FilterOperation(int value) { + this.value = value; + } + + public static FilterOperation get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/enums/NodeState.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java similarity index 94% rename from models/src/main/java/info/frostfs/sdk/enums/NodeState.java rename to models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java index 3f833e3..ac6ae78 100644 --- a/models/src/main/java/info/frostfs/sdk/enums/NodeState.java +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/NodeState.java @@ -1,4 +1,4 @@ -package info.frostfs.sdk.enums; +package info.frostfs.sdk.enums.netmap; import java.util.Collections; import java.util.HashMap; diff --git a/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java b/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java new file mode 100644 index 0000000..f375f88 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/enums/netmap/SelectorClause.java @@ -0,0 +1,32 @@ +package info.frostfs.sdk.enums.netmap; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public enum SelectorClause { + CLAUSE_UNSPECIFIED(0), + SAME(1), + DISTINCT(2), + ; + + private static final Map ENUM_MAP_BY_VALUE; + + static { + Map map = new HashMap<>(); + for (SelectorClause nodeState : SelectorClause.values()) { + map.put(nodeState.value, nodeState); + } + ENUM_MAP_BY_VALUE = Collections.unmodifiableMap(map); + } + + public final int value; + + SelectorClause(int value) { + this.value = value; + } + + public static SelectorClause get(int value) { + return ENUM_MAP_BY_VALUE.get(value); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/chain/ChainMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/chain/ChainMapper.java deleted file mode 100644 index 20197f7..0000000 --- a/models/src/main/java/info/frostfs/sdk/mappers/chain/ChainMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package info.frostfs.sdk.mappers.chain; - -import frostfs.ape.Types; -import info.frostfs.sdk.dto.chain.Chain; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.List; -import java.util.stream.Collectors; - -import static java.util.Objects.isNull; - -public class ChainMapper { - private ChainMapper() { - } - - public static List toModels(List chains) { - if (CollectionUtils.isEmpty(chains)) { - return null; - } - - return chains.stream().map(ChainMapper::toModel).collect(Collectors.toList()); - } - - public static Chain toModel(Types.Chain chain) { - if (isNull(chain) || chain.getSerializedSize() == 0) { - return null; - } - - return new Chain(chain.getRaw().toByteArray()); - } -} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerIdMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerIdMapper.java index 26e941c..4daf0fc 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerIdMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerIdMapper.java @@ -7,7 +7,6 @@ import info.frostfs.sdk.dto.container.ContainerId; import static java.util.Objects.isNull; public class ContainerIdMapper { - private ContainerIdMapper() { } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java index 76f8479..0c96e32 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/container/ContainerMapper.java @@ -3,30 +3,44 @@ package info.frostfs.sdk.mappers.container; import com.google.protobuf.ByteString; import frostfs.container.Types; import info.frostfs.sdk.dto.container.Container; -import info.frostfs.sdk.enums.BasicAcl; -import info.frostfs.sdk.exceptions.ProcessFrostFSException; import info.frostfs.sdk.mappers.netmap.PlacementPolicyMapper; import info.frostfs.sdk.mappers.netmap.VersionMapper; +import info.frostfs.sdk.mappers.object.OwnerIdMapper; + +import java.util.Optional; +import java.util.stream.Collectors; import static info.frostfs.sdk.UuidExtension.asBytes; import static info.frostfs.sdk.UuidExtension.asUuid; -import static info.frostfs.sdk.constants.ErrorConst.UNKNOWN_ENUM_VALUE_TEMPLATE; import static java.util.Objects.isNull; public class ContainerMapper { private ContainerMapper() { } - public static Types.Container toGrpcMessage(Container container) { + public static Types.Container.Builder toGrpcMessage(Container container) { if (isNull(container)) { return null; } - return Types.Container.newBuilder() - .setBasicAcl(container.getBasicAcl().value) + var attributes = container.getAttributes().entrySet().stream() + .map(entry -> + Types.Container.Attribute.newBuilder() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build() + ) + .collect(Collectors.toList()); + + var containerGrpc = Types.Container.newBuilder() .setPlacementPolicy(PlacementPolicyMapper.toGrpcMessage(container.getPlacementPolicy())) .setNonce(ByteString.copyFrom(asBytes(container.getNonce()))) - .build(); + .addAllAttributes(attributes); + + Optional.ofNullable(OwnerIdMapper.toGrpcMessage(container.getOwnerId())).ifPresent(containerGrpc::setOwnerId); + Optional.ofNullable(VersionMapper.toGrpcMessage(container.getVersion())).ifPresent(containerGrpc::setVersion); + + return containerGrpc; } public static Container toModel(Types.Container containerGrpc) { @@ -34,16 +48,15 @@ public class ContainerMapper { return null; } - var basicAcl = BasicAcl.get(containerGrpc.getBasicAcl()); - if (isNull(basicAcl)) { - throw new ProcessFrostFSException( - String.format(UNKNOWN_ENUM_VALUE_TEMPLATE, BasicAcl.class.getName(), containerGrpc.getBasicAcl()) - ); - } + var attributes = containerGrpc.getAttributesList().stream() + .collect(Collectors.toMap(Types.Container.Attribute::getKey, Types.Container.Attribute::getValue)); - var container = new Container(basicAcl, PlacementPolicyMapper.toModel(containerGrpc.getPlacementPolicy())); - container.setNonce(asUuid(containerGrpc.getNonce().toByteArray())); - container.setVersion(VersionMapper.toModel(containerGrpc.getVersion())); - return container; + return new Container( + asUuid(containerGrpc.getNonce().toByteArray()), + PlacementPolicyMapper.toModel(containerGrpc.getPlacementPolicy()), + VersionMapper.toModel(containerGrpc.getVersion()), + OwnerIdMapper.toModel(containerGrpc.getOwnerId()), + attributes + ); } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java new file mode 100644 index 0000000..529b26d --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/FilterMapper.java @@ -0,0 +1,85 @@ +package info.frostfs.sdk.mappers.netmap; + +import frostfs.netmap.Types; +import info.frostfs.sdk.dto.netmap.Filter; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING_TEMPLATE; +import static info.frostfs.sdk.constants.ErrorConst.UNKNOWN_ENUM_VALUE_TEMPLATE; +import static java.util.Objects.isNull; + +public class FilterMapper { + private FilterMapper() { + } + + public static List toGrpcMessages(Filter[] filters) { + if (ArrayUtils.isEmpty(filters)) { + return Collections.emptyList(); + } + + return Arrays.stream(filters).map(FilterMapper::toGrpcMessage).collect(Collectors.toList()); + } + + public static Types.Filter toGrpcMessage(Filter filter) { + if (isNull(filter)) { + throw new ValidationFrostFSException( + String.format(INPUT_PARAM_IS_MISSING_TEMPLATE, Filter.class.getName()) + ); + } + + var operation = Types.Operation.forNumber(filter.getOperation().value); + if (isNull(operation)) { + throw new ProcessFrostFSException(String.format( + UNKNOWN_ENUM_VALUE_TEMPLATE, + Types.Operation.class.getName(), + filter.getOperation().name() + )); + } + + return Types.Filter.newBuilder() + .setName(filter.getName()) + .setKey(filter.getKey()) + .setOp(operation) + .setValue(filter.getValue()) + .addAllFilters(toGrpcMessages(filter.getFilters())) + .build(); + } + + public static Filter[] toModels(List filters) { + if (CollectionUtils.isEmpty(filters)) { + return null; + } + + return filters.stream().map(FilterMapper::toModel).toArray(Filter[]::new); + } + + public static Filter toModel(Types.Filter filter) { + if (isNull(filter) || filter.getSerializedSize() == 0) { + return null; + } + + var operation = FilterOperation.get(filter.getOpValue()); + if (isNull(operation)) { + throw new ProcessFrostFSException( + String.format(UNKNOWN_ENUM_VALUE_TEMPLATE, FilterOperation.class.getName(), filter.getOp()) + ); + } + + return new Filter( + filter.getName(), + filter.getKey(), + operation, + filter.getValue(), + toModels(filter.getFiltersList()) + ); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NetmapSnapshotMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NetmapSnapshotMapper.java index b7abf4e..7cad56b 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NetmapSnapshotMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NetmapSnapshotMapper.java @@ -8,7 +8,6 @@ import java.util.stream.Collectors; import static java.util.Objects.isNull; public class NetmapSnapshotMapper { - private NetmapSnapshotMapper() { } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java index f319583..c28cd35 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapper.java @@ -4,7 +4,7 @@ import frostfs.netmap.Service; import frostfs.netmap.Types; import frostfs.netmap.Types.NodeInfo.Attribute; import info.frostfs.sdk.dto.netmap.NodeInfo; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import java.util.stream.Collectors; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapper.java index 22c65bb..03cdc5f 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapper.java @@ -2,7 +2,6 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.PlacementPolicy; -import info.frostfs.sdk.dto.netmap.Replica; import static java.util.Objects.isNull; @@ -15,14 +14,13 @@ public class PlacementPolicyMapper { return null; } - var pp = Types.PlacementPolicy.newBuilder() - .setUnique(placementPolicy.isUnique()); - - for (Replica replica : placementPolicy.getReplicas()) { - pp.addReplicas(ReplicaMapper.toGrpcMessage(replica)); - } - - return pp.build(); + return Types.PlacementPolicy.newBuilder() + .setUnique(placementPolicy.isUnique()) + .setContainerBackupFactor(placementPolicy.getBackupFactory()) + .addAllFilters(FilterMapper.toGrpcMessages(placementPolicy.getFilters())) + .addAllSelectors(SelectorMapper.toGrpcMessages(placementPolicy.getSelectors())) + .addAllReplicas(ReplicaMapper.toGrpcMessages(placementPolicy.getReplicas())) + .build(); } public static PlacementPolicy toModel(Types.PlacementPolicy placementPolicy) { @@ -31,8 +29,11 @@ public class PlacementPolicyMapper { } return new PlacementPolicy( - placementPolicy.getReplicasList().stream().map(ReplicaMapper::toModel).toArray(Replica[]::new), - placementPolicy.getUnique() + ReplicaMapper.toModels(placementPolicy.getReplicasList()), + placementPolicy.getUnique(), + placementPolicy.getContainerBackupFactor(), + FilterMapper.toModels(placementPolicy.getFiltersList()), + SelectorMapper.toModels(placementPolicy.getSelectorsList()) ); } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java index 6bb3939..3ac0105 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/ReplicaMapper.java @@ -2,30 +2,63 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Replica; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING_TEMPLATE; import static java.util.Objects.isNull; public class ReplicaMapper { - private ReplicaMapper() { } + public static List toGrpcMessages(Replica[] replicas) { + if (ArrayUtils.isEmpty(replicas)) { + return Collections.emptyList(); + } + + return Arrays.stream(replicas).map(ReplicaMapper::toGrpcMessage).collect(Collectors.toList()); + } + public static Types.Replica toGrpcMessage(Replica replica) { if (isNull(replica)) { - return null; + throw new ValidationFrostFSException( + String.format(INPUT_PARAM_IS_MISSING_TEMPLATE, Replica.class.getName()) + ); } return Types.Replica.newBuilder() .setCount(replica.getCount()) .setSelector(replica.getSelector()) + .setEcDataCount((int) replica.getEcDataCount()) + .setEcParityCount((int) replica.getEcParityCount()) .build(); } + public static Replica[] toModels(List replicas) { + if (CollectionUtils.isEmpty(replicas)) { + return null; + } + + return replicas.stream().map(ReplicaMapper::toModel).toArray(Replica[]::new); + } + public static Replica toModel(Types.Replica replica) { if (isNull(replica) || replica.getSerializedSize() == 0) { return null; } - return new Replica(replica.getCount(), replica.getSelector()); + return new Replica( + replica.getCount(), + replica.getSelector(), + replica.getEcDataCount(), + replica.getEcParityCount() + ); } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java new file mode 100644 index 0000000..462519a --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/SelectorMapper.java @@ -0,0 +1,85 @@ +package info.frostfs.sdk.mappers.netmap; + +import frostfs.netmap.Types; +import info.frostfs.sdk.dto.netmap.Selector; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ArrayUtils; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.ErrorConst.INPUT_PARAM_IS_MISSING_TEMPLATE; +import static info.frostfs.sdk.constants.ErrorConst.UNKNOWN_ENUM_VALUE_TEMPLATE; +import static java.util.Objects.isNull; + +public class SelectorMapper { + private SelectorMapper() { + } + + public static List toGrpcMessages(Selector[] selectors) { + if (ArrayUtils.isEmpty(selectors)) { + return Collections.emptyList(); + } + + return Arrays.stream(selectors).map(SelectorMapper::toGrpcMessage).collect(Collectors.toList()); + } + + public static Types.Selector toGrpcMessage(Selector selector) { + if (isNull(selector)) { + throw new ValidationFrostFSException( + String.format(INPUT_PARAM_IS_MISSING_TEMPLATE, Selector.class.getName()) + ); + } + + var clause = Types.Clause.forNumber(selector.getClause().value); + if (isNull(clause)) { + throw new ProcessFrostFSException(String.format( + UNKNOWN_ENUM_VALUE_TEMPLATE, + Types.Clause.class.getName(), + selector.getClause().name() + )); + } + + return Types.Selector.newBuilder() + .setName(selector.getName()) + .setCount(selector.getCount()) + .setClause(clause) + .setAttribute(selector.getAttribute()) + .setFilter(selector.getFilter()) + .build(); + } + + public static Selector[] toModels(List selectors) { + if (CollectionUtils.isEmpty(selectors)) { + return null; + } + + return selectors.stream().map(SelectorMapper::toModel).toArray(Selector[]::new); + } + + public static Selector toModel(Types.Selector selector) { + if (isNull(selector) || selector.getSerializedSize() == 0) { + return null; + } + + var clause = SelectorClause.get(selector.getClauseValue()); + if (isNull(clause)) { + throw new ProcessFrostFSException( + String.format(UNKNOWN_ENUM_VALUE_TEMPLATE, SelectorClause.class.getName(), selector.getClause()) + ); + } + + return new Selector( + selector.getName(), + selector.getCount(), + clause, + selector.getAttribute(), + selector.getFilter() + ); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/netmap/VersionMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/netmap/VersionMapper.java index 386cb78..ecfdd57 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/netmap/VersionMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/netmap/VersionMapper.java @@ -6,7 +6,6 @@ import info.frostfs.sdk.dto.netmap.Version; import static java.util.Objects.isNull; public class VersionMapper { - private VersionMapper() { } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapper.java index 687aa86..c0c55a6 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapper.java @@ -2,14 +2,26 @@ package info.frostfs.sdk.mappers.object; import frostfs.object.Types; import info.frostfs.sdk.dto.object.ObjectAttribute; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; import static java.util.Objects.isNull; public class ObjectAttributeMapper { - private ObjectAttributeMapper() { } + public static List toGrpcMessages(List attributes) { + if (CollectionUtils.isEmpty(attributes)) { + return Collections.emptyList(); + } + + return attributes.stream().map(ObjectAttributeMapper::toGrpcMessage).collect(Collectors.toList()); + } + public static Types.Header.Attribute toGrpcMessage(ObjectAttribute attribute) { if (isNull(attribute)) { return null; diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectFrostFSMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectFrostFSMapper.java index 228661b..2427a7b 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectFrostFSMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectFrostFSMapper.java @@ -7,7 +7,6 @@ import info.frostfs.sdk.dto.object.ObjectId; import static java.util.Objects.isNull; public class ObjectFrostFSMapper { - private ObjectFrostFSMapper() { } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectIdMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectIdMapper.java index 5c05d17..e095244 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectIdMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/ObjectIdMapper.java @@ -7,7 +7,6 @@ import info.frostfs.sdk.dto.object.ObjectId; import static java.util.Objects.isNull; public class ObjectIdMapper { - private ObjectIdMapper() { } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java index 905f879..535905f 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/OwnerIdMapper.java @@ -3,11 +3,11 @@ package info.frostfs.sdk.mappers.object; import com.google.protobuf.ByteString; import frostfs.refs.Types; import info.frostfs.sdk.dto.object.OwnerId; +import io.neow3j.crypto.Base58; import static java.util.Objects.isNull; public class OwnerIdMapper { - private OwnerIdMapper() { } @@ -20,4 +20,12 @@ public class OwnerIdMapper { .setValue(ByteString.copyFrom(ownerId.toHash())) .build(); } + + public static OwnerId toModel(Types.OwnerID ownerId) { + if (isNull(ownerId) || ownerId.getSerializedSize() == 0) { + return null; + } + + return new OwnerId(Base58.encode(ownerId.getValue().toByteArray())); + } } diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/SplitInfoMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/SplitInfoMapper.java new file mode 100644 index 0000000..ac5f6ae --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/SplitInfoMapper.java @@ -0,0 +1,30 @@ +package info.frostfs.sdk.mappers.object; + +import frostfs.object.Types; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.SplitId; +import info.frostfs.sdk.dto.object.SplitInfo; + +import static info.frostfs.sdk.UuidExtension.asUuid; +import static java.util.Objects.isNull; + +public class SplitInfoMapper { + private SplitInfoMapper() { + } + + public static SplitInfo toModel(Types.SplitInfo splitInfo) { + if (isNull(splitInfo) || splitInfo.getSerializedSize() == 0) { + return null; + } + + var splitId = new SplitId(asUuid(splitInfo.getSplitId().toByteArray())); + var link = splitInfo.getLink().getSerializedSize() == 0 + ? null + : new ObjectId(splitInfo.getLink().getValue().toByteArray()); + var lastPart = splitInfo.getLastPart().getSerializedSize() == 0 + ? null + : new ObjectId(splitInfo.getLastPart().getValue().toByteArray()); + + return new SplitInfo(splitInfo, splitId, link, lastPart); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/patch/AddressMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/patch/AddressMapper.java new file mode 100644 index 0000000..1e34d62 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/patch/AddressMapper.java @@ -0,0 +1,24 @@ +package info.frostfs.sdk.mappers.object.patch; + +import frostfs.refs.Types; +import info.frostfs.sdk.dto.object.patch.Address; +import info.frostfs.sdk.mappers.container.ContainerIdMapper; +import info.frostfs.sdk.mappers.object.ObjectIdMapper; + +import static java.util.Objects.isNull; + +public class AddressMapper { + private AddressMapper() { + } + + public static Types.Address toGrpcMessage(Address address) { + if (isNull(address)) { + return null; + } + + return Types.Address.newBuilder() + .setContainerId(ContainerIdMapper.toGrpcMessage(address.getContainerId())) + .setObjectId(ObjectIdMapper.toGrpcMessage(address.getObjectId())) + .build(); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/object/patch/RangeMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/object/patch/RangeMapper.java new file mode 100644 index 0000000..0e3ebe5 --- /dev/null +++ b/models/src/main/java/info/frostfs/sdk/mappers/object/patch/RangeMapper.java @@ -0,0 +1,35 @@ +package info.frostfs.sdk.mappers.object.patch; + +import frostfs.object.Service; +import info.frostfs.sdk.dto.object.patch.Range; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static java.util.Objects.isNull; + +public class RangeMapper { + private RangeMapper() { + } + + public static List toGrpcMessages(List ranges) { + if (CollectionUtils.isEmpty(ranges)) { + return Collections.emptyList(); + } + + return ranges.stream().map(RangeMapper::toGrpcMessage).collect(Collectors.toList()); + } + + public static Service.Range toGrpcMessage(Range range) { + if (isNull(range)) { + return null; + } + + return Service.Range.newBuilder() + .setOffset(range.getOffset()) + .setLength(range.getLength()) + .build(); + } +} diff --git a/models/src/main/java/info/frostfs/sdk/mappers/response/ResponseStatusMapper.java b/models/src/main/java/info/frostfs/sdk/mappers/response/ResponseStatusMapper.java index cb935dd..59c1563 100644 --- a/models/src/main/java/info/frostfs/sdk/mappers/response/ResponseStatusMapper.java +++ b/models/src/main/java/info/frostfs/sdk/mappers/response/ResponseStatusMapper.java @@ -5,6 +5,9 @@ import info.frostfs.sdk.dto.response.ResponseStatus; import info.frostfs.sdk.enums.StatusCode; import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import java.util.stream.Collectors; + +import static info.frostfs.sdk.constants.ErrorConst.FIELDS_DELIMITER_COMMA; import static info.frostfs.sdk.constants.ErrorConst.UNKNOWN_ENUM_VALUE_TEMPLATE; import static java.util.Objects.isNull; @@ -24,6 +27,10 @@ public class ResponseStatusMapper { ); } - return new ResponseStatus(statusCode, status.getMessage()); + var stringDetails = status.getDetailsList().stream() + .map(t -> t.getValue().toStringUtf8()) + .collect(Collectors.toList()); + + return new ResponseStatus(statusCode, status.getMessage(), String.join(FIELDS_DELIMITER_COMMA, stringDetails)); } } diff --git a/models/src/test/java/info/frostfs/sdk/mappers/chain/ChainMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/chain/ChainMapperTest.java deleted file mode 100644 index 7d2c4b6..0000000 --- a/models/src/test/java/info/frostfs/sdk/mappers/chain/ChainMapperTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package info.frostfs.sdk.mappers.chain; - -import com.google.protobuf.ByteString; -import frostfs.ape.Types; -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; - -public class ChainMapperTest { - - @Test - void toModels_success() { - //Given - var chain1 = Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(new byte[]{1, 2, 3, 4, 5})) - .build(); - var chain2 = Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(new byte[]{6, 7, 8, 9, 10})) - .build(); - - //When - var result = ChainMapper.toModels(List.of(chain1, chain2)); - - //Then - assertNotNull(result); - assertThat(result).hasSize(2); - assertThat(result.get(0).getRaw()).containsOnly(chain1.getRaw().toByteArray()); - assertThat(result.get(1).getRaw()).containsOnly(chain2.getRaw().toByteArray()); - } - - @Test - void toModels_null() { - //When + Then - assertNull(ChainMapper.toModels(null)); - assertNull(ChainMapper.toModels(Collections.emptyList())); - } - - @Test - void toModel_success() { - //Given - var chain = Types.Chain.newBuilder() - .setRaw(ByteString.copyFrom(new byte[]{1, 2, 3, 4, 5})) - .build(); - - //When - var result = ChainMapper.toModel(chain); - - //Then - assertNotNull(result); - assertThat(result.getRaw()).containsOnly(chain.getRaw().toByteArray()); - } - - @Test - void toModel_null() { - //When + Then - assertNull(ChainMapper.toModel(null)); - assertNull(ChainMapper.toModel(Types.Chain.getDefaultInstance())); - } -} diff --git a/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java index d4c6dd4..c390e40 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/container/ContainerMapperTest.java @@ -5,34 +5,42 @@ import frostfs.container.Types; import info.frostfs.sdk.dto.container.Container; import info.frostfs.sdk.dto.netmap.PlacementPolicy; import info.frostfs.sdk.dto.netmap.Replica; -import info.frostfs.sdk.enums.BasicAcl; -import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.dto.netmap.Version; +import info.frostfs.sdk.dto.object.OwnerId; +import info.frostfs.sdk.mappers.object.OwnerIdMapper; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; import java.util.UUID; import static info.frostfs.sdk.UuidExtension.asBytes; import static info.frostfs.sdk.UuidExtension.asUuid; +import static info.frostfs.sdk.constants.AttributeConst.DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE; import static org.junit.jupiter.api.Assertions.*; public class ContainerMapperTest { + private static final String OWNER_ID = "NVxUSpEEJzYXZZtUs18PrJTD9QZkLLNQ8S"; @Test - void toGrpcMessage_success() { + void toGrpcMessage_successFullMessage() { //Given - var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(1)}, true); - var container = new Container(BasicAcl.PUBLIC_RW, placementPolicy); + var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(3)}, true, 1); + var container = new Container(placementPolicy); + container.getAttributes().put("key1", "val1"); + container.getAttributes().put(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, "false"); + container.setVersion(new Version()); + container.setOwnerId(new OwnerId(OWNER_ID)); //When var result = ContainerMapper.toGrpcMessage(container); //Then assertNotNull(result); - assertEquals(container.getBasicAcl().value, result.getBasicAcl()); assertEquals(container.getNonce(), asUuid(result.getNonce().toByteArray())); assertEquals(container.getPlacementPolicy().isUnique(), result.getPlacementPolicy().getUnique()); + assertEquals( + container.getPlacementPolicy().getBackupFactory(), + result.getPlacementPolicy().getContainerBackupFactor() + ); assertEquals(placementPolicy.getReplicas().length, result.getPlacementPolicy().getReplicasCount()); assertEquals( container.getPlacementPolicy().getReplicas()[0].getCount(), @@ -42,6 +50,50 @@ public class ContainerMapperTest { container.getPlacementPolicy().getReplicas()[0].getSelector(), result.getPlacementPolicy().getReplicasList().get(0).getSelector() ); + + assertEquals("key1", result.getAttributes(0).getKey()); + assertEquals("val1", result.getAttributes(0).getValue()); + assertEquals(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, result.getAttributes(1).getKey()); + assertEquals("false", result.getAttributes(1).getValue()); + + assertEquals(container.getVersion().getMajor(), result.getVersion().getMajor()); + assertEquals(container.getVersion().getMinor(), result.getVersion().getMinor()); + assertEquals(container.getOwnerId().toString(), OwnerIdMapper.toModel(result.getOwnerId()).toString()); + } + + @Test + void toGrpcMessage_success() { + //Given + var placementPolicy = new PlacementPolicy(new Replica[]{new Replica(3)}, true, 1); + var container = new Container(placementPolicy); + container.getAttributes().put("key1", "val1"); + container.getAttributes().put(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, "false"); + + //When + var result = ContainerMapper.toGrpcMessage(container); + + //Then + assertNotNull(result); + assertEquals(container.getNonce(), asUuid(result.getNonce().toByteArray())); + assertEquals(container.getPlacementPolicy().isUnique(), result.getPlacementPolicy().getUnique()); + assertEquals( + container.getPlacementPolicy().getBackupFactory(), + result.getPlacementPolicy().getContainerBackupFactor() + ); + assertEquals(placementPolicy.getReplicas().length, result.getPlacementPolicy().getReplicasCount()); + assertEquals( + container.getPlacementPolicy().getReplicas()[0].getCount(), + result.getPlacementPolicy().getReplicasList().get(0).getCount() + ); + assertEquals( + container.getPlacementPolicy().getReplicas()[0].getSelector(), + result.getPlacementPolicy().getReplicasList().get(0).getSelector() + ); + + assertEquals("key1", result.getAttributes(0).getKey()); + assertEquals("val1", result.getAttributes(0).getValue()); + assertEquals(DISABLE_HOMOMORPHIC_HASHING_ATTRIBUTE, result.getAttributes(1).getKey()); + assertEquals("false", result.getAttributes(1).getValue()); } @Test @@ -50,9 +102,8 @@ public class ContainerMapperTest { assertNull(ContainerMapper.toGrpcMessage(null)); } - @ParameterizedTest - @EnumSource(value = BasicAcl.class) - void toModel_success(BasicAcl basicAcl) { + @Test + void toModel_success() { //Given var version = frostfs.refs.Types.Version.newBuilder() .setMajor(1) @@ -67,13 +118,25 @@ public class ContainerMapperTest { var placementPolicy = frostfs.netmap.Types.PlacementPolicy.newBuilder() .setUnique(true) .addReplicas(replica) + .setContainerBackupFactor(2) + .build(); + + var attribute1 = Types.Container.Attribute.newBuilder() + .setKey("key1") + .setValue("val1") + .build(); + + var attribute2 = Types.Container.Attribute.newBuilder() + .setKey("key2") + .setValue("val2") .build(); var container = Types.Container.newBuilder() - .setBasicAcl(basicAcl.value) .setNonce(ByteString.copyFrom(asBytes(UUID.randomUUID()))) .setVersion(version) .setPlacementPolicy(placementPolicy) + .addAttributes(attribute1) + .addAttributes(attribute2) .build(); //When @@ -81,9 +144,12 @@ public class ContainerMapperTest { //Then assertNotNull(result); - assertEquals(container.getBasicAcl(), result.getBasicAcl().value); assertEquals(asUuid(container.getNonce().toByteArray()), result.getNonce()); assertEquals(container.getPlacementPolicy().getUnique(), result.getPlacementPolicy().isUnique()); + assertEquals( + container.getPlacementPolicy().getContainerBackupFactor(), + result.getPlacementPolicy().getBackupFactory() + ); assertEquals(placementPolicy.getReplicasCount(), result.getPlacementPolicy().getReplicas().length); assertEquals( container.getPlacementPolicy().getReplicasList().get(0).getCount(), @@ -95,6 +161,9 @@ public class ContainerMapperTest { ); assertEquals(version.getMajor(), result.getVersion().getMajor()); assertEquals(version.getMinor(), result.getVersion().getMinor()); + + assertEquals(attribute1.getValue(), result.getAttributes().get(attribute1.getKey())); + assertEquals(attribute2.getValue(), result.getAttributes().get(attribute2.getKey())); } @Test @@ -103,15 +172,4 @@ public class ContainerMapperTest { assertNull(ContainerMapper.toModel(null)); assertNull(ContainerMapper.toModel(Types.Container.getDefaultInstance())); } - - @Test - void toModel_notValid() { - //Given - var container = Types.Container.newBuilder() - .setBasicAcl(-1) - .build(); - - //When + Then - assertThrows(ProcessFrostFSException.class, () -> ContainerMapper.toModel(container)); - } } diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java new file mode 100644 index 0000000..788d71e --- /dev/null +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/FilterMapperTest.java @@ -0,0 +1,209 @@ +package info.frostfs.sdk.mappers.netmap; + +import frostfs.netmap.Types; +import info.frostfs.sdk.dto.netmap.Filter; +import info.frostfs.sdk.enums.netmap.FilterOperation; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.MockedStatic; + +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mockStatic; + +public class FilterMapperTest { + + @ParameterizedTest + @EnumSource(value = FilterOperation.class) + void toGrpcMessages_success(FilterOperation operation) { + //Given + var filter1 = new Filter("name1", "key1", operation, "value1", null); + var filter2 = new Filter("name2", "key2", operation, "value2", null); + + //When + var result = FilterMapper.toGrpcMessages(new Filter[]{filter1, filter2}); + + //Then + assertThat(result).isNotNull().hasSize(2); + + assertEquals(filter1.getKey(), result.get(0).getKey()); + assertEquals(filter1.getName(), result.get(0).getName()); + assertEquals(filter1.getOperation().value, result.get(0).getOpValue()); + assertEquals(filter1.getValue(), result.get(0).getValue()); + assertEquals(0, result.get(0).getFiltersCount()); + + assertEquals(filter2.getKey(), result.get(1).getKey()); + assertEquals(filter2.getName(), result.get(1).getName()); + assertEquals(filter2.getOperation().value, result.get(1).getOpValue()); + assertEquals(filter2.getValue(), result.get(1).getValue()); + assertEquals(0, result.get(1).getFiltersCount()); + } + + @Test + void toGrpcMessages_null() { + //When + Then + assertEquals(Collections.emptyList(), FilterMapper.toGrpcMessages(null)); + assertEquals(Collections.emptyList(), FilterMapper.toGrpcMessages(new Filter[]{})); + } + + @ParameterizedTest + @EnumSource(value = FilterOperation.class) + void toGrpcMessage_success(FilterOperation operation) { + //Given + var filterChild = new Filter("name1", "key1", operation, "value1", null); + var filterParent = new Filter("name2", "key2", operation, "value2", new Filter[]{filterChild}); + + //When + var result = FilterMapper.toGrpcMessage(filterParent); + + //Then + assertNotNull(result); + assertEquals(filterParent.getKey(), result.getKey()); + assertEquals(filterParent.getName(), result.getName()); + assertEquals(filterParent.getOperation().value, result.getOpValue()); + assertEquals(filterParent.getValue(), result.getValue()); + assertEquals(filterParent.getFilters().length, result.getFiltersCount()); + + var filterChildGrpc = result.getFilters(0); + assertEquals(filterChild.getKey(), filterChildGrpc.getKey()); + assertEquals(filterChild.getName(), filterChildGrpc.getName()); + assertEquals(filterChild.getOperation().value, filterChildGrpc.getOpValue()); + assertEquals(filterChild.getValue(), filterChildGrpc.getValue()); + assertEquals(0, filterChildGrpc.getFiltersCount()); + } + + @Test + void toGrpcMessage_null() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> FilterMapper.toGrpcMessage(null)); + } + + @Test + void toGrpcMessage_notValidOperation() { + //Given + var filter = new Filter("name1", "key1", FilterOperation.EQ, "value1", null); + + + //When + Then + try (MockedStatic mockStatic = mockStatic(Types.Operation.class)) { + mockStatic.when(() -> Types.Operation.forNumber(filter.getOperation().value)) + .thenReturn(null); + + assertThrows(ProcessFrostFSException.class, () -> FilterMapper.toGrpcMessage(filter)); + } + } + + @ParameterizedTest + @EnumSource(value = Types.Operation.class, names = "UNRECOGNIZED", mode = EnumSource.Mode.EXCLUDE) + void toModels_success(Types.Operation operation) { + //Given + var filter1 = Types.Filter.newBuilder() + .setName("name1") + .setKey("key1") + .setOp(operation) + .setValue("value1") + .build(); + var filter2 = Types.Filter.newBuilder() + .setName("name2") + .setKey("key2") + .setOp(operation) + .setValue("value2") + .build(); + + //When + var result = FilterMapper.toModels(List.of(filter1, filter2)); + + //Then + assertThat(result).isNotNull().hasSize(2); + + + assertNotNull(result); + assertEquals(filter1.getKey(), result[0].getKey()); + assertEquals(filter1.getName(), result[0].getName()); + assertEquals(filter1.getOpValue(), result[0].getOperation().value); + assertEquals(filter1.getValue(), result[0].getValue()); + assertNull(result[0].getFilters()); + + assertEquals(filter2.getKey(), result[1].getKey()); + assertEquals(filter2.getName(), result[1].getName()); + assertEquals(filter2.getOpValue(), result[1].getOperation().value); + assertEquals(filter2.getValue(), result[1].getValue()); + assertNull(result[1].getFilters()); + } + + @Test + void toModels_null() { + //When + Then + assertNull(FilterMapper.toModels(null)); + assertNull(FilterMapper.toModels(Collections.emptyList())); + } + + @ParameterizedTest + @EnumSource(value = Types.Operation.class, names = "UNRECOGNIZED", mode = EnumSource.Mode.EXCLUDE) + void toModel_success(Types.Operation operation) { + //Given + var filterChild = Types.Filter.newBuilder() + .setName("name1") + .setKey("key1") + .setOp(operation) + .setValue("value1") + .build(); + var filterParent = Types.Filter.newBuilder() + .setName("name2") + .setKey("key2") + .setOp(operation) + .setValue("value2") + .addFilters(filterChild) + .build(); + + //When + var result = FilterMapper.toModel(filterParent); + + //Then + assertNotNull(result); + assertEquals(filterParent.getKey(), result.getKey()); + assertEquals(filterParent.getName(), result.getName()); + assertEquals(filterParent.getOpValue(), result.getOperation().value); + assertEquals(filterParent.getValue(), result.getValue()); + assertEquals(filterParent.getFiltersCount(), result.getFilters().length); + + var filterChildModel = result.getFilters()[0]; + assertEquals(filterChild.getKey(), filterChildModel.getKey()); + assertEquals(filterChild.getName(), filterChildModel.getName()); + assertEquals(filterChild.getOpValue(), filterChildModel.getOperation().value); + assertEquals(filterChild.getValue(), filterChildModel.getValue()); + assertNull(filterChildModel.getFilters()); + } + + @Test + void toModel_null() { + //When + Then + assertNull(FilterMapper.toModel(null)); + assertNull(FilterMapper.toModel(Types.Filter.getDefaultInstance())); + } + + @Test + void toModel_notValidScheme() { + //Given + var filter = Types.Filter.newBuilder() + .setName("name1") + .setKey("key1") + .setOp(Types.Operation.EQ) + .setValue("value1") + .build(); + + //When + Then + try (MockedStatic mockStatic = mockStatic(FilterOperation.class)) { + mockStatic.when(() -> FilterOperation.get(Types.Operation.EQ.getNumber())) + .thenReturn(null); + + assertThrows(ProcessFrostFSException.class, () -> FilterMapper.toModel(filter)); + } + } +} diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java index e2b6c29..dd2dc1f 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/NodeInfoMapperTest.java @@ -3,7 +3,7 @@ package info.frostfs.sdk.mappers.netmap; import com.google.protobuf.ByteString; import frostfs.netmap.Service; import frostfs.netmap.Types; -import info.frostfs.sdk.enums.NodeState; +import info.frostfs.sdk.enums.netmap.NodeState; import info.frostfs.sdk.exceptions.ProcessFrostFSException; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapperTest.java index 903d574..793ad92 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/PlacementPolicyMapperTest.java @@ -15,7 +15,7 @@ public class PlacementPolicyMapperTest { var replica1 = new Replica(1, "test1"); var replica2 = new Replica(2, "test2"); - var placementPolicy = new PlacementPolicy(new Replica[]{replica1, replica2}, true); + var placementPolicy = new PlacementPolicy(new Replica[]{replica1, replica2}, true, 1); //When var result = PlacementPolicyMapper.toGrpcMessage(placementPolicy); @@ -23,6 +23,7 @@ public class PlacementPolicyMapperTest { //Then assertNotNull(result); assertEquals(placementPolicy.isUnique(), result.getUnique()); + assertEquals(placementPolicy.getBackupFactory(), result.getContainerBackupFactor()); assertEquals(placementPolicy.getReplicas().length, result.getReplicasCount()); assertEquals(replica1.getCount(), result.getReplicas(0).getCount()); assertEquals(replica1.getSelector(), result.getReplicas(0).getSelector()); @@ -53,6 +54,7 @@ public class PlacementPolicyMapperTest { .setUnique(true) .addReplicas(replica1) .addReplicas(replica2) + .setContainerBackupFactor(1) .build(); //When @@ -61,6 +63,7 @@ public class PlacementPolicyMapperTest { //Then assertNotNull(result); assertEquals(placementPolicy.getUnique(), result.isUnique()); + assertEquals(placementPolicy.getContainerBackupFactor(), result.getBackupFactory()); assertEquals(placementPolicy.getReplicasCount(), result.getReplicas().length); assertEquals(replica1.getCount(), result.getReplicas()[0].getCount()); assertEquals(replica1.getSelector(), result.getReplicas()[0].getSelector()); diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/ReplicaMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/ReplicaMapperTest.java index 973e74c..ca486dc 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/netmap/ReplicaMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/ReplicaMapperTest.java @@ -2,6 +2,7 @@ package info.frostfs.sdk.mappers.netmap; import frostfs.netmap.Types; import info.frostfs.sdk.dto.netmap.Replica; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -25,7 +26,7 @@ public class ReplicaMapperTest { @Test void toGrpcMessage_null() { //When + Then - assertNull(ReplicaMapper.toGrpcMessage(null)); + assertThrows(ValidationFrostFSException.class, () -> ReplicaMapper.toGrpcMessage(null)); } @Test diff --git a/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java new file mode 100644 index 0000000..6d46553 --- /dev/null +++ b/models/src/test/java/info/frostfs/sdk/mappers/netmap/SelectorMapperTest.java @@ -0,0 +1,191 @@ +package info.frostfs.sdk.mappers.netmap; + +import frostfs.netmap.Types; +import info.frostfs.sdk.dto.netmap.Selector; +import info.frostfs.sdk.enums.netmap.SelectorClause; +import info.frostfs.sdk.exceptions.ProcessFrostFSException; +import info.frostfs.sdk.exceptions.ValidationFrostFSException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.MockedStatic; + +import java.util.Collections; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mockStatic; + +public class SelectorMapperTest { + + @ParameterizedTest + @EnumSource(value = SelectorClause.class) + void toGrpcMessages_success(SelectorClause clause) { + //Given + var selector1 = new Selector("name1", 1, clause, "attribute1", "filter1"); + var selector2 = new Selector("name2", 2, clause, "attribute2", "filter2"); + + //When + var result = SelectorMapper.toGrpcMessages(new Selector[]{selector1, selector2}); + + //Then + assertThat(result).isNotNull().hasSize(2); + + assertEquals(selector1.getName(), result.get(0).getName()); + assertEquals(selector1.getCount(), result.get(0).getCount()); + assertEquals(selector1.getClause().value, result.get(0).getClauseValue()); + assertEquals(selector1.getAttribute(), result.get(0).getAttribute()); + assertEquals(selector1.getFilter(), result.get(0).getFilter()); + + assertEquals(selector2.getName(), result.get(1).getName()); + assertEquals(selector2.getCount(), result.get(1).getCount()); + assertEquals(selector2.getClause().value, result.get(1).getClauseValue()); + assertEquals(selector2.getAttribute(), result.get(1).getAttribute()); + assertEquals(selector2.getFilter(), result.get(1).getFilter()); + } + + @Test + void toGrpcMessages_null() { + //When + Then + assertEquals(Collections.emptyList(), SelectorMapper.toGrpcMessages(null)); + assertEquals(Collections.emptyList(), SelectorMapper.toGrpcMessages(new Selector[]{})); + } + + @ParameterizedTest + @EnumSource(value = SelectorClause.class) + void toGrpcMessage_success(SelectorClause clause) { + //Given + var selector = new Selector("name", 1, clause, "attribute", "filter"); + + //When + var result = SelectorMapper.toGrpcMessage(selector); + + //Then + assertNotNull(result); + assertEquals(selector.getName(), result.getName()); + assertEquals(selector.getCount(), result.getCount()); + assertEquals(selector.getClause().value, result.getClauseValue()); + assertEquals(selector.getAttribute(), result.getAttribute()); + assertEquals(selector.getFilter(), result.getFilter()); + } + + @Test + void toGrpcMessage_null() { + //When + Then + assertThrows(ValidationFrostFSException.class, () -> SelectorMapper.toGrpcMessage(null)); + } + + @Test + void toGrpcMessage_notValidOperation() { + //Given + var selector = new Selector("name", 1, SelectorClause.SAME, "attribute", "filter"); + + + //When + Then + try (MockedStatic mockStatic = mockStatic(Types.Clause.class)) { + mockStatic.when(() -> Types.Clause.forNumber(selector.getClause().value)) + .thenReturn(null); + + assertThrows(ProcessFrostFSException.class, () -> SelectorMapper.toGrpcMessage(selector)); + } + } + + @ParameterizedTest + @EnumSource(value = Types.Clause.class, names = "UNRECOGNIZED", mode = EnumSource.Mode.EXCLUDE) + void toModels_success(Types.Clause clause) { + //Given + var selector1 = Types.Selector.newBuilder() + .setName("name1") + .setCount(1) + .setClause(clause) + .setAttribute("attribute1") + .setFilter("filter1") + .build(); + var selector2 = Types.Selector.newBuilder() + .setName("name2") + .setCount(2) + .setClause(clause) + .setAttribute("attribute2") + .setFilter("filter2") + .build(); + + //When + var result = SelectorMapper.toModels(List.of(selector1, selector2)); + + //Then + assertThat(result).isNotNull().hasSize(2); + + + assertNotNull(result); + assertEquals(selector1.getName(), result[0].getName()); + assertEquals(selector1.getCount(), result[0].getCount()); + assertEquals(selector1.getClauseValue(), result[0].getClause().value); + assertEquals(selector1.getAttribute(), result[0].getAttribute()); + assertEquals(selector1.getFilter(), result[0].getFilter()); + + assertEquals(selector2.getName(), result[1].getName()); + assertEquals(selector2.getCount(), result[1].getCount()); + assertEquals(selector2.getClauseValue(), result[1].getClause().value); + assertEquals(selector2.getAttribute(), result[1].getAttribute()); + assertEquals(selector2.getFilter(), result[1].getFilter()); + } + + @Test + void toModels_null() { + //When + Then + assertNull(SelectorMapper.toModels(null)); + assertNull(SelectorMapper.toModels(Collections.emptyList())); + } + + @ParameterizedTest + @EnumSource(value = Types.Clause.class, names = "UNRECOGNIZED", mode = EnumSource.Mode.EXCLUDE) + void toModel_success(Types.Clause clause) { + //Given + var selector = Types.Selector.newBuilder() + .setName("name") + .setCount(1) + .setClause(clause) + .setAttribute("attribute") + .setFilter("filter") + .build(); + + //When + var result = SelectorMapper.toModel(selector); + + //Then + assertNotNull(result); + assertEquals(selector.getName(), result.getName()); + assertEquals(selector.getCount(), result.getCount()); + assertEquals(selector.getClauseValue(), result.getClause().value); + assertEquals(selector.getAttribute(), result.getAttribute()); + assertEquals(selector.getFilter(), result.getFilter()); + } + + @Test + void toModel_null() { + //When + Then + assertNull(SelectorMapper.toModel(null)); + assertNull(SelectorMapper.toModel(Types.Selector.getDefaultInstance())); + } + + @Test + void toModel_notValidScheme() { + //Given + var selector = Types.Selector.newBuilder() + .setName("name") + .setCount(1) + .setClause(Types.Clause.SAME) + .setAttribute("attribute") + .setFilter("filter") + .build(); + + //When + Then + try (MockedStatic mockStatic = mockStatic(SelectorClause.class)) { + mockStatic.when(() -> SelectorClause.get(Types.Clause.SAME.getNumber())) + .thenReturn(null); + + assertThrows(ProcessFrostFSException.class, () -> SelectorMapper.toModel(selector)); + } + } +} diff --git a/models/src/test/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapperTest.java index 4ae2902..4eb5501 100644 --- a/models/src/test/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapperTest.java +++ b/models/src/test/java/info/frostfs/sdk/mappers/object/ObjectAttributeMapperTest.java @@ -2,12 +2,42 @@ package info.frostfs.sdk.mappers.object; import frostfs.object.Types; import info.frostfs.sdk.dto.object.ObjectAttribute; +import org.apache.commons.collections4.CollectionUtils; import org.junit.jupiter.api.Test; +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; public class ObjectAttributeMapperTest { + @Test + void toGrpcMessages_success() { + //Given + var objectAttribute1 = new ObjectAttribute("key1", "value1"); + var objectAttribute2 = new ObjectAttribute("key2", "value2"); + + //When + var result = ObjectAttributeMapper.toGrpcMessages(Arrays.asList(objectAttribute1, objectAttribute2)); + + //Then + assertNotNull(result); + assertThat(result).isNotNull().hasSize(2); + assertEquals(objectAttribute1.getKey(), result.get(0).getKey()); + assertEquals(objectAttribute1.getValue(), result.get(0).getValue()); + assertEquals(objectAttribute2.getKey(), result.get(1).getKey()); + assertEquals(objectAttribute2.getValue(), result.get(1).getValue()); + } + + @Test + void toGrpcMessages_null() { + //When + Then + assertTrue(CollectionUtils.isEmpty(ObjectAttributeMapper.toGrpcMessages(null))); + assertTrue(CollectionUtils.isEmpty(ObjectAttributeMapper.toGrpcMessages(Collections.emptyList()))); + } + @Test void toGrpcMessage_success() { //Given diff --git a/models/src/test/java/info/frostfs/sdk/mappers/object/SplitInfoMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/object/SplitInfoMapperTest.java new file mode 100644 index 0000000..3e9ca2b --- /dev/null +++ b/models/src/test/java/info/frostfs/sdk/mappers/object/SplitInfoMapperTest.java @@ -0,0 +1,61 @@ +package info.frostfs.sdk.mappers.object; + +import com.google.protobuf.ByteString; +import frostfs.object.Types; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.SplitId; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +public class SplitInfoMapperTest { + @Test + void toModel_successLastPart() { + //Given + var splitId = new SplitId(); + var objectId = new ObjectId("85orCLKSu3X1jGiTFmwmTUsBU88RBARNwuRwrEy5pyww"); + var splitInfo = Types.SplitInfo.newBuilder() + .setSplitId(ByteString.copyFrom(splitId.toBinary())) + .setLastPart(ObjectIdMapper.toGrpcMessage(objectId)) + .build(); + + //When + var result = SplitInfoMapper.toModel(splitInfo); + + //Then + assertNotNull(result); + assertNull(result.getLink()); + assertThat(result.getSplitInfo()).isEqualTo(splitInfo); + assertThat(result.getSplitId().toBinary()).containsExactly(splitId.toBinary()); + assertEquals(objectId.getValue(), result.getLastPart().getValue()); + } + + @Test + void toModel_successLink() { + //Given + var splitId = new SplitId(); + var objectId = new ObjectId("85orCLKSu3X1jGiTFmwmTUsBU88RBARNwuRwrEy5pyww"); + var splitInfo = Types.SplitInfo.newBuilder() + .setSplitId(ByteString.copyFrom(splitId.toBinary())) + .setLink(ObjectIdMapper.toGrpcMessage(objectId)) + .build(); + + //When + var result = SplitInfoMapper.toModel(splitInfo); + + //Then + assertNotNull(result); + assertNull(result.getLastPart()); + assertThat(result.getSplitInfo()).isEqualTo(splitInfo); + assertThat(result.getSplitId().toBinary()).containsExactly(splitId.toBinary()); + assertEquals(objectId.getValue(), result.getLink().getValue()); + } + + @Test + void toModel_null() { + //When + Then + assertNull(SplitInfoMapper.toModel(null)); + assertNull(SplitInfoMapper.toModel(Types.SplitInfo.getDefaultInstance())); + } +} diff --git a/models/src/test/java/info/frostfs/sdk/mappers/object/patch/AddressMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/object/patch/AddressMapperTest.java new file mode 100644 index 0000000..b53ff87 --- /dev/null +++ b/models/src/test/java/info/frostfs/sdk/mappers/object/patch/AddressMapperTest.java @@ -0,0 +1,39 @@ +package info.frostfs.sdk.mappers.object.patch; + +import info.frostfs.sdk.dto.container.ContainerId; +import info.frostfs.sdk.dto.object.ObjectId; +import info.frostfs.sdk.dto.object.patch.Address; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +public class AddressMapperTest { + + @Test + void toGrpcMessage_success() { + //Given + var objectId = new ObjectId("85orCLKSu3X1jGiTFmwmTUsBU88RBARNwuRwrEy5pyww"); + var containerId = new ContainerId("EQGx2QeYHJb53uRwYGzcQaW191sZpdNrjutk6veUSV2R"); + var address = new Address(objectId, containerId); + + //When + var result = AddressMapper.toGrpcMessage(address); + + //Then + assertNotNull(result); + assertEquals( + address.getContainerId().getValue(), + new ContainerId(result.getContainerId().getValue().toByteArray()).getValue() + ); + assertEquals( + address.getObjectId().getValue(), + new ObjectId(result.getObjectId().getValue().toByteArray()).getValue() + ); + } + + @Test + void toGrpcMessage_null() { + //When + Then + assertNull(AddressMapper.toGrpcMessage(null)); + } +} diff --git a/models/src/test/java/info/frostfs/sdk/mappers/object/patch/RangeMapperTest.java b/models/src/test/java/info/frostfs/sdk/mappers/object/patch/RangeMapperTest.java new file mode 100644 index 0000000..f78a482 --- /dev/null +++ b/models/src/test/java/info/frostfs/sdk/mappers/object/patch/RangeMapperTest.java @@ -0,0 +1,59 @@ +package info.frostfs.sdk.mappers.object.patch; + +import info.frostfs.sdk.dto.object.patch.Range; +import org.apache.commons.collections4.CollectionUtils; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +public class RangeMapperTest { + + @Test + void toGrpcMessages_success() { + //Given + var range1 = new Range(1, 10); + var range2 = new Range(2, 20); + + //When + var result = RangeMapper.toGrpcMessages(Arrays.asList(range1, range2)); + + //Then + assertNotNull(result); + assertThat(result).isNotNull().hasSize(2); + assertEquals(range1.getOffset(), result.get(0).getOffset()); + assertEquals(range1.getLength(), result.get(0).getLength()); + assertEquals(range2.getOffset(), result.get(1).getOffset()); + assertEquals(range2.getLength(), result.get(1).getLength()); + } + + @Test + void toGrpcMessages_null() { + //When + Then + assertTrue(CollectionUtils.isEmpty(RangeMapper.toGrpcMessages(null))); + assertTrue(CollectionUtils.isEmpty(RangeMapper.toGrpcMessages(Collections.emptyList()))); + } + + @Test + void toGrpcMessage_success() { + //Given + var range = new Range(1, 10); + + //When + var result = RangeMapper.toGrpcMessage(range); + + //Then + assertNotNull(result); + assertEquals(range.getOffset(), result.getOffset()); + assertEquals(range.getLength(), result.getLength()); + } + + @Test + void toGrpcMessage_null() { + //When + Then + assertNull(RangeMapper.toGrpcMessage(null)); + } +} diff --git a/pom.xml b/pom.xml index bd46c8b..d8de384 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} pom client @@ -17,6 +17,8 @@ + 0.12.0 + 11 11 UTF-8 @@ -26,6 +28,7 @@ 3.26.3 1.18.34 3.23.0 + 3.23.0 @@ -39,6 +42,11 @@ commons-lang3 3.14.0 + + commons-codec + commons-codec + 1.18.0 + org.projectlombok lombok @@ -46,6 +54,11 @@ provided true + + io.neow3j + contract + ${neow3j.version} + org.junit.jupiter junit-jupiter @@ -70,6 +83,11 @@ ${mockito.version} test + + org.yaml + snakeyaml + 2.4 + @@ -126,6 +144,46 @@ + + org.codehaus.mojo + flatten-maven-plugin + 1.0.0 + + true + + + + flatten + process-resources + + flatten + + + + flatten.clean + clean + + clean + + + + - \ No newline at end of file + + + TrueCloudLab + https://git.frostfs.info/api/packages/TrueCloudLab/maven + + + + + TrueCloudLab + https://git.frostfs.info/api/packages/TrueCloudLab/maven + + + TrueCloudLab + https://git.frostfs.info/api/packages/TrueCloudLab/maven + + + diff --git a/protos/pom.xml b/protos/pom.xml index 5381838..f5ea90a 100644 --- a/protos/pom.xml +++ b/protos/pom.xml @@ -6,7 +6,7 @@ info.frostfs.sdk frostfs-sdk-java - 0.1.0 + ${revision} protos