[#3] Added generate proto script create container method

Signed-off-by: Ilyas Niyazov <i.niyazov@yadro.com>
This commit is contained in:
Ilyas Niyazov 2025-03-25 11:13:48 +03:00
parent 19282f13cc
commit 297e107b10
52 changed files with 1380 additions and 74 deletions

View file

@ -0,0 +1,9 @@
from frostfs_sdk.models.dto.container import Container, ContainerId
from frostfs_sdk.models.dto.filter import Filter
from frostfs_sdk.models.dto.placement_policy import PlacementPolicy
from frostfs_sdk.models.dto.replica import Replica
from frostfs_sdk.models.dto.selector import Selector
from frostfs_sdk.models.enums.basic_acl import BasicAcl
from frostfs_sdk.models.enums.filter_operation import FilterOperation
from frostfs_sdk.models.enums.selector_clause import SelectorClause

View file

@ -0,0 +1,29 @@
from dataclasses import dataclass, field
from typing import Dict, Optional
import uuid
from frostfs_sdk.models.dto.owner_id import OwnerId
from frostfs_sdk.models.dto.version import Version
from frostfs_sdk.models.enums.basic_acl import BasicAcl
from frostfs_sdk.models.dto.placement_policy import PlacementPolicy
@dataclass
class Container:
# basicAcl: BasicAcl # TODO: will remove it?
placementPolicy: PlacementPolicy
nonce: uuid.UUID = field(default_factory=uuid.uuid4)
version: Optional[Version] = None
owner_id: Optional[OwnerId] = None
attributes: Dict[str, str] = field(default_factory=dict)
def __init__(self, placementPolicy: PlacementPolicy):
self.nonce = uuid.uuid4()
self.placementPolicy = placementPolicy
self.attributes = {}
@dataclass
class ContainerId:
value: str

View file

@ -0,0 +1,14 @@
from dataclasses import dataclass
from frostfs_sdk.models.enums.filter_operation import FilterOperation
@dataclass(frozen=True)
class Filter:
"""
Data Transfer Object for Filter configuration
"""
name: str
key: str
operation: FilterOperation
value: str

View file

@ -0,0 +1,47 @@
from dataclasses import dataclass
from typing import Optional
from frostfs_sdk.models.dto.version import Version, DEFAULT_MAJOR_VERSION, DEFAULT_MINOR_VERSION
class MetaHeader:
def __init__(self, version=None, epoch: int = 0, ttl: int = 2):
"""
Initializes a MetaHeader object.
:param version: A Version object representing the version.
:param epoch: An integer representing the epoch (default: 0).
:param ttl: An integer representing the time-to-live (default: 2).
"""
self._version = None
self._epoch = None
self._ttl = None
# Set default values if not provided
self.set_version(version or Version(DEFAULT_MAJOR_VERSION, DEFAULT_MINOR_VERSION))
self.set_epoch(epoch)
self.set_ttl(ttl)
def get_version(self):
return self._version
def get_epoch(self):
return self._epoch
def get_ttl(self):
return self._ttl
def set_version(self, version):
if version is None:
raise ValueError(f"Input parameter is missing: {Version.__name__}")
self._version = version
def set_epoch(self, epoch: int):
if epoch < 0:
raise ValueError("Epoch must be greater than or equal to zero")
self._epoch = epoch
def set_ttl(self, ttl: int):
if ttl <= 0:
raise ValueError("TTL must be greater than zero")
self._ttl = ttl

View file

@ -0,0 +1,21 @@
from base58 import b58decode
from dataclasses import dataclass
@dataclass(frozen=True)
class OwnerId:
value: str
def __post_init__(self):
if not self.value or self.value.strip() == "":
raise ValueError(f"{self.__class__.__name__} value is not present")
def to_hash(self) -> bytes:
"""Decodes the Base58-encoded value into a byte array."""
try:
return b58decode(self.value)
except Exception as e:
raise ValueError(f"Failed to decode Base58 value: {self.value}") from e
def __str__(self) -> str:
return self.value

View file

@ -0,0 +1,14 @@
from dataclasses import dataclass
from typing import List
from frostfs_sdk.models.dto.replica import Replica
from frostfs_sdk.models.dto.selector import Selector
from frostfs_sdk.models.dto.filter import Filter
@dataclass
class PlacementPolicy:
replicas: List[Replica]
unique: bool
backup_factory: int
filters: List[Filter] = None
selectors: List[Selector] = None

View file

@ -0,0 +1,12 @@
from dataclasses import dataclass, field
EMPTY_STRING = ""
@dataclass
class Replica:
count: int
selector: str = field(default=EMPTY_STRING)
def __post_init__(self):
self.selector = self.selector if self.selector else EMPTY_STRING

View file

@ -0,0 +1,15 @@
from dataclasses import dataclass
from frostfs_sdk.models.enums.selector_clause import SelectorClause
@dataclass(frozen=True)
class Selector:
"""
Data Transfer Object for Selector configuration
"""
name: str
count: int
clause: SelectorClause
attribute: str
filter: str

View file

@ -0,0 +1,19 @@
from dataclasses import dataclass
DEFAULT_MAJOR_VERSION = 2
DEFAULT_MINOR_VERSION = 13
@dataclass(frozen=True)
class Version:
major: int = DEFAULT_MAJOR_VERSION
minor: int = DEFAULT_MINOR_VERSION
def __str__(self) -> str:
return f"v{self.major}.{self.minor}"
def is_supported(self, other):
if not isinstance(other, Version):
return False
return self.major == other.major

View file

@ -0,0 +1,7 @@
from enum import Enum
class BasicAcl(Enum):
PRIVATE = 0x1C8C8CCC
PUBLIC_RO = 0x1FBF8CFF
PUBLIC_RW = 0x1FBFBFFF
PUBLIC_APPEND = 0x1FBF9FFF

View file

@ -0,0 +1,42 @@
class FilterOperation:
"""
Enum for filter operations with integer value mapping
"""
OPERATION_UNSPECIFIED = 0
EQ = 1
NE = 2
GT = 3
GE = 4
LT = 5
LE = 6
OR = 7
AND = 8
NOT = 9
LIKE = 10
_value_map = {
0: OPERATION_UNSPECIFIED,
1: EQ,
2: NE,
3: GT,
4: GE,
5: LT,
6: LE,
7: OR,
8: AND,
9: NOT,
10: LIKE
}
@classmethod
def get(cls, value: int) -> 'FilterOperation':
"""
Get enum instance by integer value
Args:
value: Integer value of the operation
Returns:
Corresponding FilterOperation instance
"""
return cls._value_map.get(value)

View file

@ -0,0 +1,14 @@
from enum import Enum, unique
@unique
class SelectorClause(Enum):
CLAUSE_UNSPECIFIED = 0
SAME = 1
DISTINCT = 2
@classmethod
def get(cls, value: int):
try:
return cls(value)
except ValueError:
raise KeyError(f"Unknown enum value: {value}")

View file

@ -0,0 +1,77 @@
from typing import Optional
import grpc
from frostfs_sdk.client.models.client_environment import ClientEnvironment
from frostfs_sdk.client.services.context_accessor import ContextAccessor
from frostfs_sdk.models.mappers.placement_policy_mapper import PlacementPolicyMapper
from frostfs_sdk.models.mappers.owner_id_mapper import OwnerIdMapper
from frostfs_sdk.models.mappers.version_mapper import VersionMapper
from frostfs_sdk.models.mappers.uuid_extension import UuidExtension
from frostfs_sdk.models.dto.container import Container
from frostfs_sdk.protos.models.container import types_pb2 as types_pb2_container
class ContainerMapper:
@staticmethod
def to_grpc_message(container: Container, client_context: ClientEnvironment) -> Optional[types_pb2_container.Container]:
"""
Converts Container DTO to gRPC message
Args:
container: Container DTO object
Returns:
gRPC Container message builder
"""
if not container:
return None
attributes = [
types_pb2_container.Container.Attribute(key=k, value=v)
for k, v in container.attributes.items()
]
if container.owner_id:
owner_id = OwnerIdMapper.to_grpc_message(container.owner_id)
else:
owner_id = OwnerIdMapper.to_grpc_message(client_context.owner_id)
if container.version:
version = VersionMapper.to_grpc_message(container.version)
else:
version = VersionMapper.to_grpc_message(client_context.version)
grpc_container = types_pb2_container.Container(
nonce=container.nonce.bytes,
placement_policy=PlacementPolicyMapper.to_grpc_message(container.placementPolicy),
owner_id=owner_id,
version=version,
attributes=attributes
)
return grpc_container
@staticmethod
def to_model(container_grpc: types_pb2_container.Container) -> Optional[Container]:
"""
Converts gRPC message to Container DTO
Args:
container_grpc: gRPC Container message
Returns:
Container DTO object
"""
if not container_grpc or container_grpc.ByteSize() == 0:
return None
attributes = {attr.key: attr.value for attr in container_grpc.attributes}
return Container(
nonce=UuidExtension.to_uuid(container_grpc.nonce),
placement_policy=PlacementPolicyMapper.to_model(container_grpc.placement_policy),
version=VersionMapper.to_model(container_grpc.version),
owner_id=OwnerIdMapper.to_model(container_grpc.owner_id),
attributes=attributes
)

View file

@ -0,0 +1,60 @@
from typing import List, Optional
from frostfs_sdk.models.enums.filter_operation import FilterOperation
from frostfs_sdk.models.dto.filter import Filter
from frostfs_sdk.protos.models.netmap import types_pb2 as types_pb2_netmap
class FilterMapper:
@staticmethod
def to_grpc_messages(filters: List[Filter]) -> List[types_pb2_netmap.Filter]:
"""
Converts list of Filter DTOs to gRPC messages with nested conversion
"""
if not filters:
return []
return [FilterMapper.to_grpc_message(f) for f in filters]
@staticmethod
def to_grpc_message(filter_dto: Filter) -> types_pb2_netmap.Filter:
"""
Converts Filter DTO to gRPC message with nested filters
"""
operation = types_pb2_netmap.Filter.Operation.Value(filter_dto.operation.value)
return types_pb2_netmap.Filter(
name=filter_dto.name,
key=filter_dto.key,
op=operation,
value=filter_dto.value,
filters=FilterMapper.to_grpc_messages(filter_dto.filters)
)
@staticmethod
def to_models(filters_grpc: List[types_pb2_netmap.Filter]) -> Optional[List[Filter]]:
"""
Converts gRPC messages to Filter DTOs with nested conversion
"""
if not filters_grpc:
return None
return [FilterMapper.to_model(f) for f in filters_grpc]
@staticmethod
def to_model(filter_grpc: types_pb2_netmap.Filter) -> Optional[Filter]:
"""
Converts gRPC message to Filter DTO with nested filters
"""
if not filter_grpc or filter_grpc.ByteSize() == 0:
return None
operation = FilterOperation(filter_grpc.op)
return Filter(
name=filter_grpc.name,
key=filter_grpc.key,
operation=operation,
value=filter_grpc.value,
filters=FilterMapper.to_models(filter_grpc.filters)
)

View file

@ -0,0 +1,26 @@
from frostfs_sdk.models.mappers.version_mapper import VersionMapper
from frostfs_sdk.models.dto.meta_header import MetaHeader
from frostfs_sdk.protos.models.session import types_pb2 as types_pb2_session
class MetaHeaderMapper:
"""
Maps a MetaHeader object to a Protobuf RequestMetaHeader object.
"""
@staticmethod
def to_grpc_message(meta_header: MetaHeader):
"""
Converts a MetaHeader object to a Protobuf RequestMetaHeader object.
:param meta_header: A MetaHeader object.
:return: A Protobuf RequestMetaHeader object.
:raises ValueError: If the input meta_header is None.
"""
if meta_header is None:
raise ValueError(f"Input parameter is missing: {MetaHeader.__name__}")
return types_pb2_session.RequestMetaHeader(
version=VersionMapper.to_grpc_message(meta_header.get_version()),
epoch=meta_header.get_epoch(),
ttl=meta_header.get_ttl()
)

View file

@ -0,0 +1,32 @@
from base58 import b58encode
from frostfs_sdk.models.dto.owner_id import OwnerId
from frostfs_sdk.protos.models.refs import types_pb2 as types_pb2_refs
class OwnerIdMapper:
@staticmethod
def to_grpc_message(owner_id: OwnerId) -> types_pb2_refs.OwnerID:
"""
Converts OwnerId DTO to gRPC message
"""
if not owner_id:
return None
return types_pb2_refs.OwnerID(
value=owner_id.to_hash(),
)
@staticmethod
def to_model(owner_id_grpc: types_pb2_refs.OwnerID) -> OwnerId:
"""
Converts gRPC message to OwnerId DTO
"""
if not owner_id_grpc or owner_id_grpc.ByteSize() == 0:
return None
try:
return OwnerId(
value=b58encode(owner_id_grpc.value).decode('utf-8')
)
except Exception as e:
raise ValueError(f"Failed to encode Base58 value: {owner_id_grpc.value}")

View file

@ -0,0 +1,53 @@
from typing import Optional
from frostfs_sdk.models.mappers.filter_mapper import FilterMapper
from frostfs_sdk.models.mappers.selector_mapper import SelectorMapper
from frostfs_sdk.models.mappers.replica_mapper import ReplicaMapper
from frostfs_sdk.models.dto.placement_policy import PlacementPolicy
from frostfs_sdk.protos.models.netmap import types_pb2 as types_pb2_netmap
class PlacementPolicyMapper:
@staticmethod
def to_grpc_message(policy: PlacementPolicy) -> Optional[types_pb2_netmap.PlacementPolicy]:
"""
Converts PlacementPolicy DTO to gRPC message
Args:
policy: PlacementPolicy DTO object
Returns:
gRPC PlacementPolicy message
"""
if not policy:
return None
return types_pb2_netmap.PlacementPolicy(
unique=policy.unique,
container_backup_factor=policy.backup_factory,
filters=FilterMapper.to_grpc_messages(policy.filters),
selectors=SelectorMapper.to_grpc_messages(policy.selectors),
replicas=ReplicaMapper.to_grpc_messages(policy.replicas)
)
@staticmethod
def to_model(policy_grpc: types_pb2_netmap.PlacementPolicy) -> Optional[PlacementPolicy]:
"""
Converts gRPC message to PlacementPolicy DTO
Args:
policy_grpc: gRPC PlacementPolicy message
Returns:
PlacementPolicy DTO object
"""
if not policy_grpc or policy_grpc.ByteSize() == 0:
return None
return PlacementPolicy(
replicas=ReplicaMapper.to_models(policy_grpc.replicas),
unique=policy_grpc.unique,
backup_factory=policy_grpc.container_backup_factor,
filters=FilterMapper.to_models(policy_grpc.filters),
selectors=SelectorMapper.to_models(policy_grpc.selectors)
)

View file

@ -0,0 +1,56 @@
from typing import List, Optional
from frostfs_sdk.models.dto.replica import Replica
from frostfs_sdk.protos.models.netmap import types_pb2 as types_pb2_netmap
class ReplicaMapper:
@staticmethod
def to_grpc_messages(replicas: List[Replica]) -> Optional[List[types_pb2_netmap.Replica]]:
if not replicas:
return None
return [ReplicaMapper.to_grpc_message(selector) for selector in replicas]
@staticmethod
def to_grpc_message(replica: Replica) -> Optional[types_pb2_netmap.Replica]:
"""
Converts Replica DTO to gRPC message
Args:
replice: Replica DTO object
Returns:
gRPC Replica message
"""
if not replica:
return None
return types_pb2_netmap.Replica(
count=replica.count,
selector=replica.selector
)
@staticmethod
def to_models(grpc_replicas: List[types_pb2_netmap.Replica]) -> Optional[List[Replica]]:
if not grpc_replicas:
return None
return [ReplicaMapper.to_model(grpc_replica) for grpc_replica in grpc_replicas]
@staticmethod
def to_model(grpc_replica: types_pb2_netmap.Replica) -> Optional[Replica]:
"""
Converts gRPC message to Replica DTO
Args:
grpc_replica: gRPC Replica message
Returns:
Replica DTO object
"""
if not grpc_replica or grpc_replica.ByteSize() == 0:
return None
return Replica(
count=grpc_replica.count,
selectors=grpc_replica.selector
)

View file

@ -0,0 +1,73 @@
from typing import List, Optional
from frostfs_sdk.models.dto.selector import Selector
from frostfs_sdk.models.enums.selector_clause import SelectorClause
from frostfs_sdk.protos.models.netmap import types_pb2 as types_pb2_netmap
class SelectorMapper:
@staticmethod
def to_grpc_messages(selectors: List[Selector]) -> Optional[List[types_pb2_netmap.Selector]]:
if not selectors:
return None
return [SelectorMapper.to_grpc_message(selector) for selector in selectors]
@staticmethod
def to_grpc_message(selector: Selector) -> Optional[types_pb2_netmap.Selector]:
"""
Converts Selector DTO to gRPC message
Args:
selector: Selector DTO object
Returns:
gRPC Selector message
"""
if not selector:
return None
clause_grpc = types_pb2_netmap.Clause(selector.clause.value)
if clause_grpc is None:
raise ValueError(f"Unknown enum value: {selector.clause.name} for {types_pb2_netmap.Clause.__name__}")
return types_pb2_netmap.Selector(
name=selector.name,
count=selector.count,
clause=clause_grpc,
attribute=selector.attribute,
filter=selector.filter
)
@staticmethod
def to_models(grpc_selectors: List[types_pb2_netmap.Selector]) -> Optional[List[Selector]]:
if not grpc_selectors:
return None
return [SelectorMapper.to_model(grpc_selector) for grpc_selector in grpc_selectors]
@staticmethod
def to_model(selector_grpc: types_pb2_netmap.Selector) -> Optional[Selector]:
"""
Converts gRPC message to Selector DTO
Args:
selector_grpc: gRPC Selector message
Returns:
Selector DTO object
"""
if not selector_grpc or selector_grpc.ByteSize() == 0:
return None
clause = SelectorClause.get(selector_grpc.clause)
if clause is None:
raise ValueError(f"Unknown enum value: {selector_grpc.clause} for {SelectorClause.__name__}")
return Selector(
name=selector_grpc.name,
count=selector_grpc.count,
clause=clause,
attribute=selector_grpc.attribute,
filter=selector_grpc.filter
)

View file

@ -0,0 +1,44 @@
from google.protobuf.message import DecodeError
from frostfs_sdk.protos.models.session import types_pb2 as types_pb2_session
class SessionMapper:
@staticmethod
def serialize(token: types_pb2_session.SessionToken) -> bytes:
"""
Serializes a SessionToken object into a byte array.
:param token: A SessionToken Protobuf object.
:return: A byte array representing the serialized SessionToken.
:raises ValueError: If the input token is None.
:raises Exception: If serialization fails.
"""
if token is None:
raise ValueError(f"Input parameter is missing: {types_pb2_session.SessionToken.__name__}")
try:
# Serialize the token to bytes
return token.SerializeToString()
except Exception as e:
raise Exception(f"Serialization failed: {str(e)}")
@staticmethod
def deserialize_session_token(bytes_data: bytes) -> types_pb2_session.SessionToken:
"""
Deserializes a byte array into a SessionToken object.
:param bytes_data: A byte array representing the serialized SessionToken.
:return: A SessionToken Protobuf object.
:raises ValueError: If the input byte array is None or empty.
:raises Exception: If deserialization fails.
"""
if not bytes_data:
raise ValueError(f"Input parameter is missing: {types_pb2_session.SessionToken.__name__}")
try:
# Deserialize the byte array into a SessionToken object
session_token = types_pb2_session.SessionToken()
session_token.ParseFromString(bytes_data)
return session_token
except DecodeError as e:
raise Exception(f"Deserialization failed: {str(e)}")

View file

@ -0,0 +1,31 @@
import uuid
from typing import Optional
UUID_BYTE_ARRAY_LENGTH = 16
class UuidExtension:
@staticmethod
def to_uuid(bytes_: Optional[bytes]) -> uuid.UUID:
"""
Converts a byte array into a UUID object.
"""
if bytes_ is None or len(bytes_) != UUID_BYTE_ARRAY_LENGTH:
raise ValueError(f"Wrong UUID size: expected {UUID_BYTE_ARRAY_LENGTH} bytes, got {len(bytes_)}")
# Unpack the byte array into two 64-bit integers
most_sig_bits = int.from_bytes(bytes_[:8], byteorder='big', signed=False)
least_sig_bits = int.from_bytes(bytes_[8:], byteorder='big', signed=False)
return uuid.UUID(int=(most_sig_bits << 64) | least_sig_bits)
@staticmethod
def to_bytes(uuid_: Optional[uuid.UUID]) -> bytes:
"""
Converts a UUID object into a byte array.
"""
if uuid_ is None:
raise ValueError(f"Input parameter is missing: {uuid.UUID.__name__}")
# Pack the UUID into a 16-byte array
return (uuid_.int >> 64).to_bytes(8, byteorder='big') + (uuid_.int & 0xFFFFFFFFFFFFFFFF).to_bytes(8, byteorder='big')

View file

@ -0,0 +1,31 @@
from typing import Optional
from frostfs_sdk.models.dto.version import Version
from frostfs_sdk.protos.models.refs import types_pb2 as types_pb2_refs
class VersionMapper:
@staticmethod
def to_grpc_message(version: Optional[Version]) -> Optional[types_pb2_refs.Version]:
"""
Converts a Version object to a gRPC Version message.
"""
if version is None:
return None
return types_pb2_refs.Version(
major=version.major,
minor=version.minor
)
@staticmethod
def to_model(grpc_version: Optional[types_pb2_refs.Version]) -> Optional[Version]:
"""
Converts a gRPC Version message to a Version object.
"""
if grpc_version is None or grpc_version.ByteSize() == 0:
return None
return Version(
major=grpc_version.major,
minor=grpc_version.minor
)