From 035f1d566719fd64ae974c4357fe7209b84955ad Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Mon, 21 Feb 2022 21:36:02 +0300 Subject: [PATCH] [#147] client: Send 3MB per message in `ObjectWriter.WritePayloadChunk` In previous implementation payload chunks were split into pieces with 512B length. This led to sending a large number of messages with a large amount of payload. Increase per-message payload chunk limit to 3MB. Signed-off-by: Leonard Lyubich --- client/object_put.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/client/object_put.go b/client/object_put.go index 4937bad..3d0adb8 100644 --- a/client/object_put.go +++ b/client/object_put.go @@ -105,10 +105,26 @@ func (x *ObjectWriter) WritePayloadChunk(chunk []byte) bool { } for ln := len(chunk); ln > 0; ln = len(chunk) { - if ln > 512 { - ln = 512 + // maxChunkLen restricts maximum byte length of the chunk + // transmitted in a single stream message. It depends on + // server settings and other message fields, but for now + // we simply assume that 3MB is large enough to reduce the + // number of messages, and not to exceed the limit + // (4MB by default for gRPC servers). + const maxChunkLen = 3 << 20 + if ln > maxChunkLen { + ln = maxChunkLen } + // we deal with size limit overflow above, but there is another case: + // what if method is called with "small" chunk many times? We write + // a message to the stream on each call. Alternatively, we could use buffering. + // In most cases, the chunk length does not vary between calls. Given this + // assumption, as well as the length of the payload from the header, it is + // possible to buffer the data of intermediate chunks, and send a message when + // the allocated buffer is filled, or when the last chunk is received. + // It is mentally assumed that allocating and filling the buffer is better than + // synchronous sending, but this needs to be tested. x.partChunk.SetChunk(chunk[:ln]) if !x.ctxCall.writeRequest() {