[#379] Add Iana CharsetReader for Oracle integration #379

Merged
alexvanin merged 1 commit from pogpp/frostfs-s3-gw:bugfix/update_decoder into master 2024-05-03 07:22:57 +00:00
9 changed files with 35 additions and 8 deletions
Showing only changes of commit db05021786 - Show all commits

View file

@ -12,6 +12,7 @@ This document outlines major changes between releases.
- Fix flaky `TestErrorTimeoutChecking` (`make test` sometimes failed) (#290) - Fix flaky `TestErrorTimeoutChecking` (`make test` sometimes failed) (#290)
- Fix user owner ID in billing metrics (#321) - Fix user owner ID in billing metrics (#321)
- Fix HTTP/2 requests (#341) - Fix HTTP/2 requests (#341)
- Fix Decoder.CharsetReader is nil (#379)
### Added ### Added
- Add new `frostfs.buffer_max_size_for_put` config param and sync TZ hash for PUT operations (#197) - Add new `frostfs.buffer_max_size_for_put` config param and sync TZ hash for PUT operations (#197)

View file

@ -399,7 +399,7 @@ func (h *handler) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil { } else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) h.logAndSendError(w, "could not parse bucket acl", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
return return
} }
@ -615,7 +615,7 @@ func (h *handler) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
} else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil { } else if err = h.cfg.NewXMLDecoder(r.Body).Decode(list); err != nil {
h.logAndSendError(w, "could not parse bucket acl", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) h.logAndSendError(w, "could not parse bucket acl", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
return return
} }

View file

@ -2,6 +2,7 @@ package handler
import ( import (
"encoding/xml" "encoding/xml"
"fmt"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
@ -179,7 +180,7 @@ func (h *handler) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Re
// Unmarshal list of keys to be deleted. // Unmarshal list of keys to be deleted.
requested := &DeleteObjectsRequest{} requested := &DeleteObjectsRequest{}
if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil { if err := h.cfg.NewXMLDecoder(r.Body).Decode(requested); err != nil {
h.logAndSendError(w, "couldn't decode body", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) h.logAndSendError(w, "couldn't decode body", reqInfo, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
return return
} }

View file

@ -441,7 +441,7 @@ func (h *handler) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.
reqBody := new(CompleteMultipartUpload) reqBody := new(CompleteMultipartUpload)
if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil { if err = h.cfg.NewXMLDecoder(r.Body).Decode(reqBody); err != nil {
h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo, h.logAndSendError(w, "could not read complete multipart upload xml", reqInfo,
errors.GetAPIError(errors.ErrMalformedXML), additional...) fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()), additional...)
return return
} }
if len(reqBody.Parts) == 0 { if len(reqBody.Parts) == 0 {

View file

@ -485,7 +485,8 @@ func (h *handler) PostObject(w http.ResponseWriter, r *http.Request) {
buffer := bytes.NewBufferString(tagging) buffer := bytes.NewBufferString(tagging)
tags := new(data.Tagging) tags := new(data.Tagging)
if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil { if err = h.cfg.NewXMLDecoder(buffer).Decode(tags); err != nil {
h.logAndSendError(w, "could not decode tag set", reqInfo, errors.GetAPIError(errors.ErrMalformedXML)) h.logAndSendError(w, "could not decode tag set", reqInfo,
fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error()))
return return
} }
tagSet, err = h.readTagSet(tags) tagSet, err = h.readTagSet(tags)
@ -1211,7 +1212,7 @@ func (h *handler) parseLocationConstraint(r *http.Request) (*createBucketParams,
params := new(createBucketParams) params := new(createBucketParams)
if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil { if err := h.cfg.NewXMLDecoder(r.Body).Decode(params); err != nil {
return nil, errors.GetAPIError(errors.ErrMalformedXML) return nil, fmt.Errorf("%w: %s", errors.GetAPIError(errors.ErrMalformedXML), err.Error())
} }
return params, nil return params, nil
} }

View file

@ -484,7 +484,7 @@ func determineRequestTags(r *http.Request, decoder XMLDecoder, op string) (map[s
if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) { if strings.HasSuffix(op, PutObjectTaggingOperation) || strings.HasSuffix(op, PutBucketTaggingOperation) {
tagging := new(data.Tagging) tagging := new(data.Tagging)
if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil { if err := decoder.NewXMLDecoder(r.Body).Decode(tagging); err != nil {
return nil, apiErr.GetAPIError(apiErr.ErrMalformedXML) return nil, fmt.Errorf("%w: %s", apiErr.GetAPIError(apiErr.ErrMalformedXML), err.Error())
} }
GetReqInfo(r.Context()).Tagging = tagging GetReqInfo(r.Context()).Tagging = tagging

View file

@ -50,6 +50,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"golang.org/x/text/encoding/ianaindex"
"google.golang.org/grpc" "google.golang.org/grpc"
) )
@ -317,6 +318,13 @@ func (s *appSettings) DefaultCopiesNumbers(namespace string) []uint32 {
func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder { func (s *appSettings) NewXMLDecoder(r io.Reader) *xml.Decoder {
dec := xml.NewDecoder(r) dec := xml.NewDecoder(r)
dec.CharsetReader = func(charset string, reader io.Reader) (io.Reader, error) {
enc, err := ianaindex.IANA.Encoding(charset)
if err != nil {
return nil, fmt.Errorf("charset %s: %w", charset, err)
}
return enc.NewDecoder().Reader(reader), nil
dkirillov marked this conversation as resolved Outdated

Since setting this field doesn't require any config parameter we can (and should) move assignment out of critical section (mutex)

Since setting this field doesn't require any config parameter we can (and should) move assignment out of critical section (mutex)
}
s.mu.RLock() s.mu.RLock()
dkirillov marked this conversation as resolved Outdated

Let's write

return nil, fmt.Errorf("charset %s: %w", charset, err)
Let's write ```golang return nil, fmt.Errorf("charset %s: %w", charset, err) ```
if s.defaultXMLNS { if s.defaultXMLNS {

View file

@ -34,6 +34,15 @@ func TestDefaultNamespace(t *testing.T) {
</Part> </Part>
</CompleteMultipartUpload> </CompleteMultipartUpload>
` `
xmlASCII := `<?xml version="1.0" encoding="US-ASCII"?>
<CompleteMultipartUpload>
<Part>
<PartNumber>1</PartNumber>
<ETag>
dkirillov marked this conversation as resolved Outdated

It's better write like this

<ETag>8b73814bee405ec32b5d1fc88cd5d97a</ETag>
It's better write like this ```golang <ETag>8b73814bee405ec32b5d1fc88cd5d97a</ETag> ```
8b73814bee405ec32b5d1fc88cd5d97a
</ETag>
</Part>
</CompleteMultipartUpload>`
for _, tc := range []struct { for _, tc := range []struct {
settings *appSettings settings *appSettings
@ -82,6 +91,13 @@ func TestDefaultNamespace(t *testing.T) {
input: xmlBodyWithInvalidNamespace, input: xmlBodyWithInvalidNamespace,
err: true, err: true,
}, },
{
settings: &appSettings{
defaultXMLNS: true,
},
input: xmlASCII,
err: false,
},
} { } {
t.Run("", func(t *testing.T) { t.Run("", func(t *testing.T) {
model := new(handler.CompleteMultipartUpload) model := new(handler.CompleteMultipartUpload)

2
go.mod
View file

@ -31,6 +31,7 @@ require (
golang.org/x/crypto v0.21.0 golang.org/x/crypto v0.21.0
golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
golang.org/x/net v0.23.0 golang.org/x/net v0.23.0
golang.org/x/text v0.14.0
google.golang.org/grpc v1.59.0 google.golang.org/grpc v1.59.0
google.golang.org/protobuf v1.33.0 google.golang.org/protobuf v1.33.0
) )
@ -95,7 +96,6 @@ require (
golang.org/x/sync v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.18.0 // indirect golang.org/x/sys v0.18.0 // indirect
golang.org/x/term v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect