From 1c30d8c395872ebf342f6dd72d0765fb41785860 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Thu, 22 Apr 2021 17:22:09 +0300 Subject: [PATCH 1/4] oracle: make JSONPath compatible with C# implementation C# node uses simplified implementation which is easy to port. --- go.mod | 1 - go.sum | 5 - pkg/core/oracle_test.go | 2 +- pkg/services/oracle/filter.go | 11 +- pkg/services/oracle/filter_test.go | 10 +- pkg/services/oracle/jsonpath/jsonpath.go | 413 ++++++++++++++++++ pkg/services/oracle/jsonpath/jsonpath_test.go | 230 ++++++++++ 7 files changed, 656 insertions(+), 16 deletions(-) create mode 100644 pkg/services/oracle/jsonpath/jsonpath.go create mode 100644 pkg/services/oracle/jsonpath/jsonpath_test.go diff --git a/go.mod b/go.mod index e7897be89..f824717eb 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,6 @@ module github.com/nspcc-dev/neo-go require ( - github.com/PaesslerAG/jsonpath v0.1.1 github.com/Workiva/go-datastructures v1.0.50 github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db github.com/alicebob/miniredis v2.5.0+incompatible diff --git a/go.sum b/go.sum index 3a3e43c7a..85cbb14d8 100644 --- a/go.sum +++ b/go.sum @@ -8,11 +8,6 @@ github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8= -github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= -github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= -github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= -github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1beOFPfGaLvo= github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA= github.com/abiosoft/ishell v2.0.0+incompatible h1:zpwIuEHc37EzrsIYah3cpevrIc8Oma7oZPxr03tlmmw= diff --git a/pkg/core/oracle_test.go b/pkg/core/oracle_test.go index e10c8f547..ea266a63b 100644 --- a/pkg/core/oracle_test.go +++ b/pkg/core/oracle_test.go @@ -143,7 +143,7 @@ func TestOracle(t *testing.T) { putOracleRequest(t, cs.Hash, bc, "https://get.maxallowed", nil, "handle", []byte{}, 10_000_000) putOracleRequest(t, cs.Hash, bc, "https://get.maxallowed", nil, "handle", []byte{}, 100_000_000) - flt := "Values[1]" + flt := "$.Values[1]" putOracleRequest(t, cs.Hash, bc, "https://get.filter", &flt, "handle", []byte{}, 10_000_000) putOracleRequest(t, cs.Hash, bc, "https://get.filterinv", &flt, "handle", []byte{}, 10_000_000) diff --git a/pkg/services/oracle/filter.go b/pkg/services/oracle/filter.go index 87d762626..1aa5508a3 100644 --- a/pkg/services/oracle/filter.go +++ b/pkg/services/oracle/filter.go @@ -5,9 +5,9 @@ import ( "errors" "unicode/utf8" - "github.com/PaesslerAG/jsonpath" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/services/oracle/jsonpath" ) func filter(value []byte, path string) ([]byte, error) { @@ -18,11 +18,12 @@ func filter(value []byte, path string) ([]byte, error) { if err := json.Unmarshal(value, &v); err != nil { return nil, err } - result, err := jsonpath.Get(path, v) - if err != nil { - return nil, err + + result, ok := jsonpath.Get(path, v) + if !ok { + return nil, errors.New("invalid filter") } - return json.Marshal([]interface{}{result}) + return json.Marshal(result) } func filterRequest(result []byte, req *state.OracleRequest) (transaction.OracleResponseCode, []byte) { diff --git a/pkg/services/oracle/filter_test.go b/pkg/services/oracle/filter_test.go index d42573067..7a7de548c 100644 --- a/pkg/services/oracle/filter_test.go +++ b/pkg/services/oracle/filter_test.go @@ -29,10 +29,12 @@ func TestFilter(t *testing.T) { testCases := []struct { result, path string }{ - {`["Acme Co"]`, "Manufacturers[0].Name"}, - {`[50]`, "Manufacturers[0].Products[0].Price"}, - {`["Elbow Grease"]`, "Manufacturers[1].Products[0].Name"}, - {`[{"Name":"Elbow Grease","Price":99.95}]`, "Manufacturers[1].Products[0]"}, + {"[]", "$.Name"}, + {`["Acme Co"]`, "$.Manufacturers[0].Name"}, + {`["Acme Co"]`, "$..Manufacturers[0].Name"}, + {`[50]`, "$.Manufacturers[0].Products[0].Price"}, + {`["Elbow Grease"]`, "$.Manufacturers[1].Products[0].Name"}, + {`[{"Name":"Elbow Grease","Price":99.95}]`, "$.Manufacturers[1].Products[0]"}, } for _, tc := range testCases { diff --git a/pkg/services/oracle/jsonpath/jsonpath.go b/pkg/services/oracle/jsonpath/jsonpath.go new file mode 100644 index 000000000..7a1957f61 --- /dev/null +++ b/pkg/services/oracle/jsonpath/jsonpath.go @@ -0,0 +1,413 @@ +package jsonpath + +import ( + "encoding/json" + "sort" + "strconv" + "strings" +) + +type ( + // pathTokenType represents single JSONPath token. + pathTokenType byte + + // pathParser combines JSONPath and a position to start parsing from. + pathParser struct { + s string + i int + } +) + +const ( + pathInvalid pathTokenType = iota + pathRoot + pathDot + pathLeftBracket + pathRightBracket + pathAsterisk + pathComma + pathColon + pathIdentifier + pathString + pathNumber +) + +// Get returns substructures of value selected by path. +// The result is always non-nil unless path is invalid. +func Get(path string, value interface{}) ([]interface{}, bool) { + p := pathParser{s: path} + + typ, _ := p.nextToken() + if typ != pathRoot { + return nil, false + } + + objs := []interface{}{value} + for p.i < len(p.s) { + var ok bool + + switch typ, _ := p.nextToken(); typ { + case pathDot: + objs, ok = p.processDot(objs) + case pathLeftBracket: + objs, ok = p.processLeftBracket(objs) + } + + if !ok { + return nil, false + } + } + + if objs == nil { + objs = []interface{}{} + } + return objs, true +} + +func (p *pathParser) nextToken() (pathTokenType, string) { + var ( + typ pathTokenType + value string + ok = true + numRead = 1 + ) + + if p.i >= len(p.s) { + return pathInvalid, "" + } + + switch c := p.s[p.i]; c { + case '$': + typ = pathRoot + case '.': + typ = pathDot + case '[': + typ = pathLeftBracket + case ']': + typ = pathRightBracket + case '*': + typ = pathAsterisk + case ',': + typ = pathComma + case ':': + typ = pathColon + case '\'': + typ = pathString + value, numRead, ok = p.parseString() + default: + switch { + case c == '_' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'): + typ = pathIdentifier + value, numRead, ok = p.parseIdent() + case c == '-' || ('0' <= c && c <= '9'): + typ = pathNumber + value, numRead, ok = p.parseNumber() + default: + return pathInvalid, "" + } + } + + if !ok { + return pathInvalid, "" + } + + p.i += numRead + return typ, value +} + +// parseString parses JSON string surrounded by single quotes. +// It returns number of characters were consumed and true on success. +func (p *pathParser) parseString() (string, int, bool) { + var end int + for end = p.i + 1; end < len(p.s); end++ { + if p.s[end] == '\'' { + return p.s[p.i : end+1], end + 1 - p.i, true + } + } + + return "", 0, false +} + +// parseIdent parses alphanumeric identifier. +// It returns number of characters were consumed and true on success. +func (p *pathParser) parseIdent() (string, int, bool) { + var end int + for end = p.i + 1; end < len(p.s); end++ { + c := p.s[end] + if c != '_' && !('a' <= c && c <= 'z') && + !('A' <= c && c <= 'Z') && !('0' <= c && c <= '9') { + break + } + } + + return p.s[p.i:end], end - p.i, true +} + +// parseNumber parses integer number. +// Only string representation is returned, size-checking is done on the first use. +// It also returns number of characters were consumed and true on success. +func (p *pathParser) parseNumber() (string, int, bool) { + var end int + for end = p.i + 1; end < len(p.s); end++ { + c := p.s[end] + if c < '0' || '9' < c { + break + } + } + + return p.s[p.i:end], end - p.i, true +} + +// processDot handles `.` operator. +// It either descends 1 level down or performs recursive descent. +func (p *pathParser) processDot(objs []interface{}) ([]interface{}, bool) { + typ, value := p.nextToken() + switch typ { + case pathAsterisk: + return p.descend(objs) + case pathDot: + return p.descendRecursive(objs) + case pathIdentifier: + return p.descendByIdent(objs, value) + default: + return nil, false + } +} + +// descend descends 1 level down. +// It flattens arrays and returns map values for maps. +func (p *pathParser) descend(objs []interface{}) ([]interface{}, bool) { + var values []interface{} + for i := range objs { + switch obj := objs[i].(type) { + case []interface{}: + values = append(values, obj...) + case map[string]interface{}: + keys := make([]string, 0, len(obj)) + for k := range obj { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, k := range keys { + values = append(values, obj[k]) + } + } + } + + return values, true +} + +// descendRecursive performs recursive descent. +func (p *pathParser) descendRecursive(objs []interface{}) ([]interface{}, bool) { + typ, val := p.nextToken() + if typ != pathIdentifier { + return nil, false + } + + var values []interface{} + + for len(objs) > 0 { + newObjs, _ := p.descendByIdent(objs, val) + values = append(values, newObjs...) + objs, _ = p.descend(objs) + } + + return values, true +} + +// descendByIdent performs map's field access by name. +func (p *pathParser) descendByIdent(objs []interface{}, names ...string) ([]interface{}, bool) { + var values []interface{} + for i := range objs { + obj, ok := objs[i].(map[string]interface{}) + if !ok { + continue + } + + for j := range names { + if v, ok := obj[names[j]]; ok { + values = append(values, v) + } + } + } + return values, true +} + +// descendByIndex performs array access by index. +func (p *pathParser) descendByIndex(objs []interface{}, indices ...int) ([]interface{}, bool) { + var values []interface{} + for i := range objs { + obj, ok := objs[i].([]interface{}) + if !ok { + continue + } + + for _, j := range indices { + if j < 0 { + j += len(obj) + } + if 0 <= j && j < len(obj) { + values = append(values, obj[j]) + } + } + } + + return values, true +} + +// processLeftBracket processes index expressions which can be either +// array/map access, array sub-slice or union of indices. +func (p *pathParser) processLeftBracket(objs []interface{}) ([]interface{}, bool) { + typ, value := p.nextToken() + switch typ { + case pathAsterisk: + typ, _ := p.nextToken() + if typ != pathRightBracket { + return nil, false + } + + return p.descend(objs) + case pathColon: + return p.processSlice(objs, 0) + case pathNumber: + subTyp, _ := p.nextToken() + switch subTyp { + case pathColon: + index, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, false + } + + return p.processSlice(objs, int(index)) + case pathComma: + return p.processUnion(objs, pathNumber, value) + case pathRightBracket: + index, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return nil, false + } + + return p.descendByIndex(objs, int(index)) + default: + return nil, false + } + case pathString: + subTyp, _ := p.nextToken() + switch subTyp { + case pathComma: + return p.processUnion(objs, pathString, value) + case pathRightBracket: + s := strings.Trim(value, "'") + err := json.Unmarshal([]byte(`"`+s+`"`), &s) + if err != nil { + return nil, false + } + return p.descendByIdent(objs, s) + default: + return nil, false + } + default: + return nil, false + } +} + +// processUnion processes union of multiple indices. +// firstTyp is assumed to be either pathNumber or pathString. +func (p *pathParser) processUnion(objs []interface{}, firstTyp pathTokenType, firstVal string) ([]interface{}, bool) { + items := []string{firstVal} + for { + typ, val := p.nextToken() + if typ != firstTyp { + return nil, false + } + + items = append(items, val) + typ, val = p.nextToken() + if typ == pathRightBracket { + break + } else if typ != pathComma { + return nil, false + } + } + + switch firstTyp { + case pathNumber: + values := make([]int, len(items)) + for i := range items { + index, err := strconv.ParseInt(items[i], 10, 32) + if err != nil { + return nil, false + } + values[i] = int(index) + } + return p.descendByIndex(objs, values...) + case pathString: + for i := range items { + s := strings.Trim(items[i], "'") + err := json.Unmarshal([]byte(`"`+s+`"`), &items[i]) + if err != nil { + return nil, false + } + } + return p.descendByIdent(objs, items...) + default: + panic("token in union must be either number or string") + } +} + +// processSlice processes slice with the specified start index. +func (p *pathParser) processSlice(objs []interface{}, start int) ([]interface{}, bool) { + typ, val := p.nextToken() + switch typ { + case pathNumber: + typ, _ := p.nextToken() + if typ != pathRightBracket { + return nil, false + } + + index, err := strconv.ParseInt(val, 10, 32) + if err != nil { + return nil, false + } + + return p.descendByRange(objs, start, int(index)) + case pathRightBracket: + return p.descendByRange(objs, start, 0) + default: + return nil, false + } +} + +// descendByRange is similar to descend but skips maps and returns sub-slices for arrays. +func (p *pathParser) descendByRange(objs []interface{}, start, end int) ([]interface{}, bool) { + var values []interface{} + for i := range objs { + arr, ok := objs[i].([]interface{}) + if !ok { + continue + } + + subStart := start + if subStart < 0 { + subStart += len(arr) + } + + subEnd := end + if subEnd <= 0 { + subEnd += len(arr) + } + + if subEnd > len(arr) { + subEnd = len(arr) + } + + if subEnd <= subStart { + continue + } + values = append(values, arr[subStart:subEnd]...) + } + + return values, true +} diff --git a/pkg/services/oracle/jsonpath/jsonpath_test.go b/pkg/services/oracle/jsonpath/jsonpath_test.go new file mode 100644 index 000000000..94be5d4c0 --- /dev/null +++ b/pkg/services/oracle/jsonpath/jsonpath_test.go @@ -0,0 +1,230 @@ +package jsonpath + +import ( + "encoding/json" + "math" + "strconv" + "testing" + + "github.com/stretchr/testify/require" +) + +type pathTestCase struct { + path string + result string +} + +func unmarshalGet(t *testing.T, js string, path string) ([]interface{}, bool) { + var v interface{} + require.NoError(t, json.Unmarshal([]byte(js), &v)) + return Get(path, v) +} + +func (p *pathTestCase) testUnmarshalGet(t *testing.T, js string) { + res, ok := unmarshalGet(t, js, p.path) + require.True(t, ok) + + data, err := json.Marshal(res) + require.NoError(t, err) + require.JSONEq(t, p.result, string(data)) +} + +func TestInvalidPaths(t *testing.T) { + bigNum := strconv.FormatInt(int64(math.MaxInt32)+1, 10) + + // errCases contains invalid json path expressions. + // These are either invalid(&) or unexpected token in some positions + // or big number/invalid string. + errCases := []string{ + ".", + "$1", + "&", + "$&", + "$.&", + "$.[0]", + "$..", + "$..*", + "$..&", + "$..1", + "$[&]", + "$[**]", + "$[1&]", + "$[" + bigNum + "]", + "$[" + bigNum + ":]", + "$[:" + bigNum + "]", + "$[1," + bigNum + "]", + "$[" + bigNum + "[]]", + "$['a'&]", + "$['a'1]", + "$['a", + "$['\\u123']", + "$['s','\\u123']", + "$[[]]", + "$[1,'a']", + "$[1,1&", + "$[1,1[]]", + "$[1:&]", + "$[1:1[]]", + "$[1:[]]", + "$[1:[]]", + "$[", + } + + for _, tc := range errCases { + t.Run(tc, func(t *testing.T) { + _, ok := unmarshalGet(t, "{}", tc) + require.False(t, ok) + }) + } +} + +func TestDescendByIdent(t *testing.T) { + js := `{ + "store": { + "name": "big", + "sub": [ + { "name": "sub1" }, + { "name": "sub2" } + ], + "partner": { "name": "ppp" } + }, + "another": { "name": "small" } + }` + + testCases := []pathTestCase{ + {"$.store.name", `["big"]`}, + {"$['store']['name']", `["big"]`}, + {"$[*].name", `["small","big"]`}, + {"$.*.name", `["small","big"]`}, + {"$..store.name", `["big"]`}, + {"$.store..name", `["big","ppp","sub1","sub2"]`}, + {"$..sub[*].name", `["sub1","sub2"]`}, + {"$..sub.name", `[]`}, + {"$..sub..name", `["sub1","sub2"]`}, + } + for _, tc := range testCases { + t.Run(tc.path, func(t *testing.T) { + tc.testUnmarshalGet(t, js) + }) + } +} + +func TestDescendByIndex(t *testing.T) { + js := `["a","b","c","d"]` + + testCases := []pathTestCase{ + {"$[0]", `["a"]`}, + {"$[3]", `["d"]`}, + {"$[1:2]", `["b"]`}, + {"$[1:-1]", `["b","c"]`}, + {"$[-3:-1]", `["b","c"]`}, + {"$[-3:3]", `["b","c"]`}, + {"$[:3]", `["a","b","c"]`}, + {"$[:100]", `["a","b","c","d"]`}, + {"$[1:]", `["b","c","d"]`}, + {"$[2:1]", `[]`}, + } + + for _, tc := range testCases { + t.Run(tc.path, func(t *testing.T) { + tc.testUnmarshalGet(t, js) + }) + } + + t.Run("$[:][1], skip wrong types", func(t *testing.T) { + js := `[[1,2],{"1":"4"},[5,6]]` + p := pathTestCase{"$[:][1]", "[2,6]"} + p.testUnmarshalGet(t, js) + }) + + t.Run("$[*].*, flatten", func(t *testing.T) { + js := `[[1,2],{"1":"4"},[5,6]]` + p := pathTestCase{"$[*].*", "[1,2,\"4\",5,6]"} + p.testUnmarshalGet(t, js) + }) + + t.Run("$[*].[1:], skip wrong types", func(t *testing.T) { + js := `[[1,2],3,{"1":"4"},[5,6]]` + p := pathTestCase{"$[*][1:]", "[2,6]"} + p.testUnmarshalGet(t, js) + }) +} + +func TestUnion(t *testing.T) { + js := `["a",{"x":1,"y":2,"z":3},"c","d"]` + + testCases := []pathTestCase{ + {"$[0,2]", `["a","c"]`}, + {"$[1]['x','z']", `[1,3]`}, + } + + for _, tc := range testCases { + t.Run(tc.path, func(t *testing.T) { + tc.testUnmarshalGet(t, js) + }) + } +} + +// These tests are taken directly from C# code. +func TestCSharpCompat(t *testing.T) { + js := `{ + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + }, + "expensive": 10 +}` + + // FIXME(fyrchik): some tests are commented because of how maps are processed. + testCases := []pathTestCase{ + {"$.store.book[*].author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`}, + {"$..author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`}, + //{"$.store.*", `[[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95}]`}, + {"$.store..price", `[19.95,8.95,12.99,8.99,22.99]`}, + {"$..book[2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`}, + {"$..book[-2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`}, + {"$..book[0,1]", `[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`}, + {"$..book[:2]", `[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`}, + {"$..book[1:2]", `[{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`}, + {"$..book[-2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, + {"$..book[2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, + //{"$..*", `[{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},10,[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95},{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99},"red",19.95,"reference","Nigel Rees","Sayings of the Century",8.95,"fiction","Evelyn Waugh","Sword of Honour",12.99,"fiction","Herman Melville","Moby Dick","0-553-21311-3",8.99,"fiction","J. R. R. Tolkien","The Lord of the Rings","0-395-19395-8",22.99]`}, + {"$..invalidfield", `[]`}, + } + + for _, tc := range testCases { + t.Run(tc.path, func(t *testing.T) { + tc.testUnmarshalGet(t, js) + }) + } +} From 6890688b8fecb74e535f214efa9e7db77129d8bb Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Fri, 30 Apr 2021 10:35:33 +0300 Subject: [PATCH 2/4] oracle: add max nesting depth to JSONPath filter --- pkg/services/oracle/filter_test.go | 1 - pkg/services/oracle/jsonpath/jsonpath.go | 40 +++++++++++++-- pkg/services/oracle/jsonpath/jsonpath_test.go | 50 ++++++++++++++++++- 3 files changed, 85 insertions(+), 6 deletions(-) diff --git a/pkg/services/oracle/filter_test.go b/pkg/services/oracle/filter_test.go index 7a7de548c..134d94b6d 100644 --- a/pkg/services/oracle/filter_test.go +++ b/pkg/services/oracle/filter_test.go @@ -31,7 +31,6 @@ func TestFilter(t *testing.T) { }{ {"[]", "$.Name"}, {`["Acme Co"]`, "$.Manufacturers[0].Name"}, - {`["Acme Co"]`, "$..Manufacturers[0].Name"}, {`[50]`, "$.Manufacturers[0].Products[0].Price"}, {`["Elbow Grease"]`, "$.Manufacturers[1].Products[0].Name"}, {`[{"Name":"Elbow Grease","Price":99.95}]`, "$.Manufacturers[1].Products[0]"}, diff --git a/pkg/services/oracle/jsonpath/jsonpath.go b/pkg/services/oracle/jsonpath/jsonpath.go index 7a1957f61..94887926e 100644 --- a/pkg/services/oracle/jsonpath/jsonpath.go +++ b/pkg/services/oracle/jsonpath/jsonpath.go @@ -13,8 +13,9 @@ type ( // pathParser combines JSONPath and a position to start parsing from. pathParser struct { - s string - i int + s string + i int + depth int } ) @@ -32,10 +33,15 @@ const ( pathNumber ) +const maxNestingDepth = 6 + // Get returns substructures of value selected by path. // The result is always non-nil unless path is invalid. func Get(path string, value interface{}) ([]interface{}, bool) { - p := pathParser{s: path} + p := pathParser{ + depth: maxNestingDepth, + s: path, + } typ, _ := p.nextToken() if typ != pathRoot { @@ -177,6 +183,11 @@ func (p *pathParser) processDot(objs []interface{}) ([]interface{}, bool) { // descend descends 1 level down. // It flattens arrays and returns map values for maps. func (p *pathParser) descend(objs []interface{}) ([]interface{}, bool) { + if p.depth <= 0 { + return nil, false + } + p.depth-- + var values []interface{} for i := range objs { switch obj := objs[i].(type) { @@ -209,7 +220,7 @@ func (p *pathParser) descendRecursive(objs []interface{}) ([]interface{}, bool) var values []interface{} for len(objs) > 0 { - newObjs, _ := p.descendByIdent(objs, val) + newObjs, _ := p.descendByIdentAux(objs, false, val) values = append(values, newObjs...) objs, _ = p.descend(objs) } @@ -219,6 +230,17 @@ func (p *pathParser) descendRecursive(objs []interface{}) ([]interface{}, bool) // descendByIdent performs map's field access by name. func (p *pathParser) descendByIdent(objs []interface{}, names ...string) ([]interface{}, bool) { + return p.descendByIdentAux(objs, true, names...) +} + +func (p *pathParser) descendByIdentAux(objs []interface{}, checkDepth bool, names ...string) ([]interface{}, bool) { + if checkDepth { + if p.depth <= 0 { + return nil, false + } + p.depth-- + } + var values []interface{} for i := range objs { obj, ok := objs[i].(map[string]interface{}) @@ -237,6 +259,11 @@ func (p *pathParser) descendByIdent(objs []interface{}, names ...string) ([]inte // descendByIndex performs array access by index. func (p *pathParser) descendByIndex(objs []interface{}, indices ...int) ([]interface{}, bool) { + if p.depth <= 0 { + return nil, false + } + p.depth-- + var values []interface{} for i := range objs { obj, ok := objs[i].([]interface{}) @@ -382,6 +409,11 @@ func (p *pathParser) processSlice(objs []interface{}, start int) ([]interface{}, // descendByRange is similar to descend but skips maps and returns sub-slices for arrays. func (p *pathParser) descendByRange(objs []interface{}, start, end int) ([]interface{}, bool) { + if p.depth <= 0 { + return nil, false + } + p.depth-- + var values []interface{} for i := range objs { arr, ok := objs[i].([]interface{}) diff --git a/pkg/services/oracle/jsonpath/jsonpath_test.go b/pkg/services/oracle/jsonpath/jsonpath_test.go index 94be5d4c0..c6cff988c 100644 --- a/pkg/services/oracle/jsonpath/jsonpath_test.go +++ b/pkg/services/oracle/jsonpath/jsonpath_test.go @@ -98,7 +98,6 @@ func TestDescendByIdent(t *testing.T) { {"$.*.name", `["small","big"]`}, {"$..store.name", `["big"]`}, {"$.store..name", `["big","ppp","sub1","sub2"]`}, - {"$..sub[*].name", `["sub1","sub2"]`}, {"$..sub.name", `[]`}, {"$..sub..name", `["sub1","sub2"]`}, } @@ -107,6 +106,28 @@ func TestDescendByIdent(t *testing.T) { tc.testUnmarshalGet(t, js) }) } + + t.Run("big depth", func(t *testing.T) { + js := `{"a":{"b":{"c":{"d":{"e":{"f":{"g":1}}}}}}}` + t.Run("single field", func(t *testing.T) { + t.Run("max", func(t *testing.T) { + p := pathTestCase{"$.a.b.c.d.e.f", `[{"g":1}]`} + p.testUnmarshalGet(t, js) + }) + + _, ok := unmarshalGet(t, js, "$.a.b.c.d.e.f.g") + require.False(t, ok) + }) + t.Run("wildcard", func(t *testing.T) { + t.Run("max", func(t *testing.T) { + p := pathTestCase{"$.*.*.*.*.*.*", `[{"g":1}]`} + p.testUnmarshalGet(t, js) + }) + + _, ok := unmarshalGet(t, js, "$.*.*.*.*.*.*.*") + require.False(t, ok) + }) + }) } func TestDescendByIndex(t *testing.T) { @@ -131,6 +152,28 @@ func TestDescendByIndex(t *testing.T) { }) } + t.Run("big depth", func(t *testing.T) { + js := `[[[[[[[1]]]]]]]` + t.Run("single index", func(t *testing.T) { + t.Run("max", func(t *testing.T) { + p := pathTestCase{"$[0][0][0][0][0][0]", "[[1]]"} + p.testUnmarshalGet(t, js) + }) + + _, ok := unmarshalGet(t, js, "$[0][0][0][0][0][0][0]") + require.False(t, ok) + }) + t.Run("slice", func(t *testing.T) { + t.Run("max", func(t *testing.T) { + p := pathTestCase{"$[0:][0:][0:][0:][0:][0:]", "[[1]]"} + p.testUnmarshalGet(t, js) + }) + + _, ok := unmarshalGet(t, js, "$[0:][0:][0:][0:][0:][0:][0:]") + require.False(t, ok) + }) + }) + t.Run("$[:][1], skip wrong types", func(t *testing.T) { js := `[[1,2],{"1":"4"},[5,6]]` p := pathTestCase{"$[:][1]", "[2,6]"} @@ -227,4 +270,9 @@ func TestCSharpCompat(t *testing.T) { tc.testUnmarshalGet(t, js) }) } + + t.Run("bad cases", func(t *testing.T) { + _, ok := unmarshalGet(t, js, `$..book[*].author"`) + require.False(t, ok) + }) } From 5fc81c787b46fc9cbf9ef37cfae41d461f960e62 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Fri, 7 May 2021 19:48:57 +0300 Subject: [PATCH 3/4] jsonpath: use ordered map json unmarshaler --- go.mod | 1 + go.sum | 2 ++ pkg/services/oracle/filter.go | 10 +++++-- pkg/services/oracle/jsonpath/jsonpath.go | 26 ++++++++----------- pkg/services/oracle/jsonpath/jsonpath_test.go | 17 +++++++----- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index f824717eb..daa944e09 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/stretchr/testify v1.6.1 github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73 github.com/urfave/cli v1.20.0 + github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 go.etcd.io/bbolt v1.3.4 go.uber.org/atomic v1.4.0 go.uber.org/zap v1.10.0 diff --git a/go.sum b/go.sum index 85cbb14d8..48cc9472a 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,8 @@ github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2K github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo= +github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74/go.mod h1:RmMWU37GKR2s6pgrIEB4ixgpVCt/cf7dnJv3fuH1J1c= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7 h1:Y17pEjKgx2X0A69WQPGa8hx/Myzu+4NdUxlkZpbAYio= diff --git a/pkg/services/oracle/filter.go b/pkg/services/oracle/filter.go index 1aa5508a3..65eeed59e 100644 --- a/pkg/services/oracle/filter.go +++ b/pkg/services/oracle/filter.go @@ -1,21 +1,27 @@ package oracle import ( - "encoding/json" + "bytes" "errors" "unicode/utf8" "github.com/nspcc-dev/neo-go/pkg/core/state" "github.com/nspcc-dev/neo-go/pkg/core/transaction" "github.com/nspcc-dev/neo-go/pkg/services/oracle/jsonpath" + json "github.com/virtuald/go-ordered-json" ) func filter(value []byte, path string) ([]byte, error) { if !utf8.Valid(value) { return nil, errors.New("not an UTF-8") } + + buf := bytes.NewBuffer(value) + d := json.NewDecoder(buf) + d.UseOrderedObject() + var v interface{} - if err := json.Unmarshal(value, &v); err != nil { + if err := d.Decode(&v); err != nil { return nil, err } diff --git a/pkg/services/oracle/jsonpath/jsonpath.go b/pkg/services/oracle/jsonpath/jsonpath.go index 94887926e..105776d12 100644 --- a/pkg/services/oracle/jsonpath/jsonpath.go +++ b/pkg/services/oracle/jsonpath/jsonpath.go @@ -1,10 +1,10 @@ package jsonpath import ( - "encoding/json" - "sort" "strconv" "strings" + + json "github.com/virtuald/go-ordered-json" ) type ( @@ -193,16 +193,9 @@ func (p *pathParser) descend(objs []interface{}) ([]interface{}, bool) { switch obj := objs[i].(type) { case []interface{}: values = append(values, obj...) - case map[string]interface{}: - keys := make([]string, 0, len(obj)) - for k := range obj { - keys = append(keys, k) - } - - sort.Strings(keys) - - for _, k := range keys { - values = append(values, obj[k]) + case json.OrderedObject: + for i := range obj { + values = append(values, obj[i].Value) } } } @@ -243,14 +236,17 @@ func (p *pathParser) descendByIdentAux(objs []interface{}, checkDepth bool, name var values []interface{} for i := range objs { - obj, ok := objs[i].(map[string]interface{}) + obj, ok := objs[i].(json.OrderedObject) if !ok { continue } for j := range names { - if v, ok := obj[names[j]]; ok { - values = append(values, v) + for k := range obj { + if obj[k].Key == names[j] { + values = append(values, obj[k].Value) + break + } } } } diff --git a/pkg/services/oracle/jsonpath/jsonpath_test.go b/pkg/services/oracle/jsonpath/jsonpath_test.go index c6cff988c..cd9f40ae9 100644 --- a/pkg/services/oracle/jsonpath/jsonpath_test.go +++ b/pkg/services/oracle/jsonpath/jsonpath_test.go @@ -1,12 +1,13 @@ package jsonpath import ( - "encoding/json" + "bytes" "math" "strconv" "testing" "github.com/stretchr/testify/require" + json "github.com/virtuald/go-ordered-json" ) type pathTestCase struct { @@ -16,7 +17,10 @@ type pathTestCase struct { func unmarshalGet(t *testing.T, js string, path string) ([]interface{}, bool) { var v interface{} - require.NoError(t, json.Unmarshal([]byte(js), &v)) + buf := bytes.NewBuffer([]byte(js)) + d := json.NewDecoder(buf) + d.UseOrderedObject() + require.NoError(t, d.Decode(&v)) return Get(path, v) } @@ -94,8 +98,8 @@ func TestDescendByIdent(t *testing.T) { testCases := []pathTestCase{ {"$.store.name", `["big"]`}, {"$['store']['name']", `["big"]`}, - {"$[*].name", `["small","big"]`}, - {"$.*.name", `["small","big"]`}, + {"$[*].name", `["big","small"]`}, + {"$.*.name", `["big","small"]`}, {"$..store.name", `["big"]`}, {"$.store..name", `["big","ppp","sub1","sub2"]`}, {"$..sub.name", `[]`}, @@ -248,11 +252,10 @@ func TestCSharpCompat(t *testing.T) { "expensive": 10 }` - // FIXME(fyrchik): some tests are commented because of how maps are processed. testCases := []pathTestCase{ {"$.store.book[*].author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`}, {"$..author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`}, - //{"$.store.*", `[[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95}]`}, + {"$.store.*", `[[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95}]`}, {"$.store..price", `[19.95,8.95,12.99,8.99,22.99]`}, {"$..book[2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`}, {"$..book[-2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`}, @@ -261,7 +264,7 @@ func TestCSharpCompat(t *testing.T) { {"$..book[1:2]", `[{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`}, {"$..book[-2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, {"$..book[2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, - //{"$..*", `[{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},10,[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95},{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99},"red",19.95,"reference","Nigel Rees","Sayings of the Century",8.95,"fiction","Evelyn Waugh","Sword of Honour",12.99,"fiction","Herman Melville","Moby Dick","0-553-21311-3",8.99,"fiction","J. R. R. Tolkien","The Lord of the Rings","0-395-19395-8",22.99]`}, + {"$.*", `[{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},10]`}, {"$..invalidfield", `[]`}, } From 28a0d68c0573d2d056bd66a195de21b692f34730 Mon Sep 17 00:00:00 2001 From: Evgeniy Stratonikov Date: Tue, 11 May 2021 12:35:28 +0300 Subject: [PATCH 4/4] jsonpath: allow empty paths --- pkg/services/oracle/jsonpath/jsonpath.go | 4 ++++ pkg/services/oracle/jsonpath/jsonpath_test.go | 1 + 2 files changed, 5 insertions(+) diff --git a/pkg/services/oracle/jsonpath/jsonpath.go b/pkg/services/oracle/jsonpath/jsonpath.go index 105776d12..5672572a4 100644 --- a/pkg/services/oracle/jsonpath/jsonpath.go +++ b/pkg/services/oracle/jsonpath/jsonpath.go @@ -38,6 +38,10 @@ const maxNestingDepth = 6 // Get returns substructures of value selected by path. // The result is always non-nil unless path is invalid. func Get(path string, value interface{}) ([]interface{}, bool) { + if path == "" { + return []interface{}{value}, true + } + p := pathParser{ depth: maxNestingDepth, s: path, diff --git a/pkg/services/oracle/jsonpath/jsonpath_test.go b/pkg/services/oracle/jsonpath/jsonpath_test.go index cd9f40ae9..7bffbcd6b 100644 --- a/pkg/services/oracle/jsonpath/jsonpath_test.go +++ b/pkg/services/oracle/jsonpath/jsonpath_test.go @@ -264,6 +264,7 @@ func TestCSharpCompat(t *testing.T) { {"$..book[1:2]", `[{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`}, {"$..book[-2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, {"$..book[2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`}, + {"", `[{"store":{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},"expensive":10}]`}, {"$.*", `[{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},10]`}, {"$..invalidfield", `[]`}, }