oracle: make JSONPath compatible with C# implementation

C# node uses simplified implementation which is easy to port.
This commit is contained in:
Evgeniy Stratonikov 2021-04-22 17:22:09 +03:00
parent fbcb08c5f0
commit 1c30d8c395
7 changed files with 656 additions and 16 deletions

1
go.mod
View file

@ -1,7 +1,6 @@
module github.com/nspcc-dev/neo-go
require (
github.com/PaesslerAG/jsonpath v0.1.1
github.com/Workiva/go-datastructures v1.0.50
github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db
github.com/alicebob/miniredis v2.5.0+incompatible

5
go.sum
View file

@ -8,11 +8,6 @@ github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
github.com/Workiva/go-datastructures v1.0.50 h1:slDmfW6KCHcC7U+LP3DDBbm4fqTwZGn1beOFPfGaLvo=
github.com/Workiva/go-datastructures v1.0.50/go.mod h1:Z+F2Rca0qCsVYDS8z7bAGm8f3UkzuWYS/oBZz5a7VVA=
github.com/abiosoft/ishell v2.0.0+incompatible h1:zpwIuEHc37EzrsIYah3cpevrIc8Oma7oZPxr03tlmmw=

View file

@ -143,7 +143,7 @@ func TestOracle(t *testing.T) {
putOracleRequest(t, cs.Hash, bc, "https://get.maxallowed", nil, "handle", []byte{}, 10_000_000)
putOracleRequest(t, cs.Hash, bc, "https://get.maxallowed", nil, "handle", []byte{}, 100_000_000)
flt := "Values[1]"
flt := "$.Values[1]"
putOracleRequest(t, cs.Hash, bc, "https://get.filter", &flt, "handle", []byte{}, 10_000_000)
putOracleRequest(t, cs.Hash, bc, "https://get.filterinv", &flt, "handle", []byte{}, 10_000_000)

View file

@ -5,9 +5,9 @@ import (
"errors"
"unicode/utf8"
"github.com/PaesslerAG/jsonpath"
"github.com/nspcc-dev/neo-go/pkg/core/state"
"github.com/nspcc-dev/neo-go/pkg/core/transaction"
"github.com/nspcc-dev/neo-go/pkg/services/oracle/jsonpath"
)
func filter(value []byte, path string) ([]byte, error) {
@ -18,11 +18,12 @@ func filter(value []byte, path string) ([]byte, error) {
if err := json.Unmarshal(value, &v); err != nil {
return nil, err
}
result, err := jsonpath.Get(path, v)
if err != nil {
return nil, err
result, ok := jsonpath.Get(path, v)
if !ok {
return nil, errors.New("invalid filter")
}
return json.Marshal([]interface{}{result})
return json.Marshal(result)
}
func filterRequest(result []byte, req *state.OracleRequest) (transaction.OracleResponseCode, []byte) {

View file

@ -29,10 +29,12 @@ func TestFilter(t *testing.T) {
testCases := []struct {
result, path string
}{
{`["Acme Co"]`, "Manufacturers[0].Name"},
{`[50]`, "Manufacturers[0].Products[0].Price"},
{`["Elbow Grease"]`, "Manufacturers[1].Products[0].Name"},
{`[{"Name":"Elbow Grease","Price":99.95}]`, "Manufacturers[1].Products[0]"},
{"[]", "$.Name"},
{`["Acme Co"]`, "$.Manufacturers[0].Name"},
{`["Acme Co"]`, "$..Manufacturers[0].Name"},
{`[50]`, "$.Manufacturers[0].Products[0].Price"},
{`["Elbow Grease"]`, "$.Manufacturers[1].Products[0].Name"},
{`[{"Name":"Elbow Grease","Price":99.95}]`, "$.Manufacturers[1].Products[0]"},
}
for _, tc := range testCases {

View file

@ -0,0 +1,413 @@
package jsonpath
import (
"encoding/json"
"sort"
"strconv"
"strings"
)
type (
// pathTokenType represents single JSONPath token.
pathTokenType byte
// pathParser combines JSONPath and a position to start parsing from.
pathParser struct {
s string
i int
}
)
const (
pathInvalid pathTokenType = iota
pathRoot
pathDot
pathLeftBracket
pathRightBracket
pathAsterisk
pathComma
pathColon
pathIdentifier
pathString
pathNumber
)
// Get returns substructures of value selected by path.
// The result is always non-nil unless path is invalid.
func Get(path string, value interface{}) ([]interface{}, bool) {
p := pathParser{s: path}
typ, _ := p.nextToken()
if typ != pathRoot {
return nil, false
}
objs := []interface{}{value}
for p.i < len(p.s) {
var ok bool
switch typ, _ := p.nextToken(); typ {
case pathDot:
objs, ok = p.processDot(objs)
case pathLeftBracket:
objs, ok = p.processLeftBracket(objs)
}
if !ok {
return nil, false
}
}
if objs == nil {
objs = []interface{}{}
}
return objs, true
}
func (p *pathParser) nextToken() (pathTokenType, string) {
var (
typ pathTokenType
value string
ok = true
numRead = 1
)
if p.i >= len(p.s) {
return pathInvalid, ""
}
switch c := p.s[p.i]; c {
case '$':
typ = pathRoot
case '.':
typ = pathDot
case '[':
typ = pathLeftBracket
case ']':
typ = pathRightBracket
case '*':
typ = pathAsterisk
case ',':
typ = pathComma
case ':':
typ = pathColon
case '\'':
typ = pathString
value, numRead, ok = p.parseString()
default:
switch {
case c == '_' || ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'):
typ = pathIdentifier
value, numRead, ok = p.parseIdent()
case c == '-' || ('0' <= c && c <= '9'):
typ = pathNumber
value, numRead, ok = p.parseNumber()
default:
return pathInvalid, ""
}
}
if !ok {
return pathInvalid, ""
}
p.i += numRead
return typ, value
}
// parseString parses JSON string surrounded by single quotes.
// It returns number of characters were consumed and true on success.
func (p *pathParser) parseString() (string, int, bool) {
var end int
for end = p.i + 1; end < len(p.s); end++ {
if p.s[end] == '\'' {
return p.s[p.i : end+1], end + 1 - p.i, true
}
}
return "", 0, false
}
// parseIdent parses alphanumeric identifier.
// It returns number of characters were consumed and true on success.
func (p *pathParser) parseIdent() (string, int, bool) {
var end int
for end = p.i + 1; end < len(p.s); end++ {
c := p.s[end]
if c != '_' && !('a' <= c && c <= 'z') &&
!('A' <= c && c <= 'Z') && !('0' <= c && c <= '9') {
break
}
}
return p.s[p.i:end], end - p.i, true
}
// parseNumber parses integer number.
// Only string representation is returned, size-checking is done on the first use.
// It also returns number of characters were consumed and true on success.
func (p *pathParser) parseNumber() (string, int, bool) {
var end int
for end = p.i + 1; end < len(p.s); end++ {
c := p.s[end]
if c < '0' || '9' < c {
break
}
}
return p.s[p.i:end], end - p.i, true
}
// processDot handles `.` operator.
// It either descends 1 level down or performs recursive descent.
func (p *pathParser) processDot(objs []interface{}) ([]interface{}, bool) {
typ, value := p.nextToken()
switch typ {
case pathAsterisk:
return p.descend(objs)
case pathDot:
return p.descendRecursive(objs)
case pathIdentifier:
return p.descendByIdent(objs, value)
default:
return nil, false
}
}
// descend descends 1 level down.
// It flattens arrays and returns map values for maps.
func (p *pathParser) descend(objs []interface{}) ([]interface{}, bool) {
var values []interface{}
for i := range objs {
switch obj := objs[i].(type) {
case []interface{}:
values = append(values, obj...)
case map[string]interface{}:
keys := make([]string, 0, len(obj))
for k := range obj {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
values = append(values, obj[k])
}
}
}
return values, true
}
// descendRecursive performs recursive descent.
func (p *pathParser) descendRecursive(objs []interface{}) ([]interface{}, bool) {
typ, val := p.nextToken()
if typ != pathIdentifier {
return nil, false
}
var values []interface{}
for len(objs) > 0 {
newObjs, _ := p.descendByIdent(objs, val)
values = append(values, newObjs...)
objs, _ = p.descend(objs)
}
return values, true
}
// descendByIdent performs map's field access by name.
func (p *pathParser) descendByIdent(objs []interface{}, names ...string) ([]interface{}, bool) {
var values []interface{}
for i := range objs {
obj, ok := objs[i].(map[string]interface{})
if !ok {
continue
}
for j := range names {
if v, ok := obj[names[j]]; ok {
values = append(values, v)
}
}
}
return values, true
}
// descendByIndex performs array access by index.
func (p *pathParser) descendByIndex(objs []interface{}, indices ...int) ([]interface{}, bool) {
var values []interface{}
for i := range objs {
obj, ok := objs[i].([]interface{})
if !ok {
continue
}
for _, j := range indices {
if j < 0 {
j += len(obj)
}
if 0 <= j && j < len(obj) {
values = append(values, obj[j])
}
}
}
return values, true
}
// processLeftBracket processes index expressions which can be either
// array/map access, array sub-slice or union of indices.
func (p *pathParser) processLeftBracket(objs []interface{}) ([]interface{}, bool) {
typ, value := p.nextToken()
switch typ {
case pathAsterisk:
typ, _ := p.nextToken()
if typ != pathRightBracket {
return nil, false
}
return p.descend(objs)
case pathColon:
return p.processSlice(objs, 0)
case pathNumber:
subTyp, _ := p.nextToken()
switch subTyp {
case pathColon:
index, err := strconv.ParseInt(value, 10, 32)
if err != nil {
return nil, false
}
return p.processSlice(objs, int(index))
case pathComma:
return p.processUnion(objs, pathNumber, value)
case pathRightBracket:
index, err := strconv.ParseInt(value, 10, 32)
if err != nil {
return nil, false
}
return p.descendByIndex(objs, int(index))
default:
return nil, false
}
case pathString:
subTyp, _ := p.nextToken()
switch subTyp {
case pathComma:
return p.processUnion(objs, pathString, value)
case pathRightBracket:
s := strings.Trim(value, "'")
err := json.Unmarshal([]byte(`"`+s+`"`), &s)
if err != nil {
return nil, false
}
return p.descendByIdent(objs, s)
default:
return nil, false
}
default:
return nil, false
}
}
// processUnion processes union of multiple indices.
// firstTyp is assumed to be either pathNumber or pathString.
func (p *pathParser) processUnion(objs []interface{}, firstTyp pathTokenType, firstVal string) ([]interface{}, bool) {
items := []string{firstVal}
for {
typ, val := p.nextToken()
if typ != firstTyp {
return nil, false
}
items = append(items, val)
typ, val = p.nextToken()
if typ == pathRightBracket {
break
} else if typ != pathComma {
return nil, false
}
}
switch firstTyp {
case pathNumber:
values := make([]int, len(items))
for i := range items {
index, err := strconv.ParseInt(items[i], 10, 32)
if err != nil {
return nil, false
}
values[i] = int(index)
}
return p.descendByIndex(objs, values...)
case pathString:
for i := range items {
s := strings.Trim(items[i], "'")
err := json.Unmarshal([]byte(`"`+s+`"`), &items[i])
if err != nil {
return nil, false
}
}
return p.descendByIdent(objs, items...)
default:
panic("token in union must be either number or string")
}
}
// processSlice processes slice with the specified start index.
func (p *pathParser) processSlice(objs []interface{}, start int) ([]interface{}, bool) {
typ, val := p.nextToken()
switch typ {
case pathNumber:
typ, _ := p.nextToken()
if typ != pathRightBracket {
return nil, false
}
index, err := strconv.ParseInt(val, 10, 32)
if err != nil {
return nil, false
}
return p.descendByRange(objs, start, int(index))
case pathRightBracket:
return p.descendByRange(objs, start, 0)
default:
return nil, false
}
}
// descendByRange is similar to descend but skips maps and returns sub-slices for arrays.
func (p *pathParser) descendByRange(objs []interface{}, start, end int) ([]interface{}, bool) {
var values []interface{}
for i := range objs {
arr, ok := objs[i].([]interface{})
if !ok {
continue
}
subStart := start
if subStart < 0 {
subStart += len(arr)
}
subEnd := end
if subEnd <= 0 {
subEnd += len(arr)
}
if subEnd > len(arr) {
subEnd = len(arr)
}
if subEnd <= subStart {
continue
}
values = append(values, arr[subStart:subEnd]...)
}
return values, true
}

View file

@ -0,0 +1,230 @@
package jsonpath
import (
"encoding/json"
"math"
"strconv"
"testing"
"github.com/stretchr/testify/require"
)
type pathTestCase struct {
path string
result string
}
func unmarshalGet(t *testing.T, js string, path string) ([]interface{}, bool) {
var v interface{}
require.NoError(t, json.Unmarshal([]byte(js), &v))
return Get(path, v)
}
func (p *pathTestCase) testUnmarshalGet(t *testing.T, js string) {
res, ok := unmarshalGet(t, js, p.path)
require.True(t, ok)
data, err := json.Marshal(res)
require.NoError(t, err)
require.JSONEq(t, p.result, string(data))
}
func TestInvalidPaths(t *testing.T) {
bigNum := strconv.FormatInt(int64(math.MaxInt32)+1, 10)
// errCases contains invalid json path expressions.
// These are either invalid(&) or unexpected token in some positions
// or big number/invalid string.
errCases := []string{
".",
"$1",
"&",
"$&",
"$.&",
"$.[0]",
"$..",
"$..*",
"$..&",
"$..1",
"$[&]",
"$[**]",
"$[1&]",
"$[" + bigNum + "]",
"$[" + bigNum + ":]",
"$[:" + bigNum + "]",
"$[1," + bigNum + "]",
"$[" + bigNum + "[]]",
"$['a'&]",
"$['a'1]",
"$['a",
"$['\\u123']",
"$['s','\\u123']",
"$[[]]",
"$[1,'a']",
"$[1,1&",
"$[1,1[]]",
"$[1:&]",
"$[1:1[]]",
"$[1:[]]",
"$[1:[]]",
"$[",
}
for _, tc := range errCases {
t.Run(tc, func(t *testing.T) {
_, ok := unmarshalGet(t, "{}", tc)
require.False(t, ok)
})
}
}
func TestDescendByIdent(t *testing.T) {
js := `{
"store": {
"name": "big",
"sub": [
{ "name": "sub1" },
{ "name": "sub2" }
],
"partner": { "name": "ppp" }
},
"another": { "name": "small" }
}`
testCases := []pathTestCase{
{"$.store.name", `["big"]`},
{"$['store']['name']", `["big"]`},
{"$[*].name", `["small","big"]`},
{"$.*.name", `["small","big"]`},
{"$..store.name", `["big"]`},
{"$.store..name", `["big","ppp","sub1","sub2"]`},
{"$..sub[*].name", `["sub1","sub2"]`},
{"$..sub.name", `[]`},
{"$..sub..name", `["sub1","sub2"]`},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
tc.testUnmarshalGet(t, js)
})
}
}
func TestDescendByIndex(t *testing.T) {
js := `["a","b","c","d"]`
testCases := []pathTestCase{
{"$[0]", `["a"]`},
{"$[3]", `["d"]`},
{"$[1:2]", `["b"]`},
{"$[1:-1]", `["b","c"]`},
{"$[-3:-1]", `["b","c"]`},
{"$[-3:3]", `["b","c"]`},
{"$[:3]", `["a","b","c"]`},
{"$[:100]", `["a","b","c","d"]`},
{"$[1:]", `["b","c","d"]`},
{"$[2:1]", `[]`},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
tc.testUnmarshalGet(t, js)
})
}
t.Run("$[:][1], skip wrong types", func(t *testing.T) {
js := `[[1,2],{"1":"4"},[5,6]]`
p := pathTestCase{"$[:][1]", "[2,6]"}
p.testUnmarshalGet(t, js)
})
t.Run("$[*].*, flatten", func(t *testing.T) {
js := `[[1,2],{"1":"4"},[5,6]]`
p := pathTestCase{"$[*].*", "[1,2,\"4\",5,6]"}
p.testUnmarshalGet(t, js)
})
t.Run("$[*].[1:], skip wrong types", func(t *testing.T) {
js := `[[1,2],3,{"1":"4"},[5,6]]`
p := pathTestCase{"$[*][1:]", "[2,6]"}
p.testUnmarshalGet(t, js)
})
}
func TestUnion(t *testing.T) {
js := `["a",{"x":1,"y":2,"z":3},"c","d"]`
testCases := []pathTestCase{
{"$[0,2]", `["a","c"]`},
{"$[1]['x','z']", `[1,3]`},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
tc.testUnmarshalGet(t, js)
})
}
}
// These tests are taken directly from C# code.
func TestCSharpCompat(t *testing.T) {
js := `{
"store": {
"book": [
{
"category": "reference",
"author": "Nigel Rees",
"title": "Sayings of the Century",
"price": 8.95
},
{
"category": "fiction",
"author": "Evelyn Waugh",
"title": "Sword of Honour",
"price": 12.99
},
{
"category": "fiction",
"author": "Herman Melville",
"title": "Moby Dick",
"isbn": "0-553-21311-3",
"price": 8.99
},
{
"category": "fiction",
"author": "J. R. R. Tolkien",
"title": "The Lord of the Rings",
"isbn": "0-395-19395-8",
"price": 22.99
}
],
"bicycle": {
"color": "red",
"price": 19.95
}
},
"expensive": 10
}`
// FIXME(fyrchik): some tests are commented because of how maps are processed.
testCases := []pathTestCase{
{"$.store.book[*].author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`},
{"$..author", `["Nigel Rees","Evelyn Waugh","Herman Melville","J. R. R. Tolkien"]`},
//{"$.store.*", `[[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95}]`},
{"$.store..price", `[19.95,8.95,12.99,8.99,22.99]`},
{"$..book[2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`},
{"$..book[-2]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99}]`},
{"$..book[0,1]", `[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`},
{"$..book[:2]", `[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`},
{"$..book[1:2]", `[{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99}]`},
{"$..book[-2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`},
{"$..book[2:]", `[{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}]`},
//{"$..*", `[{"book":[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],"bicycle":{"color":"red","price":19.95}},10,[{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99}],{"color":"red","price":19.95},{"category":"reference","author":"Nigel Rees","title":"Sayings of the Century","price":8.95},{"category":"fiction","author":"Evelyn Waugh","title":"Sword of Honour","price":12.99},{"category":"fiction","author":"Herman Melville","title":"Moby Dick","isbn":"0-553-21311-3","price":8.99},{"category":"fiction","author":"J. R. R. Tolkien","title":"The Lord of the Rings","isbn":"0-395-19395-8","price":22.99},"red",19.95,"reference","Nigel Rees","Sayings of the Century",8.95,"fiction","Evelyn Waugh","Sword of Honour",12.99,"fiction","Herman Melville","Moby Dick","0-553-21311-3",8.99,"fiction","J. R. R. Tolkien","The Lord of the Rings","0-395-19395-8",22.99]`},
{"$..invalidfield", `[]`},
}
for _, tc := range testCases {
t.Run(tc.path, func(t *testing.T) {
tc.testUnmarshalGet(t, js)
})
}
}