diff -Nru golang-github-tidwall-gjson-1.6.7/debian/changelog golang-github-tidwall-gjson-1.14.4/debian/changelog --- golang-github-tidwall-gjson-1.6.7/debian/changelog 2022-11-29 17:37:01.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/debian/changelog 2023-03-12 23:32:40.000000000 +0000 @@ -1,3 +1,18 @@ +golang-github-tidwall-gjson (1.14.4-2) unstable; urgency=medium + + * Upload to unstable. + + -- Cyril Brulebois Mon, 13 Mar 2023 00:32:40 +0100 + +golang-github-tidwall-gjson (1.14.4-1) experimental; urgency=medium + + * New upstream release. + * Security fixes (ReDoS – regular expression denial of service): + - CVE-2021-42248 (Closes: #1011616). + - CVE-2021-42836 (Closes: #1000225). + + -- Cyril Brulebois Sun, 05 Mar 2023 01:34:13 +0100 + golang-github-tidwall-gjson (1.6.7-2) unstable; urgency=medium [ Debian Janitor ] diff -Nru golang-github-tidwall-gjson-1.6.7/gjson.go golang-github-tidwall-gjson-1.14.4/gjson.go --- golang-github-tidwall-gjson-1.6.7/gjson.go 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/gjson.go 2022-11-22 01:54:56.000000000 +0000 @@ -2,8 +2,6 @@ package gjson import ( - "encoding/json" - "reflect" "strconv" "strings" "time" @@ -65,6 +63,9 @@ Num float64 // Index of raw value in original json, zero means index unknown Index int + // Indexes of all the elements that match on a path containing the '#' + // query character. + Indexes []int } // String returns a string representation of the value. @@ -187,14 +188,15 @@ } // Array returns back an array of values. -// If the result represents a non-existent value, then an empty array will be -// returned. If the result is not a JSON array, the return value will be an +// If the result represents a null value or is non-existent, then an empty +// array will be returned. +// If the result is not a JSON array, the return value will be an // array containing one result. func (t Result) Array() []Result { if t.Type == Null { return []Result{} } - if t.Type != JSON { + if !t.IsArray() { return []Result{t} } r := t.arrayOrMap('[', false) @@ -211,6 +213,11 @@ return t.Type == JSON && len(t.Raw) > 0 && t.Raw[0] == '[' } +// IsBool returns true if the result value is a JSON boolean. +func (t Result) IsBool() bool { + return t.Type == True || t.Type == False +} + // ForEach iterates through values. // If the result represents a non-existent value, then no values will be // iterated. If the result is an Object, the iterator will pass the key and @@ -226,17 +233,19 @@ return } json := t.Raw - var keys bool + var obj bool var i int var key, value Result for ; i < len(json); i++ { if json[i] == '{' { i++ key.Type = String - keys = true + obj = true break } else if json[i] == '[' { i++ + key.Type = Number + key.Num = -1 break } if json[i] > ' ' { @@ -246,8 +255,9 @@ var str string var vesc bool var ok bool + var idx int for ; i < len(json); i++ { - if keys { + if obj { if json[i] != '"' { continue } @@ -262,7 +272,9 @@ key.Str = str[1 : len(str)-1] } key.Raw = str - key.Index = s + key.Index = s + t.Index + } else { + key.Num += 1 } for ; i < len(json); i++ { if json[i] <= ' ' || json[i] == ',' || json[i] == ':' { @@ -275,14 +287,22 @@ if !ok { return } - value.Index = s + if t.Indexes != nil { + if idx < len(t.Indexes) { + value.Index = t.Indexes[idx] + } + } else { + value.Index = s + t.Index + } if !iterator(key, value) { return } + idx++ } } -// Map returns back an map of values. The result should be a JSON array. +// Map returns back a map of values. The result should be a JSON object. +// If the result is not a JSON object, the return value will be an empty map. func (t Result) Map() map[string]Result { if t.Type != JSON { return map[string]Result{} @@ -294,7 +314,15 @@ // Get searches result for the specified path. // The result should be a JSON array or object. func (t Result) Get(path string) Result { - return Get(t.Raw, path) + r := Get(t.Raw, path) + if r.Indexes != nil { + for i := 0; i < len(r.Indexes); i++ { + r.Indexes[i] += t.Index + } + } else { + r.Index += t.Index + } + return r } type arrayOrMapResult struct { @@ -385,6 +413,8 @@ value.Raw, value.Str = tostr(json[i:]) value.Num = 0 } + value.Index = i + t.Index + i += len(value.Raw) - 1 if r.vc == '{' { @@ -411,6 +441,17 @@ } } end: + if t.Indexes != nil { + if len(t.Indexes) != len(r.a) { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = 0 + } + } else { + for i := 0; i < len(r.a); i++ { + r.a[i].Index = t.Indexes[i] + } + } + } return } @@ -422,7 +463,8 @@ // use the Valid function first. func Parse(json string) Result { var value Result - for i := 0; i < len(json); i++ { + i := 0 + for ; i < len(json); i++ { if json[i] == '{' || json[i] == '[' { value.Type = JSON value.Raw = json[i:] // just take the entire raw @@ -432,16 +474,20 @@ continue } switch json[i] { - default: - if (json[i] >= '0' && json[i] <= '9') || json[i] == '-' { + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + value.Type = Number + value.Raw, value.Num = tonum(json[i:]) + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + // nan value.Type = Number value.Raw, value.Num = tonum(json[i:]) } else { - return Result{} + // null + value.Type = Null + value.Raw = tolit(json[i:]) } - case 'n': - value.Type = Null - value.Raw = tolit(json[i:]) case 't': value.Type = True value.Raw = tolit(json[i:]) @@ -451,9 +497,14 @@ case '"': value.Type = String value.Raw, value.Str = tostr(json[i:]) + default: + return Result{} } break } + if value.Exists() { + value.Index = i + } return value } @@ -527,20 +578,12 @@ return } // could be a '+' or '-'. let's assume so. - continue - } - if json[i] < ']' { - // probably a valid number - continue - } - if json[i] == 'e' || json[i] == 'E' { - // allow for exponential numbers - continue + } else if json[i] == ']' || json[i] == '}' { + // break on ']' or '}' + raw = json[:i] + num, _ = strconv.ParseFloat(raw, 64) + return } - // likely a ']' or '}' - raw = json[:i] - num, _ = strconv.ParseFloat(raw, 64) - return } raw = json num, _ = strconv.ParseFloat(raw, 64) @@ -585,7 +628,7 @@ continue } } - break + return json[:i+1], unescape(json[1:i]) } } var ret string @@ -715,10 +758,10 @@ alogkey string query struct { on bool + all bool path string op string value string - all bool } } @@ -732,8 +775,13 @@ } if path[i] == '.' { r.part = path[:i] - r.path = path[i+1:] - r.more = true + if !r.arrch && i < len(path)-1 && isDotPiperChar(path[i+1:]) { + r.pipe = path[i+1:] + r.piped = true + } else { + r.path = path[i+1:] + r.more = true + } return } if path[i] == '#' { @@ -746,120 +794,27 @@ } else if path[1] == '[' || path[1] == '(' { // query r.query.on = true - if true { - qpath, op, value, _, fi, ok := parseQuery(path[i:]) - if !ok { - // bad query, end now - break - } - r.query.path = qpath - r.query.op = op - r.query.value = value - i = fi - 1 - if i+1 < len(path) && path[i+1] == '#' { - r.query.all = true - } - } else { - var end byte - if path[1] == '[' { - end = ']' - } else { - end = ')' - } - i += 2 - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s := i - for ; i < len(path); i++ { - if path[i] <= ' ' || - path[i] == '!' || - path[i] == '=' || - path[i] == '<' || - path[i] == '>' || - path[i] == '%' || - path[i] == end { - break - } - } - r.query.path = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - if i < len(path) { - s = i - if path[i] == '!' { - if i < len(path)-1 && (path[i+1] == '=' || - path[i+1] == '%') { - i++ - } - } else if path[i] == '<' || path[i] == '>' { - if i < len(path)-1 && path[i+1] == '=' { - i++ - } - } else if path[i] == '=' { - if i < len(path)-1 && path[i+1] == '=' { - s++ - i++ - } - } - i++ - r.query.op = path[s:i] - // whitespace - for ; i < len(path); i++ { - if path[i] > ' ' { - break - } - } - s = i - for ; i < len(path); i++ { - if path[i] == '"' { - i++ - s2 := i - for ; i < len(path); i++ { - if path[i] > '\\' { - continue - } - if path[i] == '"' { - // look for an escaped slash - if path[i-1] == '\\' { - n := 0 - for j := i - 2; j > s2-1; j-- { - if path[j] != '\\' { - break - } - n++ - } - if n%2 == 0 { - continue - } - } - break - } - } - } else if path[i] == end { - if i+1 < len(path) && path[i+1] == '#' { - r.query.all = true - } - break - } - } - if i > len(path) { - i = len(path) - } - v := path[s:i] - for len(v) > 0 && v[len(v)-1] <= ' ' { - v = v[:len(v)-1] - } - r.query.value = v + qpath, op, value, _, fi, vesc, ok := + parseQuery(path[i:]) + if !ok { + // bad query, end now + break + } + if len(value) >= 2 && value[0] == '"' && + value[len(value)-1] == '"' { + value = value[1 : len(value)-1] + if vesc { + value = unescape(value) } } + r.query.path = qpath + r.query.op = op + r.query.value = value + + i = fi - 1 + if i+1 < len(path) && path[i+1] == '#' { + r.query.all = true + } } } continue @@ -885,11 +840,11 @@ // # middle // .cap # right func parseQuery(query string) ( - path, op, value, remain string, i int, ok bool, + path, op, value, remain string, i int, vesc, ok bool, ) { if len(query) < 2 || query[0] != '#' || (query[1] != '(' && query[1] != '[') { - return "", "", "", "", i, false + return "", "", "", "", i, false, false } i = 2 j := 0 // start of value part @@ -917,6 +872,7 @@ i++ for ; i < len(query); i++ { if query[i] == '\\' { + vesc = true i++ } else if query[i] == '"' { break @@ -925,7 +881,7 @@ } } if depth > 0 { - return "", "", "", "", i, false + return "", "", "", "", i, false, false } if j > 0 { path = trim(query[2:j]) @@ -962,7 +918,7 @@ path = trim(query[2:i]) remain = query[i+1:] } - return path, op, value, remain, i + 1, true + return path, op, value, remain, i + 1, vesc, true } func trim(s string) string { @@ -979,6 +935,26 @@ return s } +// peek at the next byte and see if it's a '@', '[', or '{'. +func isDotPiperChar(s string) bool { + if DisableModifiers { + return false + } + c := s[0] + if c == '@' { + // check that the next component is *not* a modifier. + i := 1 + for ; i < len(s); i++ { + if s[i] == '.' || s[i] == '|' || s[i] == ':' { + break + } + } + _, ok := modifiers[s[1:i]] + return ok + } + return c == '[' || c == '{' +} + type objectPathResult struct { part string path string @@ -997,12 +973,8 @@ return } if path[i] == '.' { - // peek at the next byte and see if it's a '@', '[', or '{'. r.part = path[:i] - if !DisableModifiers && - i < len(path)-1 && - (path[i+1] == '@' || - path[i+1] == '[' || path[i+1] == '{') { + if i < len(path)-1 && isDotPiperChar(path[i+1:]) { r.pipe = path[i+1:] r.piped = true } else { @@ -1032,16 +1004,13 @@ continue } else if path[i] == '.' { r.part = string(epart) - // peek at the next byte and see if it's a '@' modifier - if !DisableModifiers && - i < len(path)-1 && path[i+1] == '@' { + if i < len(path)-1 && isDotPiperChar(path[i+1:]) { r.pipe = path[i+1:] r.piped = true } else { r.path = path[i+1:] r.more = true } - r.more = true return } else if path[i] == '|' { r.part = string(epart) @@ -1175,9 +1144,9 @@ } if rp.wild { if kesc { - pmatch = match.Match(unescape(key), rp.part) + pmatch = matchLimit(unescape(key), rp.part) } else { - pmatch = match.Match(key, rp.part) + pmatch = matchLimit(key, rp.part) } } else { if kesc { @@ -1188,6 +1157,7 @@ } hit = pmatch && !rp.more for ; i < len(c.json); i++ { + var num bool switch c.json[i] { default: continue @@ -1235,15 +1205,13 @@ return i, true } } - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(c.json, i) - if hit { - c.value.Raw = val - c.value.Type = Number - c.value.Num, _ = strconv.ParseFloat(val, 64) - return i, true + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break } - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := c.json[i] i, val = parseLiteral(c.json, i) if hit { @@ -1256,16 +1224,43 @@ } return i, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + } + if num { + i, val = parseNumber(c.json, i) + if hit { + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } } break } } return i, false } + +// matchLimit will limit the complexity of the match operation to avoid ReDos +// attacks from arbritary inputs. +// See the github.com/tidwall/match.MatchLimit function for more information. +func matchLimit(str, pattern string) bool { + matched, _ := match.MatchLimit(str, pattern, 10000) + return matched +} + func queryMatches(rp *arrayPathResult, value Result) bool { rpv := rp.query.value - if len(rpv) > 2 && rpv[0] == '"' && rpv[len(rpv)-1] == '"' { - rpv = rpv[1 : len(rpv)-1] + if len(rpv) > 0 && rpv[0] == '~' { + // convert to bool + rpv = rpv[1:] + if value.Bool() { + value = Result{Type: True} + } else { + value = Result{Type: False} + } } if !value.Exists() { return false @@ -1293,9 +1288,9 @@ case ">=": return value.Str >= rpv case "%": - return match.Match(value.Str, rpv) + return matchLimit(value.Str, rpv) case "!%": - return !match.Match(value.Str, rpv) + return !matchLimit(value.Str, rpv) } case Number: rpvn, _ := strconv.ParseFloat(rpv, 64) @@ -1345,6 +1340,7 @@ var alog []int var partidx int var multires []byte + var queryIndexes []int rp := parseArrayPath(path) if !rp.arrch { n, ok := parseUint(rp.part) @@ -1365,6 +1361,10 @@ multires = append(multires, '[') } } + var tmp parseContext + tmp.value = qval + fillIndex(c.json, &tmp) + parentIndex := tmp.value.Index var res Result if qval.Type == JSON { res = qval.Get(rp.query.path) @@ -1396,6 +1396,7 @@ multires = append(multires, ',') } multires = append(multires, raw...) + queryIndexes = append(queryIndexes, res.Index+parentIndex) } } else { c.value = res @@ -1404,7 +1405,6 @@ } return false } - for i < len(c.json)+1 { if !rp.arrch { pmatch = partidx == h @@ -1423,6 +1423,7 @@ } else { ch = c.json[i] } + var num bool switch ch { default: continue @@ -1505,26 +1506,13 @@ return i, true } } - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(c.json, i) - if rp.query.on { - var qval Result - qval.Raw = val - qval.Type = Number - qval.Num, _ = strconv.ParseFloat(val, 64) - if procQuery(qval) { - return i, true - } - } else if hit { - if rp.alogok { - break - } - c.value.Raw = val - c.value.Type = Number - c.value.Num, _ = strconv.ParseFloat(val, 64) - return i, true + case 'n': + if i+1 < len(c.json) && c.json[i+1] != 'u' { + num = true + break } - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := c.json[i] i, val = parseLiteral(c.json, i) if rp.query.on { @@ -1552,6 +1540,9 @@ } return i, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true case ']': if rp.arrch && rp.part == "#" { if rp.alogok { @@ -1561,6 +1552,7 @@ c.pipe = right c.piped = true } + var indexes = make([]int, 0, 64) var jsons = make([]byte, 0, 64) jsons = append(jsons, '[') for j, k := 0, 0; j < len(alog); j++ { @@ -1586,6 +1578,7 @@ raw = res.String() } jsons = append(jsons, []byte(raw)...) + indexes = append(indexes, res.Index) k++ } } @@ -1594,6 +1587,7 @@ jsons = append(jsons, ']') c.value.Type = JSON c.value.Raw = string(jsons) + c.value.Indexes = indexes return i + 1, true } if rp.alogok { @@ -1606,14 +1600,42 @@ c.calcd = true return i + 1, true } - if len(multires) > 0 && !c.value.Exists() { - c.value = Result{ - Raw: string(append(multires, ']')), - Type: JSON, + if !c.value.Exists() { + if len(multires) > 0 { + c.value = Result{ + Raw: string(append(multires, ']')), + Type: JSON, + Indexes: queryIndexes, + } + } else if rp.query.all { + c.value = Result{ + Raw: "[]", + Type: JSON, + } } } return i + 1, false } + if num { + i, val = parseNumber(c.json, i) + if rp.query.on { + var qval Result + qval.Raw = val + qval.Type = Number + qval.Num, _ = strconv.ParseFloat(val, 64) + if procQuery(qval) { + return i, true + } + } else if hit { + if rp.alogok { + break + } + c.value.Raw = val + c.value.Type = Number + c.value.Num, _ = strconv.ParseFloat(val, 64) + return i, true + } + } break } } @@ -1729,7 +1751,7 @@ // first character in path is either '[' or '{', and has already been checked // prior to calling this function. func parseSubSelectors(path string) (sels []subSelector, out string, ok bool) { - modifer := 0 + modifier := 0 depth := 1 colon := 0 start := 1 @@ -1744,6 +1766,7 @@ } sels = append(sels, sel) colon = 0 + modifier = 0 start = i + 1 } for ; i < len(path); i++ { @@ -1751,11 +1774,11 @@ case '\\': i++ case '@': - if modifer == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { - modifer = i + if modifier == 0 && i > 0 && (path[i-1] == '.' || path[i-1] == '|') { + modifier = i } case ':': - if modifer == 0 && colon == 0 && depth == 1 { + if modifier == 0 && colon == 0 && depth == 1 { colon = i } case ',': @@ -1808,24 +1831,71 @@ return false } switch component[i] { - case '[', ']', '{', '}', '(', ')', '#', '|': + case '[', ']', '{', '}', '(', ')', '#', '|', '!': return false } } return true } -func appendJSONString(dst []byte, s string) []byte { +var hexchars = [...]byte{ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', +} + +func appendHex16(dst []byte, x uint16) []byte { + return append(dst, + hexchars[x>>12&0xF], hexchars[x>>8&0xF], + hexchars[x>>4&0xF], hexchars[x>>0&0xF], + ) +} + +// AppendJSONString is a convenience function that converts the provided string +// to a valid JSON string and appends it to dst. +func AppendJSONString(dst []byte, s string) []byte { + dst = append(dst, make([]byte, len(s)+2)...) + dst = append(dst[:len(dst)-len(s)-2], '"') for i := 0; i < len(s); i++ { - if s[i] < ' ' || s[i] == '\\' || s[i] == '"' || s[i] > 126 { - d, _ := json.Marshal(s) - return append(dst, string(d)...) + if s[i] < ' ' { + dst = append(dst, '\\') + switch s[i] { + case '\n': + dst = append(dst, 'n') + case '\r': + dst = append(dst, 'r') + case '\t': + dst = append(dst, 't') + default: + dst = append(dst, 'u') + dst = appendHex16(dst, uint16(s[i])) + } + } else if s[i] == '>' || s[i] == '<' || s[i] == '&' { + dst = append(dst, '\\', 'u') + dst = appendHex16(dst, uint16(s[i])) + } else if s[i] == '\\' { + dst = append(dst, '\\', '\\') + } else if s[i] == '"' { + dst = append(dst, '\\', '"') + } else if s[i] > 127 { + // read utf8 character + r, n := utf8.DecodeRuneInString(s[i:]) + if n == 0 { + break + } + if r == utf8.RuneError && n == 1 { + dst = append(dst, `\ufffd`...) + } else if r == '\u2028' || r == '\u2029' { + dst = append(dst, `\u202`...) + dst = append(dst, hexchars[r&0xF]) + } else { + dst = append(dst, s[i:i+n]...) + } + i = i + n - 1 + } else { + dst = append(dst, s[i]) } } - dst = append(dst, '"') - dst = append(dst, s...) - dst = append(dst, '"') - return dst + return append(dst, '"') } type parseContext struct { @@ -1872,22 +1942,25 @@ // use the Valid function first. func Get(json, path string) Result { if len(path) > 1 { - if !DisableModifiers { - if path[0] == '@' { - // possible modifier - var ok bool - var npath string - var rjson string + if (path[0] == '@' && !DisableModifiers) || path[0] == '!' { + // possible modifier + var ok bool + var npath string + var rjson string + if path[0] == '@' && !DisableModifiers { npath, rjson, ok = execModifier(json, path) - if ok { - path = npath - if len(path) > 0 && (path[0] == '|' || path[0] == '.') { - res := Get(rjson, path[1:]) - res.Index = 0 - return res - } - return Parse(rjson) + } else if path[0] == '!' { + npath, rjson, ok = execStatic(json, path) + } + if ok { + path = npath + if len(path) > 0 && (path[0] == '|' || path[0] == '.') { + res := Get(rjson, path[1:]) + res.Index = 0 + res.Indexes = nil + return res } + return Parse(rjson) } } if path[0] == '[' || path[0] == '{' { @@ -1912,14 +1985,14 @@ if sub.name[0] == '"' && Valid(sub.name) { b = append(b, sub.name...) } else { - b = appendJSONString(b, sub.name) + b = AppendJSONString(b, sub.name) } } else { last := nameOfLast(sub.path) if isSimpleName(last) { - b = appendJSONString(b, last) + b = AppendJSONString(b, last) } else { - b = appendJSONString(b, "_") + b = AppendJSONString(b, "_") } } b = append(b, ':') @@ -2124,11 +2197,15 @@ res.Raw = val res.Type = JSON } - return i, res, true + var tmp parseContext + tmp.value = res + fillIndex(json, &tmp) + return i, tmp.value, true } if json[i] <= ' ' { continue } + var num bool switch json[i] { case '"': i++ @@ -2148,15 +2225,13 @@ } } return i, res, true - case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': - i, val = parseNumber(json, i) - if hit { - res.Raw = val - res.Type = Number - res.Num, _ = strconv.ParseFloat(val, 64) + case 'n': + if i+1 < len(json) && json[i+1] != 'u' { + num = true + break } - return i, res, true - case 't', 'f', 'n': + fallthrough + case 't', 'f': vc := json[i] i, val = parseLiteral(json, i) if hit { @@ -2169,16 +2244,24 @@ } return i, res, true } + case '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'i', 'I', 'N': + num = true + } + if num { + i, val = parseNumber(json, i) + if hit { + res.Raw = val + res.Type = Number + res.Num, _ = strconv.ParseFloat(val, 64) + } + return i, res, true } + } return i, res, false } -var ( // used for testing - testWatchForFallback bool - testLastWasFallback bool -) - // GetMany searches json for the multiple paths. // The return value is a Result array where the number of items // will be equal to the number of input paths. @@ -2379,6 +2462,12 @@ // sign if data[i] == '-' { i++ + if i == len(data) { + return i, false + } + if data[i] < '0' || data[i] > '9' { + return i, false + } } // int if i == len(data) { @@ -2529,15 +2618,51 @@ return n, true } +// safeInt validates a given JSON number +// ensures it lies within the minimum and maximum representable JSON numbers func safeInt(f float64) (n int64, ok bool) { + // https://tc39.es/ecma262/#sec-number.min_safe_integer + // https://tc39.es/ecma262/#sec-number.max_safe_integer if f < -9007199254740991 || f > 9007199254740991 { return 0, false } return int64(f), true } +// execStatic parses the path to find a static value. +// The input expects that the path already starts with a '!' +func execStatic(json, path string) (pathOut, res string, ok bool) { + name := path[1:] + if len(name) > 0 { + switch name[0] { + case '{', '[', '"', '+', '-', '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9': + _, res = parseSquash(name, 0) + pathOut = name[len(res):] + return pathOut, res, true + } + } + for i := 1; i < len(path); i++ { + if path[i] == '|' { + pathOut = path[i:] + name = path[1:i] + break + } + if path[i] == '.' { + pathOut = path[i:] + name = path[1:i] + break + } + } + switch strings.ToLower(name) { + case "true", "false", "null", "nan", "inf": + return pathOut, name, true + } + return pathOut, res, false +} + // execModifier parses the path to find a matching modifier function. -// then input expects that the path already starts with a '@' +// The input expects that the path already starts with a '@' func execModifier(json, path string) (pathOut, res string, ok bool) { name := path[1:] var hasArgs bool @@ -2608,6 +2733,11 @@ "flatten": modFlatten, "join": modJoin, "valid": modValid, + "keys": modKeys, + "values": modValues, + "tostr": modToStr, + "fromstr": modFromStr, + "group": modGroup, } // AddModifier binds a custom modifier command to the GJSON syntax. @@ -2740,25 +2870,82 @@ out = append(out, '[') var idx int res.ForEach(func(_, value Result) bool { - if idx > 0 { - out = append(out, ',') - } + var raw string if value.IsArray() { if deep { - out = append(out, unwrap(modFlatten(value.Raw, arg))...) + raw = unwrap(modFlatten(value.Raw, arg)) } else { - out = append(out, unwrap(value.Raw)...) + raw = unwrap(value.Raw) } } else { - out = append(out, value.Raw...) + raw = value.Raw + } + raw = strings.TrimSpace(raw) + if len(raw) > 0 { + if idx > 0 { + out = append(out, ',') + } + out = append(out, raw...) + idx++ } - idx++ return true }) out = append(out, ']') return bytesString(out) } +// @keys extracts the keys from an object. +// {"first":"Tom","last":"Smith"} -> ["first","last"] +func modKeys(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + obj := v.IsObject() + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(key, _ Result) bool { + if i > 0 { + out.WriteByte(',') + } + if obj { + out.WriteString(key.Raw) + } else { + out.WriteString("null") + } + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + +// @values extracts the values from an object. +// {"first":"Tom","last":"Smith"} -> ["Tom","Smith"] +func modValues(json, arg string) string { + v := Parse(json) + if !v.Exists() { + return "[]" + } + if v.IsArray() { + return json + } + var out strings.Builder + out.WriteByte('[') + var i int + v.ForEach(func(_, value Result) bool { + if i > 0 { + out.WriteByte(',') + } + out.WriteString(value.Raw) + i++ + return true + }) + out.WriteByte(']') + return out.String() +} + // @join multiple objects into a single object. // [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} // The arg can be "true" to specify that duplicate keys should be preserved. @@ -2836,6 +3023,69 @@ return json } +// @fromstr converts a string to json +// "{\"id\":1023,\"name\":\"alert\"}" -> {"id":1023,"name":"alert"} +func modFromStr(json, arg string) string { + if !Valid(json) { + return "" + } + return Parse(json).String() +} + +// @tostr converts a string to json +// {"id":1023,"name":"alert"} -> "{\"id\":1023,\"name\":\"alert\"}" +func modToStr(str, arg string) string { + return string(AppendJSONString(nil, str)) +} + +func modGroup(json, arg string) string { + res := Parse(json) + if !res.IsObject() { + return "" + } + var all [][]byte + res.ForEach(func(key, value Result) bool { + if !value.IsArray() { + return true + } + var idx int + value.ForEach(func(_, value Result) bool { + if idx == len(all) { + all = append(all, []byte{}) + } + all[idx] = append(all[idx], ("," + key.Raw + ":" + value.Raw)...) + idx++ + return true + }) + return true + }) + var data []byte + data = append(data, '[') + for i, item := range all { + if i > 0 { + data = append(data, ',') + } + data = append(data, '{') + data = append(data, item[1:]...) + data = append(data, '}') + } + data = append(data, ']') + return string(data) +} + +// stringHeader instead of reflect.StringHeader +type stringHeader struct { + data unsafe.Pointer + len int +} + +// sliceHeader instead of reflect.SliceHeader +type sliceHeader struct { + data unsafe.Pointer + len int + cap int +} + // getBytes casts the input json bytes to a string and safely returns the // results as uniquely allocated data. This operation is intended to minimize // copies and allocations for the large json string->[]byte. @@ -2845,14 +3095,14 @@ // unsafe cast to string result = Get(*(*string)(unsafe.Pointer(&json)), path) // safely get the string headers - rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw)) - strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str)) + rawhi := *(*stringHeader)(unsafe.Pointer(&result.Raw)) + strhi := *(*stringHeader)(unsafe.Pointer(&result.Str)) // create byte slice headers - rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len} - strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len} - if strh.Data == 0 { + rawh := sliceHeader{data: rawhi.data, len: rawhi.len, cap: rawhi.len} + strh := sliceHeader{data: strhi.data, len: strhi.len, cap: rawhi.len} + if strh.data == nil { // str is nil - if rawh.Data == 0 { + if rawh.data == nil { // raw is nil result.Raw = "" } else { @@ -2860,19 +3110,20 @@ result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) } result.Str = "" - } else if rawh.Data == 0 { + } else if rawh.data == nil { // raw is nil result.Raw = "" // str has data, safely copy the slice header to a string result.Str = string(*(*[]byte)(unsafe.Pointer(&strh))) - } else if strh.Data >= rawh.Data && - int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len { + } else if uintptr(strh.data) >= uintptr(rawh.data) && + uintptr(strh.data)+uintptr(strh.len) <= + uintptr(rawh.data)+uintptr(rawh.len) { // Str is a substring of Raw. - start := int(strh.Data - rawh.Data) + start := uintptr(strh.data) - uintptr(rawh.data) // safely copy the raw slice header result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) // substring the raw - result.Str = result.Raw[start : start+strh.Len] + result.Str = result.Raw[start : start+uintptr(strh.len)] } else { // safely copy both the raw and str slice headers to strings result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh))) @@ -2887,9 +3138,9 @@ // used instead. func fillIndex(json string, c *parseContext) { if len(c.value.Raw) > 0 && !c.calcd { - jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json)) - rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw))) - c.value.Index = int(rhdr.Data - jhdr.Data) + jhdr := *(*stringHeader)(unsafe.Pointer(&json)) + rhdr := *(*stringHeader)(unsafe.Pointer(&(c.value.Raw))) + c.value.Index = int(uintptr(rhdr.data) - uintptr(jhdr.data)) if c.value.Index < 0 || c.value.Index >= len(json) { c.value.Index = 0 } @@ -2897,13 +3148,212 @@ } func stringBytes(s string) []byte { - return *(*[]byte)(unsafe.Pointer(&reflect.SliceHeader{ - Data: (*reflect.StringHeader)(unsafe.Pointer(&s)).Data, - Len: len(s), - Cap: len(s), + return *(*[]byte)(unsafe.Pointer(&sliceHeader{ + data: (*stringHeader)(unsafe.Pointer(&s)).data, + len: len(s), + cap: len(s), })) } func bytesString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } + +func revSquash(json string) string { + // reverse squash + // expects that the tail character is a ']' or '}' or ')' or '"' + // squash the value, ignoring all nested arrays and objects. + i := len(json) - 1 + var depth int + if json[i] != '"' { + depth++ + } + if json[i] == '}' || json[i] == ']' || json[i] == ')' { + i-- + } + for ; i >= 0; i-- { + switch json[i] { + case '"': + i-- + for ; i >= 0; i-- { + if json[i] == '"' { + esc := 0 + for i > 0 && json[i-1] == '\\' { + i-- + esc++ + } + if esc%2 == 1 { + continue + } + i += esc + break + } + } + if depth == 0 { + if i < 0 { + i = 0 + } + return json[i:] + } + case '}', ']', ')': + depth++ + case '{', '[', '(': + depth-- + if depth == 0 { + return json[i:] + } + } + } + return json +} + +// Paths returns the original GJSON paths for a Result where the Result came +// from a simple query path that returns an array, like: +// +// gjson.Get(json, "friends.#.first") +// +// The returned value will be in the form of a JSON array: +// +// ["friends.0.first","friends.1.first","friends.2.first"] +// +// The param 'json' must be the original JSON used when calling Get. +// +// Returns an empty string if the paths cannot be determined, which can happen +// when the Result came from a path that contained a multipath, modifier, +// or a nested query. +func (t Result) Paths(json string) []string { + if t.Indexes == nil { + return nil + } + paths := make([]string, 0, len(t.Indexes)) + t.ForEach(func(_, value Result) bool { + paths = append(paths, value.Path(json)) + return true + }) + if len(paths) != len(t.Indexes) { + return nil + } + return paths +} + +// Path returns the original GJSON path for a Result where the Result came +// from a simple path that returns a single value, like: +// +// gjson.Get(json, "friends.#(last=Murphy)") +// +// The returned value will be in the form of a JSON string: +// +// "friends.0" +// +// The param 'json' must be the original JSON used when calling Get. +// +// Returns an empty string if the paths cannot be determined, which can happen +// when the Result came from a path that contained a multipath, modifier, +// or a nested query. +func (t Result) Path(json string) string { + var path []byte + var comps []string // raw components + i := t.Index - 1 + if t.Index+len(t.Raw) > len(json) { + // JSON cannot safely contain Result. + goto fail + } + if !strings.HasPrefix(json[t.Index:], t.Raw) { + // Result is not at the JSON index as exepcted. + goto fail + } + for ; i >= 0; i-- { + if json[i] <= ' ' { + continue + } + if json[i] == ':' { + // inside of object, get the key + for ; i >= 0; i-- { + if json[i] != '"' { + continue + } + break + } + raw := revSquash(json[:i+1]) + i = i - len(raw) + comps = append(comps, raw) + // key gotten, now squash the rest + raw = revSquash(json[:i+1]) + i = i - len(raw) + i++ // increment the index for next loop step + } else if json[i] == '{' { + // Encountered an open object. The original result was probably an + // object key. + goto fail + } else if json[i] == ',' || json[i] == '[' { + // inside of an array, count the position + var arrIdx int + if json[i] == ',' { + arrIdx++ + i-- + } + for ; i >= 0; i-- { + if json[i] == ':' { + // Encountered an unexpected colon. The original result was + // probably an object key. + goto fail + } else if json[i] == ',' { + arrIdx++ + } else if json[i] == '[' { + comps = append(comps, strconv.Itoa(arrIdx)) + break + } else if json[i] == ']' || json[i] == '}' || json[i] == '"' { + raw := revSquash(json[:i+1]) + i = i - len(raw) + 1 + } + } + } + } + if len(comps) == 0 { + if DisableModifiers { + goto fail + } + return "@this" + } + for i := len(comps) - 1; i >= 0; i-- { + rcomp := Parse(comps[i]) + if !rcomp.Exists() { + goto fail + } + comp := escapeComp(rcomp.String()) + path = append(path, '.') + path = append(path, comp...) + } + if len(path) > 0 { + path = path[1:] + } + return string(path) +fail: + return "" +} + +// isSafePathKeyChar returns true if the input character is safe for not +// needing escaping. +func isSafePathKeyChar(c byte) bool { + return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' || + (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') +} + +// escapeComp escaped a path compontent, making it safe for generating a +// path for later use. +func escapeComp(comp string) string { + for i := 0; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp := []byte(comp[:i]) + for ; i < len(comp); i++ { + if !isSafePathKeyChar(comp[i]) { + ncomp = append(ncomp, '\\') + } + ncomp = append(ncomp, comp[i]) + } + return string(ncomp) + } + } + return comp +} diff -Nru golang-github-tidwall-gjson-1.6.7/gjson_test.go golang-github-tidwall-gjson-1.14.4/gjson_test.go --- golang-github-tidwall-gjson-1.6.7/gjson_test.go 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/gjson_test.go 2022-11-22 01:54:56.000000000 +0000 @@ -5,6 +5,7 @@ "encoding/hex" "encoding/json" "fmt" + "math" "math/rand" "strconv" "strings" @@ -107,7 +108,7 @@ } // this json block is poorly formed on purpose. -var basicJSON = `{"age":100, "name":{"here":"B\\\"R"}, +var basicJSON = ` {"age":100, "name":{"here":"B\\\"R"}, "noop":{"what is a wren?":"a bird"}, "happy":true,"immortal":false, "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7], @@ -140,9 +141,66 @@ } ] }, - "lastly":{"yay":"final"} + "lastly":{"end...ing":"soon","yay":"final"} }` -var basicJSONB = []byte(basicJSON) + +func TestPath(t *testing.T) { + json := basicJSON + r := Get(json, "@this") + path := r.Path(json) + if path != "@this" { + t.FailNow() + } + + r = Parse(json) + path = r.Path(json) + if path != "@this" { + t.FailNow() + } + + obj := Parse(json) + obj.ForEach(func(key, val Result) bool { + kp := key.Path(json) + assert(t, kp == "") + vp := val.Path(json) + if vp == "name" { + // there are two "name" keys + return true + } + val2 := obj.Get(vp) + assert(t, val2.Raw == val.Raw) + return true + }) + arr := obj.Get("loggy.programmers") + arr.ForEach(func(_, val Result) bool { + vp := val.Path(json) + val2 := Get(json, vp) + assert(t, val2.Raw == val.Raw) + return true + }) + get := func(path string) { + r1 := Get(json, path) + path2 := r1.Path(json) + r2 := Get(json, path2) + assert(t, r1.Raw == r2.Raw) + } + get("age") + get("name") + get("name.here") + get("noop") + get("noop.what is a wren?") + get("arr.0") + get("arr.1") + get("arr.2") + get("arr.3") + get("arr.3.hello") + get("arr.4") + get("arr.5") + get("loggy.programmers.2.email") + get("lastly.end\\.\\.\\.ing") + get("lastly.yay") + +} func TestTimeResult(t *testing.T) { assert(t, Get(basicJSON, "created").String() == @@ -164,14 +222,11 @@ expects := []string{"a", "b", "c"} for _, count := range counts { var gpaths []string - var gexpects []string for i := 0; i < count; i++ { if i < len(paths) { gpaths = append(gpaths, paths[i]) - gexpects = append(gexpects, expects[i]) } else { gpaths = append(gpaths, fmt.Sprintf("not%d", i)) - gexpects = append(gexpects, "null") } } results := GetMany(json, gpaths...) @@ -380,9 +435,9 @@ mtok := get(basicJSON, `loggy.programmers`) var count int mtok.ForEach(func(key, value Result) bool { - if key.Exists() { - t.Fatalf("expected %v, got %v", false, key.Exists()) - } + assert(t, key.Exists()) + assert(t, key.String() == fmt.Sprint(count)) + assert(t, key.Int() == int64(count)) count++ if count == 3 { return false @@ -720,10 +775,6 @@ } }` -func TestNewParse(t *testing.T) { - //fmt.Printf("%v\n", parse2(exampleJSON, "widget").String()) -} - func TestUnmarshalMap(t *testing.T) { var m1 = Parse(exampleJSON).Value().(map[string]interface{}) var m2 map[string]interface{} @@ -738,7 +789,7 @@ if err != nil { t.Fatal(err) } - if bytes.Compare(b1, b2) != 0 { + if !bytes.Equal(b1, b2) { t.Fatal("b1 != b2") } } @@ -784,9 +835,8 @@ "name.first":"Cat", }` -func combine(results []Result) string { - return fmt.Sprintf("%v", results) -} +var testWatchForFallback bool + func TestManyBasic(t *testing.T) { testWatchForFallback = true defer func() { @@ -804,9 +854,6 @@ fmt.Printf("%v\n", paths) t.Fatalf("expected %v, got %v", expect, results) } - //if testLastWasFallback != shouldFallback { - // t.Fatalf("expected %v, got %v", shouldFallback, testLastWasFallback) - //} } testMany(false, "[Point]", "position.type") testMany(false, `[emptya ["world peace"] 31]`, ".a", "loves", "age") @@ -867,9 +914,9 @@ } func TestIssue21(t *testing.T) { - json := `{ "Level1Field1":3, - "Level1Field4":4, - "Level1Field2":{ "Level2Field1":[ "value1", "value2" ], + json := `{ "Level1Field1":3, + "Level1Field4":4, + "Level1Field2":{ "Level2Field1":[ "value1", "value2" ], "Level2Field2":{ "Level3Field1":[ { "key1":"value1" } ] } } }` paths := []string{"Level1Field1", "Level1Field2.Level2Field1", "Level1Field2.Level2Field2.Level3Field1", "Level1Field4"} @@ -917,49 +964,6 @@ } } -type ComplicatedType struct { - unsettable int - Tagged string `json:"tagged"` - NotTagged bool - Nested struct { - Yellow string `json:"yellow"` - } - NestedTagged struct { - Green string - Map map[string]interface{} - Ints struct { - Int int `json:"int"` - Int8 int8 - Int16 int16 - Int32 int32 - Int64 int64 `json:"int64"` - } - Uints struct { - Uint uint - Uint8 uint8 - Uint16 uint16 - Uint32 uint32 - Uint64 uint64 - } - Floats struct { - Float64 float64 - Float32 float32 - } - Byte byte - Bool bool - } `json:"nestedTagged"` - LeftOut string `json:"-"` - SelfPtr *ComplicatedType - SelfSlice []ComplicatedType - SelfSlicePtr []*ComplicatedType - SelfPtrSlice *[]ComplicatedType - Interface interface{} `json:"interface"` - Array [3]int - Time time.Time `json:"time"` - Binary []byte - NonBinary []byte -} - var complicatedJSON = ` { "tagged": "OK", @@ -973,7 +977,7 @@ "nestedTagged": { "Green": "Green", "Map": { - "this": "that", + "this": "that", "and": "the other thing" }, "Ints": { @@ -1024,6 +1028,7 @@ testvalid(t, "00", false) testvalid(t, "-00", false) testvalid(t, "-.", false) + testvalid(t, "-.123", false) testvalid(t, "0.0", true) testvalid(t, "10.0", true) testvalid(t, "10e1", true) @@ -1088,6 +1093,8 @@ testvalid(t, `"a\\b\\\uFFA"`, false) testvalid(t, string(complicatedJSON), true) testvalid(t, string(exampleJSON), true) + testvalid(t, "[-]", false) + testvalid(t, "[-.123]", false) } var jsonchars = []string{"{", "[", ",", ":", "}", "]", "1", "0", "true", @@ -1180,50 +1187,6 @@ } } -// func TestRandomGetMany(t *testing.T) { -// start := time.Now() -// for time.Since(start) < time.Second*3 { -// testRandomGetMany(t) -// } -// } -func testRandomGetMany(t *testing.T) { - rand.Seed(time.Now().UnixNano()) - json, keys := randomJSON() - for _, key := range keys { - r := Get(json, key) - if !r.Exists() { - t.Fatal("should exist") - } - } - rkeysi := rand.Perm(len(keys)) - rkeysn := 1 + rand.Int()%32 - if len(rkeysi) > rkeysn { - rkeysi = rkeysi[:rkeysn] - } - var rkeys []string - for i := 0; i < len(rkeysi); i++ { - rkeys = append(rkeys, keys[rkeysi[i]]) - } - mres1 := GetMany(json, rkeys...) - var mres2 []Result - for _, rkey := range rkeys { - mres2 = append(mres2, Get(json, rkey)) - } - if len(mres1) != len(mres2) { - t.Fatalf("expected %d, got %d", len(mres2), len(mres1)) - } - for i := 0; i < len(mres1); i++ { - mres1[i].Index = 0 - mres2[i].Index = 0 - v1 := fmt.Sprintf("%#v", mres1[i]) - v2 := fmt.Sprintf("%#v", mres2[i]) - if v1 != v2 { - t.Fatalf("\nexpected %s\n"+ - " got %s", v2, v1) - } - } -} - func TestIssue54(t *testing.T) { var r []Result json := `{"MarketName":null,"Nounce":6115}` @@ -1244,93 +1207,6 @@ } } -func randomString() string { - var key string - N := 1 + rand.Int()%16 - for i := 0; i < N; i++ { - r := rand.Int() % 62 - if r < 10 { - key += string(byte('0' + r)) - } else if r-10 < 26 { - key += string(byte('a' + r - 10)) - } else { - key += string(byte('A' + r - 10 - 26)) - } - } - return `"` + key + `"` -} -func randomBool() string { - switch rand.Int() % 2 { - default: - return "false" - case 1: - return "true" - } -} -func randomNumber() string { - return strconv.FormatInt(int64(rand.Int()%1000000), 10) -} - -func randomObjectOrArray(keys []string, prefix string, array bool, depth int) ( - string, []string) { - N := 5 + rand.Int()%5 - var json string - if array { - json = "[" - } else { - json = "{" - } - for i := 0; i < N; i++ { - if i > 0 { - json += "," - } - var pkey string - if array { - pkey = prefix + "." + strconv.FormatInt(int64(i), 10) - } else { - key := randomString() - pkey = prefix + "." + key[1:len(key)-1] - json += key + `:` - } - keys = append(keys, pkey[1:]) - var kind int - if depth == 5 { - kind = rand.Int() % 4 - } else { - kind = rand.Int() % 6 - } - switch kind { - case 0: - json += randomString() - case 1: - json += randomBool() - case 2: - json += "null" - case 3: - json += randomNumber() - case 4: - var njson string - njson, keys = randomObjectOrArray(keys, pkey, true, depth+1) - json += njson - case 5: - var njson string - njson, keys = randomObjectOrArray(keys, pkey, false, depth+1) - json += njson - } - - } - if array { - json += "]" - } else { - json += "}" - } - return json, keys -} - -func randomJSON() (json string, keys []string) { - return randomObjectOrArray(nil, "", false, 0) -} - func TestIssue55(t *testing.T) { json := `{"one": {"two": 2, "three": 3}, "four": 4, "five": 5}` results := GetMany(json, "four", "five", "one.two", "one.six") @@ -1444,7 +1320,7 @@ } func TestDuplicateKeys(t *testing.T) { - // this is vaild json according to the JSON spec + // this is valid json according to the JSON spec var json = `{"name": "Alex","name": "Peter"}` if Parse(json).Get("name").String() != Parse(json).Map()["name"].String() { @@ -1470,10 +1346,10 @@ } expect := strings.Join([]string{ `gjson.Result{Type:3, Raw:"\"PERSON1\"", Str:"PERSON1", Num:0, ` + - `Index:0}`, + `Index:11, Indexes:[]int(nil)}`, `gjson.Result{Type:3, Raw:"\"PERSON2\"", Str:"PERSON2", Num:0, ` + - `Index:0}`, - `gjson.Result{Type:2, Raw:"0", Str:"", Num:0, Index:0}`, + `Index:21, Indexes:[]int(nil)}`, + `gjson.Result{Type:2, Raw:"0", Str:"", Num:0, Index:31, Indexes:[]int(nil)}`, }, "\n") if output != expect { t.Fatalf("expected '%v', got '%v'", expect, output) @@ -1671,7 +1547,7 @@ } }, { - "first": "Roger", "last": "Craig", + "first": "Roger", "last": "Craig", "extra": [40,50,60], "details": { "city": "Phoenix", @@ -1921,7 +1797,7 @@ var path, op, value, remain string var ok bool - path, op, value, remain, _, ok = + path, op, value, remain, _, _, ok = parseQuery(`#(service_roles.#(=="one").()==asdf).cap`) assert(t, ok && path == `service_roles.#(=="one").()` && @@ -1929,28 +1805,28 @@ value == `asdf` && remain == `.cap`) - path, op, value, remain, _, ok = parseQuery(`#(first_name%"Murphy").last`) + path, op, value, remain, _, _, ok = parseQuery(`#(first_name%"Murphy").last`) assert(t, ok && path == `first_name` && op == `%` && value == `"Murphy"` && remain == `.last`) - path, op, value, remain, _, ok = parseQuery(`#( first_name !% "Murphy" ).last`) + path, op, value, remain, _, _, ok = parseQuery(`#( first_name !% "Murphy" ).last`) assert(t, ok && path == `first_name` && op == `!%` && value == `"Murphy"` && remain == `.last`) - path, op, value, remain, _, ok = parseQuery(`#(service_roles.#(=="one"))`) + path, op, value, remain, _, _, ok = parseQuery(`#(service_roles.#(=="one"))`) assert(t, ok && path == `service_roles.#(=="one")` && op == `` && value == `` && remain == ``) - path, op, value, remain, _, ok = + path, op, value, remain, _, _, ok = parseQuery(`#(a\("\"(".#(=="o\"(ne")%"ab\")").remain`) assert(t, ok && path == `a\("\"(".#(=="o\"(ne")` && @@ -2016,6 +1892,9 @@ exp = `{"first":"DALE"}` assert(t, res.Raw == exp) + res = Get(readmeJSON, `{"children":children|@case:upper,"name":name.first,"age":age}`) + exp = `{"children":["SARA","ALEX","JACK"],"name":"Tom","age":37}` + assert(t, res.Raw == exp) } func TestIssue141(t *testing.T) { @@ -2203,4 +2082,512 @@ testJSON = `[#.@pretty.@join:{""[]""preserve"3,"][{]]]` Get(testJSON, testJSON) + // Issue #237 + testJSON1 := `["*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,,,,,,"]` + testJSON2 := `#[%"*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,,,,,,""*,*"]` + Get(testJSON1, testJSON2) + +} + +func TestSubpathsWithMultipaths(t *testing.T) { + const json = ` +[ + {"a": 1}, + {"a": 2, "values": ["a", "b", "c", "d", "e"]}, + true, + ["a", "b", "c", "d", "e"], + 4 +] +` + assert(t, Get(json, `1.values.@ugly`).Raw == `["a","b","c","d","e"]`) + assert(t, Get(json, `1.values.[0,3]`).Raw == `["a","d"]`) + assert(t, Get(json, `3.@ugly`).Raw == `["a","b","c","d","e"]`) + assert(t, Get(json, `3.[0,3]`).Raw == `["a","d"]`) + assert(t, Get(json, `#.@ugly`).Raw == `[{"a":1},{"a":2,"values":["a","b","c","d","e"]},true,["a","b","c","d","e"],4]`) + assert(t, Get(json, `#.[0,3]`).Raw == `[[],[],[],["a","d"],[]]`) +} + +func TestFlattenRemoveNonExist(t *testing.T) { + raw := Get("[[1],[2,[[],[3]],[4,[5],[],[[[6]]]]]]", `@flatten:{"deep":true}`).Raw + assert(t, raw == "[1,2,3,4,5,6]") +} + +func TestPipeEmptyArray(t *testing.T) { + raw := Get("[]", `#(hello)#`).Raw + assert(t, raw == "[]") +} + +func TestEncodedQueryString(t *testing.T) { + json := `{ + "friends": [ + {"first": "Dale", "last": "Mur\nphy", "age": 44}, + {"first": "Roger", "last": "Craig", "age": 68}, + {"first": "Jane", "last": "Murphy", "age": 47} + ] + }` + assert(t, Get(json, `friends.#(last=="Mur\nphy").age`).Int() == 44) + assert(t, Get(json, `friends.#(last=="Murphy").age`).Int() == 47) +} + +func TestBoolConvertQuery(t *testing.T) { + json := `{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] + }` + trues := Get(json, `vals.#(b==~true)#.a`).Raw + falses := Get(json, `vals.#(b==~false)#.a`).Raw + assert(t, trues == "[1,2,6,7,8]") + assert(t, falses == "[3,4,5,9,10,11]") +} + +func TestModifierDoubleQuotes(t *testing.T) { + josn := `{ + "data": [ + { + "name": "Product P4", + "productId": "1bb3", + "vendorId": "10de" + }, + { + "name": "Product P4", + "productId": "1cc3", + "vendorId": "20de" + }, + { + "name": "Product P4", + "productId": "1dd3", + "vendorId": "30de" + } + ] + }` + AddModifier("string", func(josn, arg string) string { + return strconv.Quote(josn) + }) + + res := Get(josn, "data.#.{name,value:{productId,vendorId}.@string.@ugly}") + + assert(t, res.Raw == `[`+ + `{"name":"Product P4","value":"{\"productId\":\"1bb3\",\"vendorId\":\"10de\"}"},`+ + `{"name":"Product P4","value":"{\"productId\":\"1cc3\",\"vendorId\":\"20de\"}"},`+ + `{"name":"Product P4","value":"{\"productId\":\"1dd3\",\"vendorId\":\"30de\"}"}`+ + `]`) + +} + +func TestIndexes(t *testing.T) { + var exampleJSON = `{ + "vals": [ + [1,66,{test: 3}], + [4,5,[6]] + ], + "objectArray":[ + {"first": "Dale", "age": 44}, + {"first": "Roger", "age": 68}, + ] + }` + + testCases := []struct { + path string + expected []string + }{ + { + `vals.#.1`, + []string{`6`, "5"}, + }, + { + `vals.#.2`, + []string{"{", "["}, + }, + { + `objectArray.#(age>43)#.first`, + []string{`"`, `"`}, + }, + { + `objectArray.@reverse.#.first`, + nil, + }, + } + + for _, tc := range testCases { + r := Get(exampleJSON, tc.path) + + assert(t, len(r.Indexes) == len(tc.expected)) + + for i, a := range r.Indexes { + assert(t, string(exampleJSON[a]) == tc.expected[i]) + } + } +} + +func TestIndexesMatchesRaw(t *testing.T) { + var exampleJSON = `{ + "objectArray":[ + {"first": "Jason", "age": 41}, + {"first": "Dale", "age": 44}, + {"first": "Roger", "age": 68}, + {"first": "Mandy", "age": 32} + ] + }` + r := Get(exampleJSON, `objectArray.#(age>43)#.first`) + assert(t, len(r.Indexes) == 2) + assert(t, Parse(exampleJSON[r.Indexes[0]:]).String() == "Dale") + assert(t, Parse(exampleJSON[r.Indexes[1]:]).String() == "Roger") + r = Get(exampleJSON, `objectArray.#(age>43)#`) + assert(t, Parse(exampleJSON[r.Indexes[0]:]).Get("first").String() == "Dale") + assert(t, Parse(exampleJSON[r.Indexes[1]:]).Get("first").String() == "Roger") +} + +func TestIssue240(t *testing.T) { + nonArrayData := `{"jsonrpc":"2.0","method":"subscription","params": + {"channel":"funny_channel","data": + {"name":"Jason","company":"good_company","number":12345} + } + }` + parsed := Parse(nonArrayData) + assert(t, len(parsed.Get("params.data").Array()) == 1) + + arrayData := `{"jsonrpc":"2.0","method":"subscription","params": + {"channel":"funny_channel","data":[ + {"name":"Jason","company":"good_company","number":12345} + ]} + }` + parsed = Parse(arrayData) + assert(t, len(parsed.Get("params.data").Array()) == 1) +} + +func TestKeysValuesModifier(t *testing.T) { + var json = `{ + "1300014": { + "code": "1300014", + "price": 59.18, + "symbol": "300014", + "update": "2020/04/15 15:59:54", + }, + "1300015": { + "code": "1300015", + "price": 43.31, + "symbol": "300015", + "update": "2020/04/15 15:59:54", + } + }` + assert(t, Get(json, `@keys`).String() == `["1300014","1300015"]`) + assert(t, Get(``, `@keys`).String() == `[]`) + assert(t, Get(`"hello"`, `@keys`).String() == `[null]`) + assert(t, Get(`[]`, `@keys`).String() == `[]`) + assert(t, Get(`[1,2,3]`, `@keys`).String() == `[null,null,null]`) + + assert(t, Get(json, `@values.#.code`).String() == `["1300014","1300015"]`) + assert(t, Get(``, `@values`).String() == `[]`) + assert(t, Get(`"hello"`, `@values`).String() == `["hello"]`) + assert(t, Get(`[]`, `@values`).String() == `[]`) + assert(t, Get(`[1,2,3]`, `@values`).String() == `[1,2,3]`) +} + +func TestNaNInf(t *testing.T) { + json := `[+Inf,-Inf,Inf,iNF,-iNF,+iNF,NaN,nan,nAn,-0,+0]` + raws := []string{"+Inf", "-Inf", "Inf", "iNF", "-iNF", "+iNF", "NaN", "nan", + "nAn", "-0", "+0"} + nums := []float64{math.Inf(+1), math.Inf(-1), math.Inf(0), math.Inf(0), + math.Inf(-1), math.Inf(+1), math.NaN(), math.NaN(), math.NaN(), + math.Copysign(0, -1), 0} + + assert(t, int(Get(json, `#`).Int()) == len(raws)) + for i := 0; i < len(raws); i++ { + r := Get(json, fmt.Sprintf("%d", i)) + assert(t, r.Raw == raws[i]) + assert(t, r.Num == nums[i] || (math.IsNaN(r.Num) && math.IsNaN(nums[i]))) + assert(t, r.Type == Number) + } + + var i int + Parse(json).ForEach(func(_, r Result) bool { + assert(t, r.Raw == raws[i]) + assert(t, r.Num == nums[i] || (math.IsNaN(r.Num) && math.IsNaN(nums[i]))) + assert(t, r.Type == Number) + i++ + return true + }) + + // Parse should also return valid numbers + assert(t, math.IsNaN(Parse("nan").Float())) + assert(t, math.IsNaN(Parse("NaN").Float())) + assert(t, math.IsNaN(Parse(" NaN").Float())) + assert(t, math.IsInf(Parse("+inf").Float(), +1)) + assert(t, math.IsInf(Parse("-inf").Float(), -1)) + assert(t, math.IsInf(Parse("+INF").Float(), +1)) + assert(t, math.IsInf(Parse("-INF").Float(), -1)) + assert(t, math.IsInf(Parse(" +INF").Float(), +1)) + assert(t, math.IsInf(Parse(" -INF").Float(), -1)) +} + +func TestEmptyValueQuery(t *testing.T) { + // issue: https://github.com/tidwall/gjson/issues/246 + assert(t, Get( + `["ig","","tw","fb","tw","ig","tw"]`, + `#(!="")#`).Raw == + `["ig","tw","fb","tw","ig","tw"]`) + assert(t, Get( + `["ig","","tw","fb","tw","ig","tw"]`, + `#(!=)#`).Raw == + `["ig","tw","fb","tw","ig","tw"]`) +} + +func TestParseIndex(t *testing.T) { + assert(t, Parse(`{}`).Index == 0) + assert(t, Parse(` {}`).Index == 1) + assert(t, Parse(` []`).Index == 1) + assert(t, Parse(` true`).Index == 1) + assert(t, Parse(` false`).Index == 1) + assert(t, Parse(` null`).Index == 1) + assert(t, Parse(` +inf`).Index == 1) + assert(t, Parse(` -inf`).Index == 1) +} + +func TestRevSquash(t *testing.T) { + assert(t, revSquash(` {}`) == `{}`) + assert(t, revSquash(` }`) == ` }`) + assert(t, revSquash(` [123]`) == `[123]`) + assert(t, revSquash(` ,123,123]`) == ` ,123,123]`) + assert(t, revSquash(` hello,[[true,false],[0,1,2,3,5],[123]]`) == `[[true,false],[0,1,2,3,5],[123]]`) + assert(t, revSquash(` "hello"`) == `"hello"`) + assert(t, revSquash(` "hel\\lo"`) == `"hel\\lo"`) + assert(t, revSquash(` "hel\\"lo"`) == `"lo"`) + assert(t, revSquash(` "hel\\\"lo"`) == `"hel\\\"lo"`) + assert(t, revSquash(`hel\\\"lo"`) == `hel\\\"lo"`) + assert(t, revSquash(`\"hel\\\"lo"`) == `\"hel\\\"lo"`) + assert(t, revSquash(`\\\"hel\\\"lo"`) == `\\\"hel\\\"lo"`) + assert(t, revSquash(`\\\\"hel\\\"lo"`) == `"hel\\\"lo"`) + assert(t, revSquash(`hello"`) == `hello"`) + json := `true,[0,1,"sadf\"asdf",{"hi":["hello","t\"\"u",{"a":"b"}]},9]` + assert(t, revSquash(json) == json[5:]) + assert(t, revSquash(json[:len(json)-3]) == `{"hi":["hello","t\"\"u",{"a":"b"}]}`) + assert(t, revSquash(json[:len(json)-4]) == `["hello","t\"\"u",{"a":"b"}]`) + assert(t, revSquash(json[:len(json)-5]) == `{"a":"b"}`) + assert(t, revSquash(json[:len(json)-6]) == `"b"`) + assert(t, revSquash(json[:len(json)-10]) == `"a"`) + assert(t, revSquash(json[:len(json)-15]) == `"t\"\"u"`) + assert(t, revSquash(json[:len(json)-24]) == `"hello"`) + assert(t, revSquash(json[:len(json)-33]) == `"hi"`) + assert(t, revSquash(json[:len(json)-39]) == `"sadf\"asdf"`) +} + +const readmeJSON = ` +{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] +} +` + +func TestQueryGetPath(t *testing.T) { + assert(t, strings.Join( + Get(readmeJSON, "friends.#.first").Paths(readmeJSON), " ") == + "friends.0.first friends.1.first friends.2.first") + assert(t, strings.Join( + Get(readmeJSON, "friends.#(last=Murphy)").Paths(readmeJSON), " ") == + "") + assert(t, Get(readmeJSON, "friends.#(last=Murphy)").Path(readmeJSON) == + "friends.0") + assert(t, strings.Join( + Get(readmeJSON, "friends.#(last=Murphy)#").Paths(readmeJSON), " ") == + "friends.0 friends.2") + arr := Get(readmeJSON, "friends.#.first").Array() + for i := 0; i < len(arr); i++ { + assert(t, arr[i].Path(readmeJSON) == fmt.Sprintf("friends.%d.first", i)) + } +} + +func TestStaticJSON(t *testing.T) { + json := `{ + "name": {"first": "Tom", "last": "Anderson"} + }` + assert(t, Get(json, + `"bar"`).Raw == + ``) + assert(t, Get(json, + `!"bar"`).Raw == + `"bar"`) + assert(t, Get(json, + `!{"name":{"first":"Tom"}}.{name.first}.first`).Raw == + `"Tom"`) + assert(t, Get(json, + `{name.last,"foo":!"bar"}`).Raw == + `{"last":"Anderson","foo":"bar"}`) + assert(t, Get(json, + `{name.last,"foo":!{"a":"b"},"that"}`).Raw == + `{"last":"Anderson","foo":{"a":"b"}}`) + assert(t, Get(json, + `{name.last,"foo":!{"c":"d"},!"that"}`).Raw == + `{"last":"Anderson","foo":{"c":"d"},"_":"that"}`) + assert(t, Get(json, + `[!true,!false,!null,!inf,!nan,!hello,{"name":!"andy",name.last},+inf,!["any","thing"]]`).Raw == + `[true,false,null,inf,nan,{"name":"andy","last":"Anderson"},["any","thing"]]`, + ) +} + +func TestArrayKeys(t *testing.T) { + N := 100 + json := "[" + for i := 0; i < N; i++ { + if i > 0 { + json += "," + } + json += fmt.Sprint(i) + } + json += "]" + var i int + Parse(json).ForEach(func(key, value Result) bool { + assert(t, key.String() == fmt.Sprint(i)) + assert(t, key.Int() == int64(i)) + i++ + return true + }) + assert(t, i == N) +} + +func TestToFromStr(t *testing.T) { + json := `{"Message":"{\"Records\":[{\"eventVersion\":\"2.1\"}]"}` + res := Get(json, "Message.@fromstr.Records.#.eventVersion.@tostr").Raw + assert(t, res == `["\"2.1\""]`) +} + +func TestGroup(t *testing.T) { + json := `{"id":["123","456","789"],"val":[2,1]}` + res := Get(json, "@group").Raw + assert(t, res == `[{"id":"123","val":2},{"id":"456","val":1},{"id":"789"}]`) + + json = ` +{ + "issues": [ + { + "fields": { + "labels": [ + "milestone_1", + "group:foo", + "plan:a", + "plan:b" + ] + }, + "id": "123" + },{ + "fields": { + "labels": [ + "milestone_1", + "group:foo", + "plan:a", + "plan" + ] + }, + "id": "456" + } + ] + } + ` + res = Get(json, `{"id":issues.#.id,"plans":issues.#.fields.labels.#(%"plan:*")#|#.#}|@group|#(plans>=2)#.id`).Raw + assert(t, res == `["123"]`) +} + +func testJSONString(t *testing.T, str string) { + gjsonString := string(AppendJSONString(nil, str)) + data, err := json.Marshal(str) + if err != nil { + panic(123) + } + goString := string(data) + if gjsonString != goString { + t.Fatal(strconv.Quote(str) + "\n\t" + + gjsonString + "\n\t" + + goString + "\n\t<<< MISMATCH >>>") + } +} + +func TestJSONString(t *testing.T) { + testJSONString(t, "hello") + testJSONString(t, "he\"llo") + testJSONString(t, "he\"l\\lo") + const input = `{"utf8":"Example emoji, KO: \ud83d\udd13, \ud83c\udfc3 ` + + `OK: \u2764\ufe0f "}` + value := Get(input, "utf8") + var s string + json.Unmarshal([]byte(value.Raw), &s) + if value.String() != s { + t.Fatalf("expected '%v', got '%v'", s, value.String()) + } + testJSONString(t, s) + testJSONString(t, "R\xfd\xfc\a!\x82eO\x16?_\x0f\x9ab\x1dr") + testJSONString(t, "_\xb9\v\xad\xb3|X!\xb6\xd9U&\xa4\x1a\x95\x04") + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + start := time.Now() + var buf [16]byte + for time.Since(start) < time.Second*2 { + if _, err := rng.Read(buf[:]); err != nil { + t.Fatal(err) + } + testJSONString(t, string(buf[:])) + } +} + +func TestIndexAtSymbol(t *testing.T) { + json := `{ + "@context": { + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "@vocab": "http://schema.org/", + "sh": "http://www.w3.org/ns/shacl#" + } + }` + assert(t, Get(json, "@context.@vocab").Index == 85) +} + +func TestDeepModifierWithOptions(t *testing.T) { + rawJson := `{"x":[{"y":[{"z":{"b":1, "c": 2, "a": 3}}]}]}` + jsonPathExpr := `x.#.y.#.z.@pretty:{"sortKeys":true}` + results := GetManyBytes([]byte(rawJson), jsonPathExpr) + assert(t, len(results) == 1) + actual := results[0].Raw + expected := `[[{ + "a": 3, + "b": 1, + "c": 2 +} +]]` + if expected != actual { + t.Fatal(strconv.Quote(rawJson) + "\n\t" + + expected + "\n\t" + + actual + "\n\t<<< MISMATCH >>>") + } +} + +func TestIssue301(t *testing.T) { + json := `{ + "children": ["Sara","Alex","Jack"], + "fav.movie": ["Deer Hunter"] + }` + + assert(t, Get(json, `children.0`).String() == "Sara") + assert(t, Get(json, `children.[0]`).String() == `["Sara"]`) + assert(t, Get(json, `children.1`).String() == "Alex") + assert(t, Get(json, `children.[1]`).String() == `["Alex"]`) + assert(t, Get(json, `children.[10]`).String() == `[]`) + assert(t, Get(json, `fav\.movie.0`).String() == "Deer Hunter") + assert(t, Get(json, `fav\.movie.[0]`).String() == `["Deer Hunter"]`) + assert(t, Get(json, `fav\.movie.1`).String() == "") + assert(t, Get(json, `fav\.movie.[1]`).String() == "[]") + } diff -Nru golang-github-tidwall-gjson-1.6.7/go.mod golang-github-tidwall-gjson-1.14.4/go.mod --- golang-github-tidwall-gjson-1.6.7/go.mod 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/go.mod 2022-11-22 01:54:56.000000000 +0000 @@ -3,6 +3,6 @@ go 1.12 require ( - github.com/tidwall/match v1.0.3 - github.com/tidwall/pretty v1.0.2 + github.com/tidwall/match v1.1.1 + github.com/tidwall/pretty v1.2.0 ) diff -Nru golang-github-tidwall-gjson-1.6.7/go.sum golang-github-tidwall-gjson-1.14.4/go.sum --- golang-github-tidwall-gjson-1.6.7/go.sum 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/go.sum 2022-11-22 01:54:56.000000000 +0000 @@ -1,6 +1,4 @@ -github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= -github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= -github.com/tidwall/match v1.0.3 h1:FQUVvBImDutD8wJLN6c5eMzWtjgONK9MwIBCOrUJKeE= -github.com/tidwall/match v1.0.3/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU= -github.com/tidwall/pretty v1.0.2/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= diff -Nru golang-github-tidwall-gjson-1.6.7/README.md golang-github-tidwall-gjson-1.14.4/README.md --- golang-github-tidwall-gjson-1.6.7/README.md 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/README.md 2022-11-22 01:54:56.000000000 +0000 @@ -4,7 +4,9 @@ width="240" height="78" border="0" alt="GJSON">
GoDoc -GJSON Playground +GJSON Playground +GJSON Syntax +

get json values quickly

@@ -14,6 +16,10 @@ Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool. +This README is a quick overview of how to use GJSON, for more information check out [GJSON Syntax](SYNTAX.md). + +GJSON is also available for [Python](https://github.com/volans-/gjson-py) and [Rust](https://github.com/tidwall/gjson.rs) + Getting Started =============== @@ -123,11 +129,12 @@ To directly access the value: ```go -result.Type // can be String, Number, True, False, Null, or JSON -result.Str // holds the string -result.Num // holds the float64 number -result.Raw // holds the raw json -result.Index // index of raw value in original json, zero means index unknown +result.Type // can be String, Number, True, False, Null, or JSON +result.Str // holds the string +result.Num // holds the float64 number +result.Raw // holds the raw json +result.Index // index of raw value in original json, zero means index unknown +result.Indexes // indexes of all the elements that match on a path containing the '#' query character. ``` There are a variety of handy functions that work on a result: @@ -150,10 +157,6 @@ The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types: -The `result.Array()` function returns back an array of values. -If the result represents a non-existent value, then an empty array will be returned. -If the result is not a JSON array, the return value will be an array containing one result. - ```go boolean >> bool number >> float64 @@ -163,13 +166,17 @@ object >> map[string]interface{} ``` +The `result.Array()` function returns back an array of values. +If the result represents a non-existent value, then an empty array will be returned. +If the result is not a JSON array, the return value will be an array containing one result. + ### 64-bit integers The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers. ```go result.Int() int64 // -9223372036854775808 to 9223372036854775807 -result.Uint() int64 // 0 to 18446744073709551615 +result.Uint() uint64 // 0 to 18446744073709551615 ``` ## Modifiers and path chaining @@ -199,6 +206,11 @@ - `@valid`: Ensure the json document is valid. - `@flatten`: Flattens an array. - `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. +- `@tostr`: Converts json to a string. Wraps a json string. +- `@fromstr`: Converts a string from json. Unwraps a json string. +- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). ### Modifier arguments @@ -433,14 +445,15 @@ and [json-iterator](https://github.com/json-iterator/go) ``` -BenchmarkGJSONGet-8 3000000 372 ns/op 0 B/op 0 allocs/op -BenchmarkGJSONUnmarshalMap-8 900000 4154 ns/op 1920 B/op 26 allocs/op -BenchmarkJSONUnmarshalMap-8 600000 9019 ns/op 3048 B/op 69 allocs/op -BenchmarkJSONDecoder-8 300000 14120 ns/op 4224 B/op 184 allocs/op -BenchmarkFFJSONLexer-8 1500000 3111 ns/op 896 B/op 8 allocs/op -BenchmarkEasyJSONLexer-8 3000000 887 ns/op 613 B/op 6 allocs/op -BenchmarkJSONParserGet-8 3000000 499 ns/op 21 B/op 0 allocs/op -BenchmarkJSONIterator-8 3000000 812 ns/op 544 B/op 9 allocs/op +BenchmarkGJSONGet-16 11644512 311 ns/op 0 B/op 0 allocs/op +BenchmarkGJSONUnmarshalMap-16 1122678 3094 ns/op 1920 B/op 26 allocs/op +BenchmarkJSONUnmarshalMap-16 516681 6810 ns/op 2944 B/op 69 allocs/op +BenchmarkJSONUnmarshalStruct-16 697053 5400 ns/op 928 B/op 13 allocs/op +BenchmarkJSONDecoder-16 330450 10217 ns/op 3845 B/op 160 allocs/op +BenchmarkFFJSONLexer-16 1424979 2585 ns/op 880 B/op 8 allocs/op +BenchmarkEasyJSONLexer-16 3000000 729 ns/op 501 B/op 5 allocs/op +BenchmarkJSONParserGet-16 3000000 366 ns/op 21 B/op 0 allocs/op +BenchmarkJSONIterator-16 3000000 869 ns/op 693 B/op 14 allocs/op ``` JSON document used: @@ -481,12 +494,4 @@ widget.text.onMouseUp ``` -*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* - - -## Contact -Josh Baker [@tidwall](http://twitter.com/tidwall) - -## License - -GJSON source code is available under the MIT [License](/LICENSE). +*These benchmarks were run on a MacBook Pro 16" 2.4 GHz Intel Core i9 using Go 1.17 and can be found [here](https://github.com/tidwall/gjson-benchmarks).* diff -Nru golang-github-tidwall-gjson-1.6.7/SYNTAX.md golang-github-tidwall-gjson-1.14.4/SYNTAX.md --- golang-github-tidwall-gjson-1.6.7/SYNTAX.md 2020-12-25 13:42:20.000000000 +0000 +++ golang-github-tidwall-gjson-1.14.4/SYNTAX.md 2022-11-22 01:54:56.000000000 +0000 @@ -13,16 +13,16 @@ - [Dot vs Pipe](#dot-vs-pipe) - [Modifiers](#modifiers) - [Multipaths](#multipaths) +- [Literals](#literals) The definitive implemenation is [github.com/tidwall/gjson](https://github.com/tidwall/gjson). Use the [GJSON Playground](https://gjson.dev) to experiment with the syntax online. - ## Path structure A GJSON Path is intended to be easily expressed as a series of components seperated by a `.` character. -Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, and `?`. +Along with `.` character, there are a few more that have special meaning, including `|`, `#`, `@`, `\`, `*`, `!`, and `?`. ## Example @@ -77,14 +77,21 @@ fav\.movie "Deer Hunter" ``` -You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in source code. +You'll also need to make sure that the `\` character is correctly escaped when hardcoding a path in your source code. ```go -res := gjson.Get(json, "fav\\.movie") // must escape the slash -res := gjson.Get(json, `fav\.movie`) // no need to escape the slash +// Go +val := gjson.Get(json, "fav\\.movie") // must escape the slash +val := gjson.Get(json, `fav\.movie`) // no need to escape the slash +``` +```rust +// Rust +let val = gjson::get(json, "fav\\.movie") // must escape the slash +let val = gjson::get(json, r#"fav\.movie"#) // no need to escape the slash ``` + ### Arrays The `#` character allows for digging into JSON Arrays. @@ -128,6 +135,37 @@ syntax. For backwards compatibility, `#[...]` will continue to work until the next major release.* +The `~` (tilde) operator will convert a value to a boolean before comparison. + +For example, using the following JSON: + +```json +{ + "vals": [ + { "a": 1, "b": true }, + { "a": 2, "b": true }, + { "a": 3, "b": false }, + { "a": 4, "b": "0" }, + { "a": 5, "b": 0 }, + { "a": 6, "b": "1" }, + { "a": 7, "b": 1 }, + { "a": 8, "b": "true" }, + { "a": 9, "b": false }, + { "a": 10, "b": null }, + { "a": 11 } + ] +} +``` + +You can now query for all true(ish) or false(ish) values: + +``` +vals.#(b==~true)#.a >> [1,2,6,7,8] +vals.#(b==~false)#.a >> [3,4,5,9,10,11] +``` + +The last value which was non-existent is treated as `false` + ### Dot vs Pipe The `.` is standard separator, but it's also possible to use a `|`. @@ -198,6 +236,11 @@ - `@valid`: Ensure the json document is valid. - `@flatten`: Flattens an array. - `@join`: Joins multiple objects into a single object. +- `@keys`: Returns an array of keys for an object. +- `@values`: Returns an array of values for an object. +- `@tostr`: Converts json to a string. Wraps a json string. +- `@fromstr`: Converts a string from json. Unwraps a json string. +- `@group`: Groups arrays of objects. See [e4fc67c](https://github.com/tidwall/gjson/commit/e4fc67c92aeebf2089fabc7872f010e340d105db). #### Modifier arguments @@ -248,13 +291,15 @@ "children.@case:lower.@reverse" ["jack","alex","sara"] ``` +*Note: Custom modifiers are not yet available in the Rust version* + ### Multipaths Starting with v1.3.0, GJSON added the ability to join multiple paths together -to form new documents. Wrapping comma-separated paths between `{...}` or -`[...]` will result in a new array or object, respectively. +to form new documents. Wrapping comma-separated paths between `[...]` or +`{...}` will result in a new array or object, respectively. -For example, using the given multipath +For example, using the given multipath: ``` {name.first,age,"the_murphys":friends.#(last="Murphy")#.first} @@ -270,8 +315,28 @@ This results in -``` +```json {"first":"Tom","age":37,"the_murphys":["Dale","Jane"]} ``` +### Literals + +Starting with v1.12.0, GJSON added support of json literals, which provides a way for constructing static blocks of json. This is can be particularly useful when constructing a new json document using [multipaths](#multipaths). + +A json literal begins with the '!' declaration character. + +For example, using the given multipath: + +``` +{name.first,age,"company":!"Happysoft","employed":!true} +``` + +Here we selected the first name and age. Then add two new fields, "company" and "employed". + +This results in + +```json +{"first":"Tom","age":37,"company":"Happysoft","employed":true} +``` +*See issue [#249](https://github.com/tidwall/gjson/issues/249) for additional context on JSON Literals.*