upgrade deps; rewrite smtp session

vendor/github.com/buger/jsonparser/.travis.yml

@@ -1,4 +1,7 @@
 language: go
+arch:
+    - amd64
+    - ppc64le
 go:
     - 1.7.x
     - 1.8.x

vendor/github.com/buger/jsonparser/README.md

@@ -1,5 +1,5 @@
 [![Go Report Card](https://goreportcard.com/badge/github.com/buger/jsonparser)](https://goreportcard.com/report/github.com/buger/jsonparser) ![License](https://img.shields.io/dub/l/vibe-d.svg)
-# Alternative JSON parser for Go (so far fastest)
+# Alternative JSON parser for Go (10x times faster standard library)
 
 It does not require you to know the structure of the payload (eg. create structs), and allows accessing fields by providing the path to them. It is up to **10 times faster** than standard `encoding/json` package (depending on payload size and usage), **allocates no memory**. See benchmarks below.
 

vendor/github.com/buger/jsonparser/bytes_unsafe.go

@@ -6,6 +6,7 @@ import (
 	"reflect"
 	"strconv"
 	"unsafe"
+	"runtime"
 )
 
 //
@@ -32,11 +33,12 @@ func bytesToString(b *[]byte) string {
 }
 
 func StringToBytes(s string) []byte {
+	b := make([]byte, 0, 0)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
 	sh := (*reflect.StringHeader)(unsafe.Pointer(&s))
-	bh := reflect.SliceHeader{
-		Data: sh.Data,
-		Len:  sh.Len,
-		Cap:  sh.Len,
-	}
-	return *(*[]byte)(unsafe.Pointer(&bh))
+	bh.Data = sh.Data
+	bh.Cap = sh.Len
+	bh.Len = sh.Len
+	runtime.KeepAlive(s)
+	return b
 }

vendor/github.com/buger/jsonparser/fuzz.go

@@ -7,3 +7,111 @@ func FuzzParseString(data []byte) int {
 	}
 	return 1
 }
+
+func FuzzEachKey(data []byte) int {
+	paths := [][]string{
+		{"name"},
+		{"order"},
+		{"nested", "a"},
+		{"nested", "b"},
+		{"nested2", "a"},
+		{"nested", "nested3", "b"},
+		{"arr", "[1]", "b"},
+		{"arrInt", "[3]"},
+		{"arrInt", "[5]"},
+		{"nested"},
+		{"arr", "["},
+		{"a\n", "b\n"},
+	}
+	EachKey(data, func(idx int, value []byte, vt ValueType, err error) {}, paths...)
+	return 1
+}
+
+func FuzzDelete(data []byte) int {
+	Delete(data, "test")
+	return 1
+}
+
+func FuzzSet(data []byte) int {
+	_, err := Set(data, []byte(`"new value"`), "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzObjectEach(data []byte) int {
+	_ = ObjectEach(data, func(key, value []byte, valueType ValueType, off int) error {
+		return nil
+	})
+	return 1
+}
+
+func FuzzParseFloat(data []byte) int {
+	_, err := ParseFloat(data)
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzParseInt(data []byte) int {
+	_, err := ParseInt(data)
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzParseBool(data []byte) int {
+	_, err := ParseBoolean(data)
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzTokenStart(data []byte) int {
+	_ = tokenStart(data)
+	return 1
+}
+
+func FuzzGetString(data []byte) int {
+	_, err := GetString(data, "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzGetFloat(data []byte) int {
+	_, err := GetFloat(data, "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzGetInt(data []byte) int {
+	_, err := GetInt(data, "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzGetBoolean(data []byte) int {
+	_, err := GetBoolean(data, "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}
+
+func FuzzGetUnsafeString(data []byte) int {
+	_, err := GetUnsafeString(data, "test")
+	if err != nil {
+		return 0
+	}
+	return 1
+}

vendor/github.com/buger/jsonparser/oss-fuzz-build.sh

@@ -0,0 +1,47 @@
+#!/bin/bash -eu
+
+git clone https://github.com/dvyukov/go-fuzz-corpus
+zip corpus.zip go-fuzz-corpus/json/corpus/*
+
+cp corpus.zip $OUT/fuzzparsestring_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzParseString fuzzparsestring
+
+cp corpus.zip $OUT/fuzzeachkey_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzEachKey fuzzeachkey
+
+cp corpus.zip $OUT/fuzzdelete_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzDelete fuzzdelete
+
+cp corpus.zip $OUT/fuzzset_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzSet fuzzset
+
+cp corpus.zip $OUT/fuzzobjecteach_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzObjectEach fuzzobjecteach
+
+cp corpus.zip $OUT/fuzzparsefloat_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzParseFloat fuzzparsefloat
+
+cp corpus.zip $OUT/fuzzparseint_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzParseInt fuzzparseint
+
+cp corpus.zip $OUT/fuzzparsebool_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzParseBool fuzzparsebool
+
+cp corpus.zip $OUT/fuzztokenstart_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzTokenStart fuzztokenstart
+
+cp corpus.zip $OUT/fuzzgetstring_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzGetString fuzzgetstring
+
+cp corpus.zip $OUT/fuzzgetfloat_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzGetFloat fuzzgetfloat
+
+cp corpus.zip $OUT/fuzzgetint_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzGetInt fuzzgetint
+
+cp corpus.zip $OUT/fuzzgetboolean_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzGetBoolean fuzzgetboolean
+
+cp corpus.zip $OUT/fuzzgetunsafestring_seed_corpus.zip
+compile_go_fuzzer github.com/buger/jsonparser FuzzGetUnsafeString fuzzgetunsafestring
+

vendor/github.com/buger/jsonparser/parser.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"math"
 	"strconv"
 )
 
@@ -309,7 +308,11 @@ func searchKeys(data []byte, keys ...string) int {
 		case '[':
 			// If we want to get array element by index
 			if keyLevel == level && keys[level][0] == '[' {
-				aIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1])
+				var keyLen = len(keys[level])
+				if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' {
+					return -1
+				}
+				aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1])
 				if err != nil {
 					return -1
 				}
@@ -356,14 +359,6 @@ func searchKeys(data []byte, keys ...string) int {
 	return -1
 }
 
-var bitwiseFlags []int64
-
-func init() {
-	for i := 0; i < 63; i++ {
-		bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i))))
-	}
-}
-
 func sameTree(p1, p2 []string) bool {
 	minLen := len(p1)
 	if len(p2) < minLen {
@@ -380,7 +375,8 @@ func sameTree(p1, p2 []string) bool {
 }
 
 func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
-	var pathFlags int64
+	var x struct{}
+	pathFlags := make([]bool, len(paths))
 	var level, pathsMatched, i int
 	ln := len(data)
 
@@ -422,13 +418,15 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
 				// for unescape: if there are no escape sequences, this is cheap; if there are, it is a
 				// bit more expensive, but causes no allocations unless len(key) > unescapeStackBufSize
 				var keyUnesc []byte
-				var stackbuf [unescapeStackBufSize]byte
 				if !keyEscaped {
 					keyUnesc = key
-				} else if ku, err := Unescape(key, stackbuf[:]); err != nil {
-					return -1
 				} else {
-					keyUnesc = ku
+					var stackbuf [unescapeStackBufSize]byte
+					if ku, err := Unescape(key, stackbuf[:]); err != nil {
+						return -1
+					} else {
+						keyUnesc = ku
+					}
 				}
 
 				if maxPath >= level {
@@ -439,17 +437,16 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
 
 					pathsBuf[level-1] = bytesToString(&keyUnesc)
 					for pi, p := range paths {
-						if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
+						if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
 							continue
 						}
 
 						match = pi
 
-						i++
 						pathsMatched++
-						pathFlags |= bitwiseFlags[pi+1]
+						pathFlags[pi] = true
 
-						v, dt, _, e := Get(data[i:])
+						v, dt, _, e := Get(data[i+1:])
 						cb(pi, v, dt, e)
 
 						if pathsMatched == len(paths) {
@@ -485,8 +482,9 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
 		case '}':
 			level--
 		case '[':
-			var arrIdxFlags int64
-			var pIdxFlags int64
+			var ok bool
+			arrIdxFlags := make(map[int]struct{})
+			pIdxFlags := make([]bool, len(paths))
 
 			if level < 0 {
 				cb(-1, nil, Unknown, MalformedJsonError)
@@ -494,31 +492,31 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
 			}
 
 			for pi, p := range paths {
-				if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
+				if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
 					continue
 				}
 				if len(p[level]) >= 2 {
 					aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
-					arrIdxFlags |= bitwiseFlags[aIdx+1]
-					pIdxFlags |= bitwiseFlags[pi+1]
+					arrIdxFlags[aIdx] = x
+					pIdxFlags[pi] = true
 				}
 			}
 
-			if arrIdxFlags > 0 {
+			if len(arrIdxFlags) > 0 {
 				level++
 
 				var curIdx int
 				arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
-					if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
+					if _, ok = arrIdxFlags[curIdx]; ok {
 						for pi, p := range paths {
-							if pIdxFlags&bitwiseFlags[pi+1] != 0 {
+							if pIdxFlags[pi] {
 								aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])
 
 								if curIdx == aIdx {
 									of := searchKeys(value, p[level:]...)
 
 									pathsMatched++
-									pathFlags |= bitwiseFlags[pi+1]
+									pathFlags[pi] = true
 
 									if of != -1 {
 										v, dt, _, e := Get(value[of:])
@@ -597,48 +595,96 @@ var (
 )
 
 func createInsertComponent(keys []string, setValue []byte, comma, object bool) []byte {
-	var buffer bytes.Buffer
 	isIndex := string(keys[0][0]) == "["
+	offset := 0
+	lk := calcAllocateSpace(keys, setValue, comma, object)
+	buffer := make([]byte, lk, lk)
 	if comma {
-		buffer.WriteString(",")
+		offset += WriteToBuffer(buffer[offset:], ",")
 	}
 	if isIndex && !comma {
-		buffer.WriteString("[")
+		offset += WriteToBuffer(buffer[offset:], "[")
 	} else {
 		if object {
-			buffer.WriteString("{")
+			offset += WriteToBuffer(buffer[offset:], "{")
 		}
 		if !isIndex {
-			buffer.WriteString("\"")
-			buffer.WriteString(keys[0])
-			buffer.WriteString("\":")
+			offset += WriteToBuffer(buffer[offset:], "\"")
+			offset += WriteToBuffer(buffer[offset:], keys[0])
+			offset += WriteToBuffer(buffer[offset:], "\":")
 		}
 	}
 
 	for i := 1; i < len(keys); i++ {
 		if string(keys[i][0]) == "[" {
-			buffer.WriteString("[")
+			offset += WriteToBuffer(buffer[offset:], "[")
 		} else {
-			buffer.WriteString("{\"")
-			buffer.WriteString(keys[i])
-			buffer.WriteString("\":")
+			offset += WriteToBuffer(buffer[offset:], "{\"")
+			offset += WriteToBuffer(buffer[offset:], keys[i])
+			offset += WriteToBuffer(buffer[offset:], "\":")
 		}
 	}
-	buffer.Write(setValue)
+	offset += WriteToBuffer(buffer[offset:], string(setValue))
 	for i := len(keys) - 1; i > 0; i-- {
 		if string(keys[i][0]) == "[" {
-			buffer.WriteString("]")
+			offset += WriteToBuffer(buffer[offset:], "]")
 		} else {
-			buffer.WriteString("}")
+			offset += WriteToBuffer(buffer[offset:], "}")
 		}
 	}
 	if isIndex && !comma {
-		buffer.WriteString("]")
+		offset += WriteToBuffer(buffer[offset:], "]")
 	}
 	if object && !isIndex {
-		buffer.WriteString("}")
+		offset += WriteToBuffer(buffer[offset:], "}")
 	}
-	return buffer.Bytes()
+	return buffer
+}
+
+func calcAllocateSpace(keys []string, setValue []byte, comma, object bool) int {
+	isIndex := string(keys[0][0]) == "["
+	lk := 0
+	if comma {
+		// ,
+		lk += 1
+	}
+	if isIndex && !comma {
+		// []
+		lk += 2
+	} else {
+		if object {
+			// {
+			lk += 1
+		}
+		if !isIndex {
+			// "keys[0]"
+			lk += len(keys[0]) + 3
+		}
+	}
+
+
+	lk += len(setValue)
+	for i := 1; i < len(keys); i++ {
+		if string(keys[i][0]) == "[" {
+			// []
+			lk += 2
+		} else {
+			// {"keys[i]":setValue}
+			lk += len(keys[i]) + 5
+		}
+	}
+
+	if object && !isIndex {
+		// }
+		lk += 1
+	}
+
+	return lk
+}
+
+func WriteToBuffer(buffer []byte, str string) int {
+	copy(buffer, str)
+	return len(str)
 }
 
 /*
@@ -766,7 +812,7 @@ func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error)
 		if endOffset == -1 {
 			firstToken := nextToken(data)
 			// We can't set a top-level key if data isn't an object
-			if len(data) == 0 || data[firstToken] != '{' {
+			if firstToken < 0 || data[firstToken] != '{' {
 				return nil, KeyPathNotFoundError
 			}
 			// Don't need a comma if the input is an empty object
@@ -916,7 +962,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType,
 		value = value[1 : len(value)-1]
 	}
 
-	return value, dataType, offset, endOffset, nil
+	return value[:len(value):len(value)], dataType, offset, endOffset, nil
 }
 
 // ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
@@ -930,7 +976,7 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
 		return -1, MalformedJsonError
 	}
 
-	offset = nT+1
+	offset = nT + 1
 
 	if len(keys) > 0 {
 		if offset = searchKeys(data, keys...); offset == -1 {
@@ -1135,7 +1181,7 @@ func GetString(data []byte, keys ...string) (val string, err error) {
 		return "", fmt.Errorf("Value is not a string: %s", string(v))
 	}
 
-	// If no escapes return raw conten
+	// If no escapes return raw content
 	if bytes.IndexByte(v, '\\') == -1 {
 		return string(v), nil
 	}