rclone/lib/encoder/internal/gen/main.go
albertony bcdfad3c83 build: update logging statements to make json log work - fixes #6038
This changes log statements from log to fs package, which is required for --use-json-log
to properly make log output in JSON format. The recently added custom linting rule,
handled by ruleguard via gocritic via golangci-lint, warns about these and suggests
the alternative. Fixing was therefore basically running "golangci-lint run --fix",
although some manual fixup of mainly imports are necessary following that.
2024-09-06 17:04:18 +01:00

631 lines
15 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Package main provides utilities for encoder.
package main
import (
"flag"
"fmt"
"math/rand"
"os"
"strconv"
"strings"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/encoder"
)
const (
edgeLeft = iota
edgeRight
)
type mapping struct {
mask encoder.MultiEncoder
src, dst []rune
}
type stringPair struct {
a, b string
}
const header = `// Code generated by ./internal/gen/main.go. DO NOT EDIT.
` + `//go:generate go run ./internal/gen/main.go
package encoder
`
var maskBits = []struct {
mask encoder.MultiEncoder
name string
}{
{encoder.EncodeZero, "EncodeZero"},
{encoder.EncodeSlash, "EncodeSlash"},
{encoder.EncodeSingleQuote, "EncodeSingleQuote"},
{encoder.EncodeBackQuote, "EncodeBackQuote"},
{encoder.EncodeLtGt, "EncodeLtGt"},
{encoder.EncodeSquareBracket, "EncodeSquareBracket"},
{encoder.EncodeSemicolon, "EncodeSemicolon"},
{encoder.EncodeExclamation, "EncodeExclamation"},
{encoder.EncodeDollar, "EncodeDollar"},
{encoder.EncodeDoubleQuote, "EncodeDoubleQuote"},
{encoder.EncodeColon, "EncodeColon"},
{encoder.EncodeQuestion, "EncodeQuestion"},
{encoder.EncodeAsterisk, "EncodeAsterisk"},
{encoder.EncodePipe, "EncodePipe"},
{encoder.EncodeHash, "EncodeHash"},
{encoder.EncodePercent, "EncodePercent"},
{encoder.EncodeBackSlash, "EncodeBackSlash"},
{encoder.EncodeCrLf, "EncodeCrLf"},
{encoder.EncodeDel, "EncodeDel"},
{encoder.EncodeCtl, "EncodeCtl"},
{encoder.EncodeLeftSpace, "EncodeLeftSpace"},
{encoder.EncodeLeftPeriod, "EncodeLeftPeriod"},
{encoder.EncodeLeftTilde, "EncodeLeftTilde"},
{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt"},
{encoder.EncodeRightSpace, "EncodeRightSpace"},
{encoder.EncodeRightPeriod, "EncodeRightPeriod"},
{encoder.EncodeRightCrLfHtVt, "EncodeRightCrLfHtVt"},
{encoder.EncodeInvalidUtf8, "EncodeInvalidUtf8"},
{encoder.EncodeDot, "EncodeDot"},
}
type edge struct {
mask encoder.MultiEncoder
name string
edge int
orig []rune
replace []rune
}
var allEdges = []edge{
{encoder.EncodeLeftSpace, "EncodeLeftSpace", edgeLeft, []rune{' '}, []rune{'␠'}},
{encoder.EncodeLeftPeriod, "EncodeLeftPeriod", edgeLeft, []rune{'.'}, []rune{''}},
{encoder.EncodeLeftTilde, "EncodeLeftTilde", edgeLeft, []rune{'~'}, []rune{''}},
{encoder.EncodeLeftCrLfHtVt, "EncodeLeftCrLfHtVt", edgeLeft,
[]rune{'\t', '\n', '\v', '\r'},
[]rune{'␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r'},
},
{encoder.EncodeRightSpace, "EncodeRightSpace", edgeRight, []rune{' '}, []rune{'␠'}},
{encoder.EncodeRightPeriod, "EncodeRightPeriod", edgeRight, []rune{'.'}, []rune{''}},
{encoder.EncodeRightCrLfHtVt, "EncodeRightCrLfHtVt", edgeRight,
[]rune{'\t', '\n', '\v', '\r'},
[]rune{'␀' + '\t', '␀' + '\n', '␀' + '\v', '␀' + '\r'},
},
}
var allMappings = []mapping{{
encoder.EncodeZero, []rune{
0,
}, []rune{
'␀',
}}, {
encoder.EncodeSlash, []rune{
'/',
}, []rune{
'',
}}, {
encoder.EncodeLtGt, []rune{
'<', '>',
}, []rune{
'', '',
}}, {
encoder.EncodeSquareBracket, []rune{
'[', ']',
}, []rune{
'', '',
}}, {
encoder.EncodeSemicolon, []rune{
';',
}, []rune{
'',
}}, {
encoder.EncodeExclamation, []rune{
'!',
}, []rune{
'',
}}, {
encoder.EncodeDoubleQuote, []rune{
'"',
}, []rune{
'',
}}, {
encoder.EncodeSingleQuote, []rune{
'\'',
}, []rune{
'',
}}, {
encoder.EncodeBackQuote, []rune{
'`',
}, []rune{
'',
}}, {
encoder.EncodeDollar, []rune{
'$',
}, []rune{
'',
}}, {
encoder.EncodeColon, []rune{
':',
}, []rune{
'',
}}, {
encoder.EncodeQuestion, []rune{
'?',
}, []rune{
'',
}}, {
encoder.EncodeAsterisk, []rune{
'*',
}, []rune{
'',
}}, {
encoder.EncodePipe, []rune{
'|',
}, []rune{
'',
}}, {
encoder.EncodeHash, []rune{
'#',
}, []rune{
'',
}}, {
encoder.EncodePercent, []rune{
'%',
}, []rune{
'',
}}, {
encoder.EncodeSlash, []rune{
'/',
}, []rune{
'',
}}, {
encoder.EncodeBackSlash, []rune{
'\\',
}, []rune{
'',
}}, {
encoder.EncodeCrLf, []rune{
rune(0x0D), rune(0x0A),
}, []rune{
'␍', '␊',
}}, {
encoder.EncodeDel, []rune{
0x7F,
}, []rune{
'␡',
}}, {
encoder.EncodeCtl,
runeRange(0x01, 0x1F),
runeRange('␁', '␟'),
}}
var (
rng *rand.Rand
printables = runeRange(0x20, 0x7E)
fullwidthPrintables = runeRange(0xFF00, 0xFF5E)
encodables = collectEncodables(allMappings)
encoded = collectEncoded(allMappings)
greek = runeRange(0x03B1, 0x03C9)
)
func main() {
seed := flag.Int64("s", 42, "random seed")
flag.Parse()
rng = rand.New(rand.NewSource(*seed))
fd, err := os.Create("encoder_cases_test.go")
fatal(err, "Unable to open encoder_cases_test.go:")
defer func() {
fatal(fd.Close(), "Failed to close encoder_cases_test.go:")
}()
fatalW(fd.WriteString(header))("Failed to write header:")
fatalW(fd.WriteString("var testCasesSingle = []testCase{\n\t"))("Write:")
_i := 0
i := func() (r int) {
r, _i = _i, _i+1
return
}
for _, m := range maskBits {
if len(getMapping(m.mask).src) == 0 {
continue
}
if _i != 0 {
fatalW(fd.WriteString(" "))("Write:")
}
in, out := buildTestString(
[]mapping{getMapping(m.mask)}, // pick
[]mapping{getMapping(encoder.EncodeZero)}, // quote
printables, fullwidthPrintables, encodables, encoded, greek) // fill
fatalW(fmt.Fprintf(fd, `{ // %d
mask: %s,
in: %s,
out: %s,
},`, i(), m.name, strconv.Quote(in), strconv.Quote(out)))("Error writing test case:")
}
fatalW(fd.WriteString(`
}
var testCasesSingleEdge = []testCase{
`))("Write:")
_i = 0
for _, e := range allEdges {
for idx, orig := range e.orig {
if _i != 0 {
fatalW(fd.WriteString(" "))("Write:")
}
fatalW(fmt.Fprintf(fd, `{ // %d
mask: %s,
in: %s,
out: %s,
},`, i(), e.name, strconv.Quote(string(orig)), strconv.Quote(string(e.replace[idx]))))("Error writing test case:")
}
for _, m := range maskBits {
if len(getMapping(m.mask).src) == 0 || invalidMask(e.mask|m.mask) {
continue
}
for idx, orig := range e.orig {
replace := e.replace[idx]
pairs := buildEdgeTestString(
[]edge{e}, []mapping{getMapping(encoder.EncodeZero), getMapping(m.mask)}, // quote
[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
testL := len(rIn)
skipOrig := false
for _, m := range testMappings {
if runePos(orig, m.src) != -1 || runePos(orig, m.dst) != -1 {
skipOrig = true
break
}
}
if !skipOrig {
rIn[10], rOut[10], quoteOut[10] = orig, orig, false
}
out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
for _, i := range []int{0, 1, testL - 2, testL - 1} {
for _, j := range []int{1, testL - 2, testL - 1} {
if j < i {
continue
}
rIn := append([]rune{}, rIn...)
rOut := append([]rune{}, rOut...)
quoteOut := append([]bool{}, quoteOut...)
for _, in := range []rune{orig, replace} {
expect, quote := in, false
if i == 0 && e.edge == edgeLeft ||
i == testL-1 && e.edge == edgeRight {
expect, quote = replace, in == replace
}
rIn[i], rOut[i], quoteOut[i] = in, expect, quote
if i != j {
for _, in := range []rune{orig, replace} {
expect, quote = in, false
if j == testL-1 && e.edge == edgeRight {
expect, quote = replace, in == replace
}
rIn[j], rOut[j], quoteOut[j] = in, expect, quote
}
}
out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
}
}
}
return
})
for _, p := range pairs {
fatalW(fmt.Fprintf(fd, ` { // %d
mask: %s | %s,
in: %s,
out: %s,
},`, i(), m.name, e.name, strconv.Quote(p.a), strconv.Quote(p.b)))("Error writing test case:")
}
}
}
}
fatalW(fmt.Fprintf(fd, ` { // %d
mask: EncodeLeftSpace,
in: " ",
out: "␠ ",
}, { // %d
mask: EncodeLeftPeriod,
in: "..",
out: ".",
}, { // %d
mask: EncodeLeftTilde,
in: "~~",
out: "~",
}, { // %d
mask: EncodeRightSpace,
in: " ",
out: " ␠",
}, { // %d
mask: EncodeRightPeriod,
in: "..",
out: ".",
}, { // %d
mask: EncodeLeftSpace | EncodeRightPeriod,
in: " .",
out: "␠.",
}, { // %d
mask: EncodeLeftSpace | EncodeRightSpace,
in: " ",
out: "␠",
}, { // %d
mask: EncodeLeftSpace | EncodeRightSpace,
in: " ",
out: "␠␠",
}, { // %d
mask: EncodeLeftSpace | EncodeRightSpace,
in: " ",
out: "␠ ␠",
}, { // %d
mask: EncodeLeftPeriod | EncodeRightPeriod,
in: "...",
out: ".",
}, { // %d
mask: EncodeRightPeriod | EncodeRightSpace,
in: "a. ",
out: "a.␠",
}, { // %d
mask: EncodeRightPeriod | EncodeRightSpace,
in: "a .",
out: "a ",
},
}
var testCasesDoubleEdge = []testCase{
`, i(), i(), i(), i(), i(), i(), i(), i(), i(), i(), i(), i()))("Error writing test case:")
_i = 0
for _, e1 := range allEdges {
for _, e2 := range allEdges {
if e1.mask == e2.mask {
continue
}
for _, m := range maskBits {
if len(getMapping(m.mask).src) == 0 || invalidMask(m.mask|e1.mask|e2.mask) {
continue
}
orig, replace := e1.orig[0], e1.replace[0]
edges := []edge{e1, e2}
pairs := buildEdgeTestString(
edges, []mapping{getMapping(encoder.EncodeZero), getMapping(m.mask)}, // quote
[][]rune{printables, fullwidthPrintables, encodables, encoded, greek}, // fill
func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) (out []stringPair) {
testL := len(rIn)
for _, i := range []int{0, testL - 1} {
for _, secondOrig := range e2.orig {
rIn := append([]rune{}, rIn...)
rOut := append([]rune{}, rOut...)
quoteOut := append([]bool{}, quoteOut...)
rIn[1], rOut[1], quoteOut[1] = secondOrig, secondOrig, false
rIn[testL-2], rOut[testL-2], quoteOut[testL-2] = secondOrig, secondOrig, false
for _, in := range []rune{orig, replace} {
rIn[i], rOut[i], quoteOut[i] = in, in, false
fixEdges(rIn, rOut, quoteOut, edges)
out = append(out, stringPair{string(rIn), quotedToString(rOut, quoteOut)})
}
}
}
return
})
for _, p := range pairs {
if _i != 0 {
fatalW(fd.WriteString(" "))("Write:")
}
fatalW(fmt.Fprintf(fd, `{ // %d
mask: %s | %s | %s,
in: %s,
out: %s,
},`, i(), m.name, e1.name, e2.name, strconv.Quote(p.a), strconv.Quote(p.b)))("Error writing test case:")
}
}
}
}
fatalW(fmt.Fprint(fd, "\n}\n"))("Error writing test case:")
}
func fatal(err error, s ...interface{}) {
if err != nil {
fs.Fatal(nil, fmt.Sprint(append(s, err)))
}
}
func fatalW(_ int, err error) func(...interface{}) {
if err != nil {
return func(s ...interface{}) {
fs.Fatal(nil, fmt.Sprint(append(s, err)))
}
}
return func(s ...interface{}) {}
}
func invalidMask(mask encoder.MultiEncoder) bool {
return mask&(encoder.EncodeCtl|encoder.EncodeCrLf) != 0 && mask&(encoder.EncodeLeftCrLfHtVt|encoder.EncodeRightCrLfHtVt) != 0
}
// construct a slice containing the runes between (l)ow (inclusive) and (h)igh (inclusive)
func runeRange(l, h rune) []rune {
if h < l {
panic("invalid range")
}
out := make([]rune, h-l+1)
for i := range out {
out[i] = l + rune(i)
}
return out
}
func getMapping(mask encoder.MultiEncoder) mapping {
for _, m := range allMappings {
if m.mask == mask {
return m
}
}
return mapping{}
}
func collectEncodables(m []mapping) (out []rune) {
for _, s := range m {
out = append(out, s.src...)
}
return
}
func collectEncoded(m []mapping) (out []rune) {
for _, s := range m {
out = append(out, s.dst...)
}
return
}
func buildTestString(mappings, testMappings []mapping, fill ...[]rune) (string, string) {
combinedMappings := append(mappings, testMappings...)
var (
rIn []rune
rOut []rune
)
for _, m := range mappings {
if len(m.src) == 0 || len(m.src) != len(m.dst) {
panic("invalid length")
}
rIn = append(rIn, m.src...)
rOut = append(rOut, m.dst...)
}
inL := len(rIn)
testL := inL * 3
if testL < 30 {
testL = 30
}
rIn = append(rIn, make([]rune, testL-inL)...)
rOut = append(rOut, make([]rune, testL-inL)...)
quoteOut := make([]bool, testL)
set := func(i int, in, out rune, quote bool) {
rIn[i] = in
rOut[i] = out
quoteOut[i] = quote
}
for i, r := range rOut[:inL] {
set(inL+i, r, r, true)
}
outer:
for pos := inL * 2; pos < testL; pos++ {
m := pos % len(fill)
i := rng.Intn(len(fill[m]))
r := fill[m][i]
for _, m := range combinedMappings {
if pSrc := runePos(r, m.src); pSrc != -1 {
set(pos, r, m.dst[pSrc], false)
continue outer
} else if pDst := runePos(r, m.dst); pDst != -1 {
set(pos, r, r, true)
continue outer
}
}
set(pos, r, r, false)
}
rng.Shuffle(testL, func(i, j int) {
rIn[i], rIn[j] = rIn[j], rIn[i]
rOut[i], rOut[j] = rOut[j], rOut[i]
quoteOut[i], quoteOut[j] = quoteOut[j], quoteOut[i]
})
var bOut strings.Builder
bOut.Grow(testL)
for i, r := range rOut {
if quoteOut[i] {
bOut.WriteRune(encoder.QuoteRune)
}
bOut.WriteRune(r)
}
return string(rIn), bOut.String()
}
func buildEdgeTestString(edges []edge, testMappings []mapping, fill [][]rune,
gen func(rIn, rOut []rune, quoteOut []bool, testMappings []mapping) []stringPair,
) []stringPair {
testL := 30
rIn := make([]rune, testL) // test input string
rOut := make([]rune, testL) // test output string without quote runes
quoteOut := make([]bool, testL) // if true insert quote rune before the output rune
set := func(i int, in, out rune, quote bool) {
rIn[i] = in
rOut[i] = out
quoteOut[i] = quote
}
// populate test strings with values from the `fill` set
outer:
for pos := 0; pos < testL; pos++ {
m := pos % len(fill)
i := rng.Intn(len(fill[m]))
r := fill[m][i]
for _, m := range testMappings {
if pSrc := runePos(r, m.src); pSrc != -1 {
set(pos, r, m.dst[pSrc], false)
continue outer
} else if pDst := runePos(r, m.dst); pDst != -1 {
set(pos, r, r, true)
continue outer
}
}
set(pos, r, r, false)
}
rng.Shuffle(testL, func(i, j int) {
rIn[i], rIn[j] = rIn[j], rIn[i]
rOut[i], rOut[j] = rOut[j], rOut[i]
quoteOut[i], quoteOut[j] = quoteOut[j], quoteOut[i]
})
fixEdges(rIn, rOut, quoteOut, edges)
return gen(rIn, rOut, quoteOut, testMappings)
}
func fixEdges(rIn, rOut []rune, quoteOut []bool, edges []edge) {
testL := len(rIn)
for _, e := range edges {
for idx, o := range e.orig {
r := e.replace[idx]
if e.edge == edgeLeft && rIn[0] == o {
rOut[0], quoteOut[0] = r, false
} else if e.edge == edgeLeft && rIn[0] == r {
quoteOut[0] = true
} else if e.edge == edgeRight && rIn[testL-1] == o {
rOut[testL-1], quoteOut[testL-1] = r, false
} else if e.edge == edgeRight && rIn[testL-1] == r {
quoteOut[testL-1] = true
}
}
}
}
func runePos(r rune, s []rune) int {
for i, c := range s {
if c == r {
return i
}
}
return -1
}
// quotedToString returns a string for the chars slice where an encoder.QuoteRune is
// inserted before a char[i] when quoted[i] is true.
func quotedToString(chars []rune, quoted []bool) string {
var out strings.Builder
out.Grow(len(chars))
for i, r := range chars {
if quoted[i] {
out.WriteRune(encoder.QuoteRune)
}
out.WriteRune(r)
}
return out.String()
}