fuzz: introduce continuous fuzzing for Caddy (#2723)

* fuzz: lay down the foundation for continuous fuzzing

* improve the fuzzers and add some

* fuzz: add Fuzzit badge to README & enable fuzzers submission in CI

* v2-fuzz: do away with the submodule approach for fuzzers

* fuzz: enable fuzzit

README.md

@@ -2,6 +2,7 @@ Caddy 2 Development Branch
 ===========================
 
 [![Build Status](https://dev.azure.com/mholt-dev/Caddy/_apis/build/status/Multiplatform%20Tests?branchName=v2)](https://dev.azure.com/mholt-dev/Caddy/_build/latest?definitionId=5&branchName=v2)
+[![fuzzit](https://app.fuzzit.dev/badge?org_id=caddyserver)](https://app.fuzzit.dev/orgs/caddyserver/dashboard)
 
 This is the development branch for Caddy 2. This code (version 2) is not yet feature-complete or production-ready, but is already being used in production, and we encourage you to deploy it today on sites that are not very visible or important so that it can obtain crucial experience in the field.
 

admin_fuzz.go

@@ -0,0 +1,30 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+import (
+	"bytes"
+)
+
+func FuzzAdmin(data []byte) (score int) {
+	err := Load(bytes.NewReader(data))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

azure-pipelines.yml

@@ -4,6 +4,25 @@
 trigger:
   - v2
 
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily midnight fuzzing
+  branches:
+    include:
+    - v2
+  always: true
+
+variables:
+  GOROOT: $(gorootDir)/go
+  GOPATH: $(system.defaultWorkingDirectory)/gopath
+  GOBIN:  $(GOPATH)/bin
+  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
+  # TODO: Remove once it's enabled by default
+  GO111MODULE: on
+
+jobs:
+- job: crossPlatformTest
+  displayName: "Cross-Platform Tests"
   strategy:
     matrix:
       linux:
@@ -18,14 +37,6 @@ strategy:
   pool:
     vmImage: $(imageName)
   
-variables:
-  GOROOT: $(gorootDir)/go
-  GOPATH: $(system.defaultWorkingDirectory)/gopath
-  GOBIN:  $(GOPATH)/bin
-  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
-  # TODO: Remove once it's enabled by default
-  GO111MODULE: on
-
   steps:
   - bash: |
       latestGo=$(curl "https://golang.org/VERSION?m=text")
@@ -144,3 +155,89 @@ steps:
       exit 1
     condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
     displayName: Coerce correct build result
+
+- job: fuzzing
+  displayName: 'Scheduled Fuzzing'
+  # Only run this job on schedules, not PRs.
+  condition: eq(variables['Build.Reason'], 'Schedule')
+  strategy:
+    matrix:
+      linux:
+        imageName: ubuntu-16.04
+        gorootDir: /usr/local
+  pool:
+    vmImage: $(imageName)
+
+  steps:
+  - bash: |
+      latestGo=$(curl "https://golang.org/VERSION?m=text")
+      echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
+      echo "Latest Go version: $latestGo"
+    displayName: "Get latest Go version"
+
+  - bash: |
+      sudo rm -f $(which go)
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+      mkdir -p '$(modulePath)'
+      shopt -s extglob
+      shopt -s dotglob
+      mv !(gopath) '$(modulePath)'
+    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
+
+  - bash: |
+      wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
+      sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
+    condition: eq( variables['Agent.OS'], 'Linux' )
+    displayName: Install Go on Linux
+
+  - bash: |
+      # Install Clang
+      sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main"
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+      sudo apt update && sudo apt install -y clang lldb lld
+
+      go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+      wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.74/fuzzit_Linux_x86_64
+      chmod a+x fuzzit
+      mv fuzzit $(GOBIN)
+    displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
+    condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
+
+  - script: fuzzit auth ${FUZZIT_API_KEY}
+    condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
+    displayName: Authenticate with Fuzzit
+    env:
+      FUZZIT_API_KEY: $(FUZZIT_API_KEY)
+
+  - bash: |
+      declare -A fuzzers_funcs=(\
+        ["./admin_fuzz.go"]="FuzzAdmin" \
+        ["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="FuzzHTTPCaddyfileAdapter" \
+        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
+        ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
+        ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
+        ["./replacer_fuzz.go"]="FuzzReplacer" \
+      )
+
+      declare -A fuzzers_targets=(\
+        ["./admin_fuzz.go"]="admin" \
+        ["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="caddyfile-adapter" \
+        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
+        ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
+        ["./listeners_fuzz.go"]="parse-listen-addr" \
+        ["./replacer_fuzz.go"]="replacer" \
+      )
+      fuzz_type="fuzzing"
+
+      for f in $(find . -name \*_fuzz.go); do
+        FUZZER_DIRECTORY=$(dirname $f)
+        echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
+        go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
+        echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
+        clang -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer"
+        fuzzit create job --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}" caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer
+        echo "Completed $f"
+      done
+    workingDirectory: '$(modulePath)'
+    displayName: Generate fuzzers & submit them to Fuzzit

caddyconfig/httpcaddyfile/adapter_fuzz.go

@@ -0,0 +1,49 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package httpcaddyfile
+
+import (
+	"bytes"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func FuzzHTTPCaddyfileAdapter(data []byte) int {
+	adapter := caddyfile.Adapter{
+		ServerType: ServerType{},
+	}
+	b, warns, err := adapter.Adapt(data, nil)
+	// Adapt func calls the Setup() func of the ServerType,
+	// thus it's going across multiple layers, each can
+	// return warnings or errors. Marking the presence of
+	// errors or warnings as interesting in this case
+	// could push the fuzzer towards a path where we only
+	// catch errors. Let's push the fuzzer to where it passes
+	// but breaks.
+	if (err != nil) || (warns != nil && len(warns) > 0) {
+		return 0
+	}
+
+	// adapted Caddyfile should be parseable by the configuration loader in admin.go
+	err = caddy.Load(bytes.NewReader(b))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -0,0 +1,29 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package httpcaddyfile
+
+func FuzzParseAddress(data []byte) int {
+	addr, err := ParseAddress(string(data))
+	if err != nil {
+		if addr == (Address{}) {
+			return 1
+		}
+		return 0
+	}
+	return 1
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.0.0
 	github.com/andybalholm/brotli v0.0.0-20190821151343-b60f0d972eeb
 	github.com/dustin/go-humanize v1.0.0
+	github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 // indirect
 	github.com/go-acme/lego/v3 v3.1.0
 	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc
 	github.com/ilibs/json5 v1.0.1

go.sum

@@ -59,6 +59,8 @@ github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s9
 github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 h1:3WV5aRRj1ELP3RcLlBp/v0WJTuy47OQMkL9GIQq8QEE=
+github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=

listeners_fuzz.go

@@ -0,0 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+func FuzzParseNetworkAddress(data []byte) int {
+	_, _, err := ParseNetworkAddress(string(data))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

replacer_fuzz.go

@@ -0,0 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+func FuzzReplacer(data []byte) (score int) {
+	NewReplacer().ReplaceAll(string(data), "")
+	NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), "")
+	NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), NewReplacer().ReplaceAll(string(data), ""))
+	NewReplacer().ReplaceAll(string(data[:len(data)/2]), string(data[len(data)/2:]))
+	return 0
+}