fuzz: introduce continuous fuzzing for Caddy (#2723)

* fuzz: lay down the foundation for continuous fuzzing

* improve the fuzzers and add some

* fuzz: add Fuzzit badge to README & enable fuzzers submission in CI

* v2-fuzz: do away with the submodule approach for fuzzers

* fuzz: enable fuzzit

README.md

@@ -2,6 +2,7 @@ Caddy 2 Development Branch
 ===========================
 
 [![Build Status](https://dev.azure.com/mholt-dev/Caddy/_apis/build/status/Multiplatform%20Tests?branchName=v2)](https://dev.azure.com/mholt-dev/Caddy/_build/latest?definitionId=5&branchName=v2)
+[![fuzzit](https://app.fuzzit.dev/badge?org_id=caddyserver)](https://app.fuzzit.dev/orgs/caddyserver/dashboard)
 
 This is the development branch for Caddy 2. This code (version 2) is not yet feature-complete or production-ready, but is already being used in production, and we encourage you to deploy it today on sites that are not very visible or important so that it can obtain crucial experience in the field.
 

admin_fuzz.go

@@ -0,0 +1,30 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+import (
+	"bytes"
+)
+
+func FuzzAdmin(data []byte) (score int) {
+	err := Load(bytes.NewReader(data))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

azure-pipelines.yml

@@ -2,9 +2,28 @@
 # https://docs.microsoft.com/azure/devops/pipelines/languages/go
 
 trigger:
-- v2
+  - v2
 
-strategy:
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily midnight fuzzing
+  branches:
+    include:
+    - v2
+  always: true
+
+variables:
+  GOROOT: $(gorootDir)/go
+  GOPATH: $(system.defaultWorkingDirectory)/gopath
+  GOBIN:  $(GOPATH)/bin
+  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
+  # TODO: Remove once it's enabled by default
+  GO111MODULE: on
+
+jobs:
+- job: crossPlatformTest
+  displayName: "Cross-Platform Tests"
+  strategy:
     matrix:
       linux:
         imageName: ubuntu-16.04
@@ -15,25 +34,17 @@ strategy:
       windows:
         imageName: windows-2019
         gorootDir: C:\
-pool:
+  pool:
     vmImage: $(imageName)
   
-variables:
-  GOROOT: $(gorootDir)/go
-  GOPATH: $(system.defaultWorkingDirectory)/gopath
-  GOBIN:  $(GOPATH)/bin
-  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
-  # TODO: Remove once it's enabled by default
-  GO111MODULE: on
-
-steps:
-- bash: |
+  steps:
+  - bash: |
       latestGo=$(curl "https://golang.org/VERSION?m=text")
       echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
       echo "Latest Go version: $latestGo"
     displayName: "Get latest Go version"
 
-- bash: |
+  - bash: |
       sudo rm -f $(which go)
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
@@ -43,22 +54,22 @@ steps:
       mv !(gopath) '$(modulePath)'
     displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
 
-# Install Go (this varies by platform)
-- bash: |
+  # Install Go (this varies by platform)
+  - bash: |
       wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
       sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
     condition: eq( variables['Agent.OS'], 'Linux' )
     displayName: Install Go on Linux
 
-- bash: |
+  - bash: |
       wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
       sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
     condition: eq( variables['Agent.OS'], 'Darwin' )
     displayName: Install Go on macOS
 
-# The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
-# Reference: https://github.com/PowerShell/PowerShell/issues/2138
-- powershell: |
+  # The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
+  # Reference: https://github.com/PowerShell/PowerShell/issues/2138
+  - powershell: |
       $ProgressPreference = 'SilentlyContinue'
       Write-Host "Downloading Go..."
       (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
@@ -67,16 +78,16 @@ steps:
     condition: eq( variables['Agent.OS'], 'Windows_NT' )
     displayName: Install Go on Windows
 
-- bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
+  - bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
     displayName: Install golangci-lint
 
-- script: |
+  - script: |
       go get github.com/axw/gocov/gocov
       go get github.com/AlekSi/gocov-xml
       go get -u github.com/jstemmer/go-junit-report
     displayName: Install test and coverage analysis tools
 
-- bash: |
+  - bash: |
       printf "Using go at: $(which go)\n"
       printf "Go version: $(go version)\n"
       printf "\n\nGo environment:\n\n"
@@ -85,27 +96,27 @@ steps:
       env
     displayName: Print Go version and environment
 
-- script: |
+  - script: |
       go get -v -t -d ./...
       mkdir test-results
     workingDirectory: '$(modulePath)'
     displayName: Get dependencies
 
-# its behavior is governed by .golangci.yml
-- script: |
+  # its behavior is governed by .golangci.yml
+  - script: |
       (golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
       exit 0
     workingDirectory: '$(modulePath)'
     continueOnError: true
     displayName: Run lint check
 
-- script: |
+  - script: |
       (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
     workingDirectory: '$(modulePath)'
     continueOnError: true
     displayName: Run tests
 
-- script: |
+  - script: |
       mkdir coverage
       gocov convert cover-profile.out > coverage/coverage.json
       # Because Windows doesn't work with input redirection like *nix, but output redirection works.
@@ -113,18 +124,18 @@ steps:
     workingDirectory: '$(modulePath)'
     displayName: Prepare coverage reports
 
-- script: |
+  - script: |
       (cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
     workingDirectory: '$(modulePath)'
     displayName: Prepare test report
 
-- task: PublishCodeCoverageResults@1
+  - task: PublishCodeCoverageResults@1
     displayName: Publish test coverage report
     inputs:
       codeCoverageTool: Cobertura 
       summaryFileLocation: $(modulePath)/coverage/coverage.xml
 
-- task: PublishTestResults@2
+  - task: PublishTestResults@2
     displayName: Publish unit test
     inputs:
       testResultsFormat: 'JUnit'
@@ -132,7 +143,7 @@ steps:
       testRunTitle: $(agent.OS) Unit Test
       mergeTestResults: false
 
-- task: PublishTestResults@2
+  - task: PublishTestResults@2
     displayName: Publish lint results
     inputs:
       testResultsFormat: 'JUnit'
@@ -140,7 +151,93 @@ steps:
       testRunTitle: $(agent.OS) Lint
       mergeTestResults: false
 
-- bash: |
+  - bash: |
       exit 1
     condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
     displayName: Coerce correct build result
+
+- job: fuzzing
+  displayName: 'Scheduled Fuzzing'
+  # Only run this job on schedules, not PRs.
+  condition: eq(variables['Build.Reason'], 'Schedule')
+  strategy:
+    matrix:
+      linux:
+        imageName: ubuntu-16.04
+        gorootDir: /usr/local
+  pool:
+    vmImage: $(imageName)
+
+  steps:
+  - bash: |
+      latestGo=$(curl "https://golang.org/VERSION?m=text")
+      echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
+      echo "Latest Go version: $latestGo"
+    displayName: "Get latest Go version"
+
+  - bash: |
+      sudo rm -f $(which go)
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+      mkdir -p '$(modulePath)'
+      shopt -s extglob
+      shopt -s dotglob
+      mv !(gopath) '$(modulePath)'
+    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
+
+  - bash: |
+      wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
+      sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
+    condition: eq( variables['Agent.OS'], 'Linux' )
+    displayName: Install Go on Linux
+
+  - bash: |
+      # Install Clang
+      sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main"
+      wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
+      sudo apt update && sudo apt install -y clang lldb lld
+
+      go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
+      wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.74/fuzzit_Linux_x86_64
+      chmod a+x fuzzit
+      mv fuzzit $(GOBIN)
+    displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
+    condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
+
+  - script: fuzzit auth ${FUZZIT_API_KEY}
+    condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
+    displayName: Authenticate with Fuzzit
+    env:
+      FUZZIT_API_KEY: $(FUZZIT_API_KEY)
+
+  - bash: |
+      declare -A fuzzers_funcs=(\
+        ["./admin_fuzz.go"]="FuzzAdmin" \
+        ["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="FuzzHTTPCaddyfileAdapter" \
+        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
+        ["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
+        ["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
+        ["./replacer_fuzz.go"]="FuzzReplacer" \
+      )
+
+      declare -A fuzzers_targets=(\
+        ["./admin_fuzz.go"]="admin" \
+        ["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="caddyfile-adapter" \
+        ["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
+        ["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
+        ["./listeners_fuzz.go"]="parse-listen-addr" \
+        ["./replacer_fuzz.go"]="replacer" \
+      )
+      fuzz_type="fuzzing"
+
+      for f in $(find . -name \*_fuzz.go); do
+        FUZZER_DIRECTORY=$(dirname $f)
+        echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
+        go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
+        echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
+        clang -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer"
+        fuzzit create job --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}" caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer
+        echo "Completed $f"
+      done
+    workingDirectory: '$(modulePath)'
+    displayName: Generate fuzzers & submit them to Fuzzit

caddyconfig/httpcaddyfile/adapter_fuzz.go

@@ -0,0 +1,49 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package httpcaddyfile
+
+import (
+	"bytes"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
+)
+
+func FuzzHTTPCaddyfileAdapter(data []byte) int {
+	adapter := caddyfile.Adapter{
+		ServerType: ServerType{},
+	}
+	b, warns, err := adapter.Adapt(data, nil)
+	// Adapt func calls the Setup() func of the ServerType,
+	// thus it's going across multiple layers, each can
+	// return warnings or errors. Marking the presence of
+	// errors or warnings as interesting in this case
+	// could push the fuzzer towards a path where we only
+	// catch errors. Let's push the fuzzer to where it passes
+	// but breaks.
+	if (err != nil) || (warns != nil && len(warns) > 0) {
+		return 0
+	}
+
+	// adapted Caddyfile should be parseable by the configuration loader in admin.go
+	err = caddy.Load(bytes.NewReader(b))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

caddyconfig/httpcaddyfile/addresses_fuzz.go

@@ -0,0 +1,29 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package httpcaddyfile
+
+func FuzzParseAddress(data []byte) int {
+	addr, err := ParseAddress(string(data))
+	if err != nil {
+		if addr == (Address{}) {
+			return 1
+		}
+		return 0
+	}
+	return 1
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.0.0
 	github.com/andybalholm/brotli v0.0.0-20190821151343-b60f0d972eeb
 	github.com/dustin/go-humanize v1.0.0
+	github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 // indirect
 	github.com/go-acme/lego/v3 v3.1.0
 	github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc
 	github.com/ilibs/json5 v1.0.1

go.sum

@@ -59,6 +59,8 @@ github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s9
 github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 h1:3WV5aRRj1ELP3RcLlBp/v0WJTuy47OQMkL9GIQq8QEE=
+github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=

listeners_fuzz.go

@@ -0,0 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+func FuzzParseNetworkAddress(data []byte) int {
+	_, _, err := ParseNetworkAddress(string(data))
+	if err != nil {
+		return 0
+	}
+	return 1
+}

replacer_fuzz.go

@@ -0,0 +1,26 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build gofuzz
+// +build gofuzz_libfuzzer
+
+package caddy
+
+func FuzzReplacer(data []byte) (score int) {
+	NewReplacer().ReplaceAll(string(data), "")
+	NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), "")
+	NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), NewReplacer().ReplaceAll(string(data), ""))
+	NewReplacer().ReplaceAll(string(data[:len(data)/2]), string(data[len(data)/2:]))
+	return 0
+}