fuzz: introduce continuous fuzzing for Caddy (#2723)

* fuzz: lay down the foundation for continuous fuzzing

* improve the fuzzers and add some

* fuzz: add Fuzzit badge to README & enable fuzzers submission in CI

* v2-fuzz: do away with the submodule approach for fuzzers

* fuzz: enable fuzzit
This commit is contained in:
Mohammed Al Sahaf 2019-10-26 03:52:16 +03:00 committed by Matt Holt
parent faf67b1067
commit 2fbe2ff40b
9 changed files with 377 additions and 116 deletions

View File

@ -2,6 +2,7 @@ Caddy 2 Development Branch
===========================
[![Build Status](https://dev.azure.com/mholt-dev/Caddy/_apis/build/status/Multiplatform%20Tests?branchName=v2)](https://dev.azure.com/mholt-dev/Caddy/_build/latest?definitionId=5&branchName=v2)
[![fuzzit](https://app.fuzzit.dev/badge?org_id=caddyserver)](https://app.fuzzit.dev/orgs/caddyserver/dashboard)
This is the development branch for Caddy 2. This code (version 2) is not yet feature-complete or production-ready, but is already being used in production, and we encourage you to deploy it today on sites that are not very visible or important so that it can obtain crucial experience in the field.

30
admin_fuzz.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build gofuzz
// +build gofuzz_libfuzzer
package caddy
import (
"bytes"
)
func FuzzAdmin(data []byte) (score int) {
err := Load(bytes.NewReader(data))
if err != nil {
return 0
}
return 1
}

View File

@ -2,21 +2,15 @@
# https://docs.microsoft.com/azure/devops/pipelines/languages/go
trigger:
- v2
- v2
strategy:
matrix:
linux:
imageName: ubuntu-16.04
gorootDir: /usr/local
mac:
imageName: macos-10.13
gorootDir: /usr/local
windows:
imageName: windows-2019
gorootDir: C:\
pool:
vmImage: $(imageName)
schedules:
- cron: "0 0 * * *"
displayName: Daily midnight fuzzing
branches:
include:
- v2
always: true
variables:
GOROOT: $(gorootDir)/go
@ -26,121 +20,224 @@ variables:
# TODO: Remove once it's enabled by default
GO111MODULE: on
steps:
- bash: |
latestGo=$(curl "https://golang.org/VERSION?m=text")
echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
echo "Latest Go version: $latestGo"
displayName: "Get latest Go version"
jobs:
- job: crossPlatformTest
displayName: "Cross-Platform Tests"
strategy:
matrix:
linux:
imageName: ubuntu-16.04
gorootDir: /usr/local
mac:
imageName: macos-10.13
gorootDir: /usr/local
windows:
imageName: windows-2019
gorootDir: C:\
pool:
vmImage: $(imageName)
- bash: |
sudo rm -f $(which go)
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
steps:
- bash: |
latestGo=$(curl "https://golang.org/VERSION?m=text")
echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
echo "Latest Go version: $latestGo"
displayName: "Get latest Go version"
# Install Go (this varies by platform)
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install Go on Linux
- bash: |
sudo rm -f $(which go)
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Darwin' )
displayName: Install Go on macOS
# Install Go (this varies by platform)
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install Go on Linux
# The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
# Reference: https://github.com/PowerShell/PowerShell/issues/2138
- powershell: |
$ProgressPreference = 'SilentlyContinue'
Write-Host "Downloading Go..."
(New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
Write-Host "Extracting Go... (I'm slow too)"
7z x "$(LATEST_GO).windows-amd64.zip" -o"$(gorootDir)"
condition: eq( variables['Agent.OS'], 'Windows_NT' )
displayName: Install Go on Windows
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Darwin' )
displayName: Install Go on macOS
- bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
displayName: Install golangci-lint
# The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
# Reference: https://github.com/PowerShell/PowerShell/issues/2138
- powershell: |
$ProgressPreference = 'SilentlyContinue'
Write-Host "Downloading Go..."
(New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
Write-Host "Extracting Go... (I'm slow too)"
7z x "$(LATEST_GO).windows-amd64.zip" -o"$(gorootDir)"
condition: eq( variables['Agent.OS'], 'Windows_NT' )
displayName: Install Go on Windows
- script: |
go get github.com/axw/gocov/gocov
go get github.com/AlekSi/gocov-xml
go get -u github.com/jstemmer/go-junit-report
displayName: Install test and coverage analysis tools
- bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.19.1
displayName: Install golangci-lint
- bash: |
printf "Using go at: $(which go)\n"
printf "Go version: $(go version)\n"
printf "\n\nGo environment:\n\n"
go env
printf "\n\nSystem environment:\n\n"
env
displayName: Print Go version and environment
- script: |
go get github.com/axw/gocov/gocov
go get github.com/AlekSi/gocov-xml
go get -u github.com/jstemmer/go-junit-report
displayName: Install test and coverage analysis tools
- script: |
go get -v -t -d ./...
mkdir test-results
workingDirectory: '$(modulePath)'
displayName: Get dependencies
- bash: |
printf "Using go at: $(which go)\n"
printf "Go version: $(go version)\n"
printf "\n\nGo environment:\n\n"
go env
printf "\n\nSystem environment:\n\n"
env
displayName: Print Go version and environment
# its behavior is governed by .golangci.yml
- script: |
(golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
exit 0
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run lint check
- script: |
go get -v -t -d ./...
mkdir test-results
workingDirectory: '$(modulePath)'
displayName: Get dependencies
- script: |
(go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run tests
# its behavior is governed by .golangci.yml
- script: |
(golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
exit 0
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run lint check
- script: |
mkdir coverage
gocov convert cover-profile.out > coverage/coverage.json
# Because Windows doesn't work with input redirection like *nix, but output redirection works.
(cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
workingDirectory: '$(modulePath)'
displayName: Prepare coverage reports
- script: |
(go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run tests
- script: |
(cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
workingDirectory: '$(modulePath)'
displayName: Prepare test report
- script: |
mkdir coverage
gocov convert cover-profile.out > coverage/coverage.json
# Because Windows doesn't work with input redirection like *nix, but output redirection works.
(cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
workingDirectory: '$(modulePath)'
displayName: Prepare coverage reports
- task: PublishCodeCoverageResults@1
displayName: Publish test coverage report
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(modulePath)/coverage/coverage.xml
- script: |
(cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
workingDirectory: '$(modulePath)'
displayName: Prepare test report
- task: PublishTestResults@2
displayName: Publish unit test
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/test-result.xml
testRunTitle: $(agent.OS) Unit Test
mergeTestResults: false
- task: PublishCodeCoverageResults@1
displayName: Publish test coverage report
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(modulePath)/coverage/coverage.xml
- task: PublishTestResults@2
displayName: Publish lint results
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/lint-result.xml
testRunTitle: $(agent.OS) Lint
mergeTestResults: false
- task: PublishTestResults@2
displayName: Publish unit test
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/test-result.xml
testRunTitle: $(agent.OS) Unit Test
mergeTestResults: false
- bash: |
exit 1
condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
displayName: Coerce correct build result
- task: PublishTestResults@2
displayName: Publish lint results
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/lint-result.xml
testRunTitle: $(agent.OS) Lint
mergeTestResults: false
- bash: |
exit 1
condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
displayName: Coerce correct build result
- job: fuzzing
displayName: 'Scheduled Fuzzing'
# Only run this job on schedules, not PRs.
condition: eq(variables['Build.Reason'], 'Schedule')
strategy:
matrix:
linux:
imageName: ubuntu-16.04
gorootDir: /usr/local
pool:
vmImage: $(imageName)
steps:
- bash: |
latestGo=$(curl "https://golang.org/VERSION?m=text")
echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
echo "Latest Go version: $latestGo"
displayName: "Get latest Go version"
- bash: |
sudo rm -f $(which go)
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install Go on Linux
- bash: |
# Install Clang
sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main"
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo apt update && sudo apt install -y clang lldb lld
go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.74/fuzzit_Linux_x86_64
chmod a+x fuzzit
mv fuzzit $(GOBIN)
displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
- script: fuzzit auth ${FUZZIT_API_KEY}
condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
displayName: Authenticate with Fuzzit
env:
FUZZIT_API_KEY: $(FUZZIT_API_KEY)
- bash: |
declare -A fuzzers_funcs=(\
["./admin_fuzz.go"]="FuzzAdmin" \
["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="FuzzHTTPCaddyfileAdapter" \
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
["./replacer_fuzz.go"]="FuzzReplacer" \
)
declare -A fuzzers_targets=(\
["./admin_fuzz.go"]="admin" \
["./caddyconfig/httpcaddyfile/adapter_fuzz.go"]="caddyfile-adapter" \
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
["./listeners_fuzz.go"]="parse-listen-addr" \
["./replacer_fuzz.go"]="replacer" \
)
fuzz_type="fuzzing"
for f in $(find . -name \*_fuzz.go); do
FUZZER_DIRECTORY=$(dirname $f)
echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
clang -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer"
fuzzit create job --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}" caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]}.fuzzer
echo "Completed $f"
done
workingDirectory: '$(modulePath)'
displayName: Generate fuzzers & submit them to Fuzzit

View File

@ -0,0 +1,49 @@
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build gofuzz
// +build gofuzz_libfuzzer
package httpcaddyfile
import (
"bytes"
"github.com/caddyserver/caddy/v2"
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
)
func FuzzHTTPCaddyfileAdapter(data []byte) int {
adapter := caddyfile.Adapter{
ServerType: ServerType{},
}
b, warns, err := adapter.Adapt(data, nil)
// Adapt func calls the Setup() func of the ServerType,
// thus it's going across multiple layers, each can
// return warnings or errors. Marking the presence of
// errors or warnings as interesting in this case
// could push the fuzzer towards a path where we only
// catch errors. Let's push the fuzzer to where it passes
// but breaks.
if (err != nil) || (warns != nil && len(warns) > 0) {
return 0
}
// adapted Caddyfile should be parseable by the configuration loader in admin.go
err = caddy.Load(bytes.NewReader(b))
if err != nil {
return 0
}
return 1
}

View File

@ -0,0 +1,29 @@
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build gofuzz
// +build gofuzz_libfuzzer
package httpcaddyfile
func FuzzParseAddress(data []byte) int {
addr, err := ParseAddress(string(data))
if err != nil {
if addr == (Address{}) {
return 1
}
return 0
}
return 1
}

1
go.mod
View File

@ -6,6 +6,7 @@ require (
github.com/Masterminds/sprig/v3 v3.0.0
github.com/andybalholm/brotli v0.0.0-20190821151343-b60f0d972eeb
github.com/dustin/go-humanize v1.0.0
github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 // indirect
github.com/go-acme/lego/v3 v3.1.0
github.com/golang/groupcache v0.0.0-20191002201903-404acd9df4cc
github.com/ilibs/json5 v1.0.1

2
go.sum
View File

@ -59,6 +59,8 @@ github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s9
github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681 h1:3WV5aRRj1ELP3RcLlBp/v0WJTuy47OQMkL9GIQq8QEE=
github.com/dvyukov/go-fuzz v0.0.0-20191022152526-8cb203812681/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=

26
listeners_fuzz.go Normal file
View File

@ -0,0 +1,26 @@
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build gofuzz
// +build gofuzz_libfuzzer
package caddy
func FuzzParseNetworkAddress(data []byte) int {
_, _, err := ParseNetworkAddress(string(data))
if err != nil {
return 0
}
return 1
}

26
replacer_fuzz.go Normal file
View File

@ -0,0 +1,26 @@
// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build gofuzz
// +build gofuzz_libfuzzer
package caddy
func FuzzReplacer(data []byte) (score int) {
NewReplacer().ReplaceAll(string(data), "")
NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), "")
NewReplacer().ReplaceAll(NewReplacer().ReplaceAll(string(data), ""), NewReplacer().ReplaceAll(string(data), ""))
NewReplacer().ReplaceAll(string(data[:len(data)/2]), string(data[len(data)/2:]))
return 0
}