github-mirror/rclone
2
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2024-11-26 10:13:52 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

serve restic: serves a remote in restic REST API format

Browse Source
This commit is contained in:
Nick Craig-Wood 2018-02-17 23:25:59 +00:00
parent 9e48748182
commit d073efdc6c
3 changed files with 489 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

392
cmd/serve/restic/restic.go Normal file
View File

@ -0,0 +1,392 @@
// Package restic serves a remote suitable for use with restic
package restic
import (
"encoding/json"
"fmt"
"io"
"net/http"
"path"
"regexp"
"strconv"
"strings"
"time"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/cmd/serve/httplib/httpflags"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/object"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fs/walk"
"github.com/spf13/cobra"
)
func init() {
httpflags.AddFlags(Command.Flags())
}
// Command definition for cobra
var Command = &cobra.Command{
Use: "restic remote:path",
Short: `Serve the remote for restic's REST API.`,
Long: `rclone serve restic implements restic's REST backend API
over HTTP. This allows restic to use rclone as a data storage
mechanism for cloud providers that restic does not support directly.
[Restic](https://restic.net/) is a command line program for doing
backups.
The server will log errors. Use -v to see access logs.
--bwlimit will be respected for file transfers. Use --stats to
control the stats printing.
### Setting up rclone for use by restic ###
First [set up a remote for your chosen cloud provider](/docs/#configure).
Once you have set up the remote, check it is working with, for example
"rclone lsd remote:". You may have called the remote something other
than "remote:" - just substitute whatever you called it in the
following instructions.
Now start the rclone restic server
rclone serve restic -v remote:backup
Where you can replace "backup" in the above by whatever path in the
remote you wish to use.
By default this will serve on "localhost:8080" you can change this
with use of the "--addr" flag.
You might wish to start this server on boot.
### Setting up restic to use rclone ###
Now you can [follow the restic
instructions](http://restic.readthedocs.io/en/latest/030_preparing_a_new_repo.html#rest-server)
on setting up restic.
Note that you will need restic 0.8.2 or later to interoperate with
rclone.
For the example above you will want to use "http://localhost:8080/" as
the URL for the REST server.
For example:
$ export RESTIC_REPOSITORY=rest:http://localhost:8080/
$ export RESTIC_PASSWORD=yourpassword
$ restic init
created restic backend 8b1a4b56ae at rest:http://localhost:8080/
Please note that knowledge of your password is required to access
the repository. Losing your password means that your data is
irrecoverably lost.
$ restic backup /path/to/files/to/backup
scan [/path/to/files/to/backup]
scanned 189 directories, 312 files in 0:00
[0:00] 100.00% 38.128 MiB / 38.128 MiB 501 / 501 items 0 errors ETA 0:00
duration: 0:00
snapshot 45c8fdd8 saved
#### Multiple repositories ####
Note that you can use the endpoint to host multiple repositories. Do
this by adding a directory name or path after the URL. Note that
these **must** end with /. Eg
$ export RESTIC_REPOSITORY=rest:http://localhost:8080/user1repo/
# backup user1 stuff
$ export RESTIC_REPOSITORY=rest:http://localhost:8080/user2repo/
# backup user2 stuff
` + httplib.Help,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
f := cmd.NewFsSrc(args)
cmd.Run(false, true, command, func() error {
s := newServer(f, &httpflags.Opt)
s.serve()
return nil
})
},
}
// server contains everything to run the server
type server struct {
f fs.Fs
srv *httplib.Server
}
func newServer(f fs.Fs, opt *httplib.Options) *server {
mux := http.NewServeMux()
s := &server{
f: f,
srv: httplib.NewServer(mux, opt),
}
mux.HandleFunc("/", s.handler)
return s
}
// serve runs the http server - doesn't return
func (s *server) serve() {
fs.Logf(s.f, "Serving restic REST API on %s", s.srv.URL())
s.srv.Serve()
}
var matchData = regexp.MustCompile("(?:^|/)data/([^/]{2,})$")
// Makes a remote from a URL path. This implements the backend layout
// required by restic.
func makeRemote(path string) string {
path = strings.Trim(path, "/")
parts := matchData.FindStringSubmatch(path)
// if no data directory, layout is flat
if parts == nil {
return path
}
// otherwise map
// data/2159dd48 to
// data/21/2159dd48
fileName := parts[1]
prefix := path[:len(path)-len(fileName)]
return prefix + fileName[:2] + "/" + fileName
}
// handler reads incoming requests and dispatches them
func (s *server) handler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Accept-Ranges", "bytes")
w.Header().Set("Server", "rclone/"+fs.Version)
path := r.URL.Path
remote := makeRemote(path)
fs.Debugf(s.f, "%s %s", r.Method, path)
// Dispatch on path then method
if strings.HasSuffix(path, "/") {
switch r.Method {
case "GET":
s.listObjects(w, r, remote)
case "POST":
s.createRepo(w, r, remote)
default:
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
}
} else {
switch r.Method {
case "GET":
s.getObject(w, r, remote)
case "HEAD":
s.headObject(w, r, remote)
case "POST":
s.postObject(w, r, remote)
case "DELETE":
s.deleteObject(w, r, remote)
default:
http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed)
}
}
}
// head request the remote
func (s *server) headObject(w http.ResponseWriter, r *http.Request, remote string) {
o, err := s.f.NewObject(remote)
if err != nil {
fs.Debugf(remote, "Head request error: %v", err)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
// Set content length since we know how long the object is
w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
}
// get the remote
func (s *server) getObject(w http.ResponseWriter, r *http.Request, remote string) {
o, err := s.f.NewObject(remote)
if err != nil {
fs.Debugf(remote, "Get request error: %v", err)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
// Set content length since we know how long the object is
w.Header().Set("Content-Length", strconv.FormatInt(o.Size(), 10))
// Decode Range request if present
code := http.StatusOK
size := o.Size()
var options []fs.OpenOption
if rangeRequest := r.Header.Get("Range"); rangeRequest != "" {
//fs.Debugf(nil, "Range: request %q", rangeRequest)
option, err := fs.ParseRangeOption(rangeRequest)
if err != nil {
fs.Debugf(remote, "Get request parse range request error: %v", err)
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
options = append(options, option)
offset, limit := option.Decode(o.Size())
end := o.Size() // exclusive
if limit >= 0 {
end = offset + limit
}
if end > o.Size() {
end = o.Size()
}
size = end - offset
// fs.Debugf(nil, "Range: offset=%d, limit=%d, end=%d, size=%d (object size %d)", offset, limit, end, size, o.Size())
// Content-Range: bytes 0-1023/146515
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, end-1, o.Size()))
// fs.Debugf(nil, "Range: Content-Range: %q", w.Header().Get("Content-Range"))
code = http.StatusPartialContent
}
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
file, err := o.Open(options...)
if err != nil {
fs.Debugf(remote, "Get request open error: %v", err)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
defer func() {
err := file.Close()
if err != nil {
fs.Errorf(remote, "Get request: close failed: %v", err)
}
}()
w.WriteHeader(code)
_, err = io.Copy(w, file)
if err != nil {
fs.Errorf(remote, "Didn't finish writing GET request: %v", err)
return
}
}
// postObject posts an object to the repository
func (s *server) postObject(w http.ResponseWriter, r *http.Request, remote string) {
// fs.Debugf(s.f, "content length = %d", r.ContentLength)
if r.ContentLength >= 0 {
// Size known use Put
info := object.NewStaticObjectInfo(remote, time.Now(), r.ContentLength, true, nil, s.f)
_, err := s.f.Put(r.Body, info)
if err != nil {
fs.Errorf(remote, "Post request put error: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
} else {
// Size unknown use Rcat
_, err := operations.Rcat(s.f, remote, r.Body, time.Now())
if err != nil {
fs.Errorf(remote, "Post request rcat error: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
}
}
// delete the remote
func (s *server) deleteObject(w http.ResponseWriter, r *http.Request, remote string) {
o, err := s.f.NewObject(remote)
if err != nil {
fs.Debugf(remote, "Delete request error: %v", err)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
if err := o.Remove(); err != nil {
fs.Errorf(remote, "Delete request remove error: %v", err)
if err == fs.ErrorObjectNotFound {
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
} else {
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}
return
}
}
// listItem is an element returned for the restic v2 list response
type listItem struct {
Name string `json:"name"`
Size int64 `json:"size"`
}
// return type for list
type listItems []listItem
// add a DirEntry to the listItems
func (ls *listItems) add(entry fs.DirEntry) {
if o, ok := entry.(fs.Object); ok {
*ls = append(*ls, listItem{
Name: path.Base(o.Remote()),
Size: o.Size(),
})
}
}
// listObjects lists all Objects of a given type in an arbitrary order.
func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote string) {
fs.Debugf(remote, "list request")
var ls listItems
err := walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error {
if err == nil {
for _, entry := range entries {
ls.add(entry)
}
}
return err
})
if err != nil {
fs.Errorf(remote, "list failed: %v", err)
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
return
}
data, err := json.Marshal(ls)
if err != nil {
fs.Errorf(remote, "list marshal failed: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
// fs.Debugf(remote, "ls result = %s", data)
w.Header().Set("Content-Type", "application/vnd.x.restic.rest.v2")
_, _ = w.Write(data)
}
// createRepo creates repository directories.
//
// We don't bother creating the data dirs as rclone will create them on the fly
func (s *server) createRepo(w http.ResponseWriter, r *http.Request, remote string) {
fs.Infof(remote, "Creating repository")
if r.URL.Query().Get("create") != "true" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
err := s.f.Mkdir(remote)
if err != nil {
fs.Errorf(remote, "Create repo failed to Mkdir: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
for _, name := range []string{"data", "index", "keys", "locks", "snapshots"} {
dirRemote := path.Join(remote, name)
err := s.f.Mkdir(dirRemote)
if err != nil {
fs.Errorf(dirRemote, "Create repo failed to Mkdir: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
}
}

95
cmd/serve/restic/restic_test.go Normal file
View File

@ -0,0 +1,95 @@
// Serve restic tests set up a server and run the integration tests
// for restic against it.
package restic
import (
"os"
"os/exec"
"testing"
_ "github.com/ncw/rclone/backend/local"
"github.com/ncw/rclone/cmd/serve/httplib"
"github.com/ncw/rclone/fstest"
"github.com/stretchr/testify/assert"
)
const (
testBindAddress = "localhost:51779"
testURL = "http://" + testBindAddress + "/"
resticSource = "../../../../../restic/restic"
)
// TestRestic runs the restic server then runs the unit tests for the
// restic remote against it.
func TestRestic(t *testing.T) {
_, err := os.Stat(resticSource)
if err != nil {
t.Skipf("Skipping test as restic source not found: %v", err)
}
opt := httplib.DefaultOpt
opt.ListenAddr = testBindAddress
fstest.Initialise()
fremote, _, clean, err := fstest.RandomRemote(*fstest.RemoteName, *fstest.SubDir)
assert.NoError(t, err)
defer clean()
err = fremote.Mkdir("")
assert.NoError(t, err)
// Start the server
w := newServer(fremote, &opt)
go w.serve()
defer w.srv.Close()
// Change directory to run the tests
err = os.Chdir(resticSource)
assert.NoError(t, err, "failed to cd to restic source code")
// Run the restic tests
runTests := func(path string) {
args := []string{"test", "./internal/backend/rest", "-run", "TestBackendRESTExternalServer", "-count=1"}
if testing.Verbose() {
args = append(args, "-v")
}
cmd := exec.Command("go", args...)
cmd.Env = append(os.Environ(),
"RESTIC_TEST_REST_REPOSITORY=rest:"+testURL+path,
)
out, err := cmd.CombinedOutput()
if len(out) != 0 {
t.Logf("\n----------\n%s----------\n", string(out))
}
assert.NoError(t, err, "Running restic integration tests")
}
// Run the tests with no path
runTests("")
//... and again with a path
runTests("potato/sausage/")
}
func TestMakeRemote(t *testing.T) {
for _, test := range []struct {
in, want string
}{
{"", ""},
{"/", ""},
{"/data", "data"},
{"/data/", "data"},
{"/data/1", "data/1"},
{"/data/12", "data/12/12"},
{"/data/123", "data/12/123"},
{"/data/123/", "data/12/123"},
{"/keys", "keys"},
{"/keys/1", "keys/1"},
{"/keys/12", "keys/12"},
{"/keys/123", "keys/123"},
} {
got := makeRemote(test.in)
assert.Equal(t, test.want, got, test.in)
}
}

2
cmd/serve/serve.go
View File

@ -5,6 +5,7 @@ import (
"github.com/ncw/rclone/cmd" "github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/serve/http" "github.com/ncw/rclone/cmd/serve/http"
"github.com/ncw/rclone/cmd/serve/restic"
"github.com/ncw/rclone/cmd/serve/webdav" "github.com/ncw/rclone/cmd/serve/webdav"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -12,6 +13,7 @@ import (
func init() { func init() {
Command.AddCommand(http.Command) Command.AddCommand(http.Command)
Command.AddCommand(webdav.Command) Command.AddCommand(webdav.Command)
Command.AddCommand(restic.Command)
cmd.Root.AddCommand(Command) cmd.Root.AddCommand(Command)
} }