lib/bucket: utilities for dealing with bucket based backends #3421

lib/bucket/bucket.go

@@ -0,0 +1,166 @@
+// Package bucket is contains utilities for managing bucket based backends
+package bucket
+
+import (
+	"errors"
+	"strings"
+	"sync"
+)
+
+var (
+	// ErrAlreadyDeleted is returned when an already deleted
+	// bucket is passed to Remove
+	ErrAlreadyDeleted = errors.New("bucket already deleted")
+)
+
+// Split takes an absolute path which includes the bucket and
+// splits it into a bucket and a path in that bucket
+// bucketPath
+func Split(absPath string) (bucket, bucketPath string) {
+	// No bucket
+	if absPath == "" {
+		return "", ""
+	}
+	slash := strings.IndexRune(absPath, '/')
+	// Bucket but no path
+	if slash < 0 {
+		return absPath, ""
+	}
+	return absPath[:slash], absPath[slash+1:]
+}
+
+// Cache stores whether buckets are available and their IDs
+type Cache struct {
+	mu       sync.Mutex      // mutex to protect created and deleted
+	status   map[string]bool // true if we have created the container, false if deleted
+	createMu sync.Mutex      // mutex to protect against simultaneous Remove
+	removeMu sync.Mutex      // mutex to protect against simultaneous Create
+}
+
+// NewCache creates an empty Cache
+func NewCache() *Cache {
+	return &Cache{
+		status: make(map[string]bool, 1),
+	}
+}
+
+// MarkOK marks the bucket as being present
+func (c *Cache) MarkOK(bucket string) {
+	if bucket != "" {
+		c.mu.Lock()
+		c.status[bucket] = true
+		c.mu.Unlock()
+	}
+}
+
+// MarkDeleted marks the bucket as being deleted
+func (c *Cache) MarkDeleted(bucket string) {
+	if bucket != "" {
+		c.mu.Lock()
+		c.status[bucket] = false
+		c.mu.Unlock()
+	}
+}
+
+type (
+	// ExistsFn should be passed to Create to see if a bucket
+	// exists or not
+	ExistsFn func() (found bool, err error)
+
+	// CreateFn should be passed to Create to make a bucket
+	CreateFn func() error
+)
+
+// Create the bucket with create() if it doesn't exist
+//
+// If exists is set then if the bucket has been deleted it will call
+// exists() to see if it still exists.
+//
+// If f returns an error we assume the bucket was not created
+func (c *Cache) Create(bucket string, create CreateFn, exists ExistsFn) (err error) {
+	c.createMu.Lock()
+	defer c.createMu.Unlock()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// if we are at the root, then it is OK
+	if bucket == "" {
+		return nil
+	}
+
+	// if have exists fuction and bucket has been deleted, check
+	// it still exists
+	if created, ok := c.status[bucket]; ok && !created && exists != nil {
+		found, err := exists()
+		if err == nil {
+			c.status[bucket] = found
+		}
+		if err != nil || found {
+			return err
+		}
+	}
+
+	// If bucket already exists then it is OK
+	if created, ok := c.status[bucket]; ok && created {
+		return nil
+	}
+
+	// Create the bucket
+	c.mu.Unlock()
+	err = create()
+	c.mu.Lock()
+	if err != nil {
+		return err
+	}
+
+	// Mark OK if successful
+	c.status[bucket] = true
+	return nil
+}
+
+// Remove the bucket with f if it exists
+//
+// If f returns an error we assume the bucket was not removed
+//
+// If the bucket has already been deleted it returns ErrAlreadyDeleted
+func (c *Cache) Remove(bucket string, f func() error) error {
+	c.removeMu.Lock()
+	defer c.removeMu.Unlock()
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	// if we are at the root, then it is OK
+	if bucket == "" {
+		return nil
+	}
+
+	// If bucket already deleted then it is OK
+	if created, ok := c.status[bucket]; ok && !created {
+		return ErrAlreadyDeleted
+	}
+
+	// Remove the bucket
+	c.mu.Unlock()
+	err := f()
+	c.mu.Lock()
+	if err != nil {
+		return err
+	}
+
+	// Mark removed if successful
+	c.status[bucket] = false
+	return err
+}
+
+// IsDeleted returns true if the bucket has definitely been deleted by
+// us, false otherwise.
+func (c *Cache) IsDeleted(bucket string) bool {
+	c.mu.Lock()
+	created, ok := c.status[bucket]
+	c.mu.Unlock()
+	// if status unknown then return false
+	if !ok {
+		return false
+	}
+	return !created
+}

lib/bucket/bucket_test.go

@@ -0,0 +1,148 @@
+package bucket
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSplit(t *testing.T) {
+	for _, test := range []struct {
+		in         string
+		wantBucket string
+		wantPath   string
+	}{
+		{in: "", wantBucket: "", wantPath: ""},
+		{in: "bucket", wantBucket: "bucket", wantPath: ""},
+		{in: "bucket/path", wantBucket: "bucket", wantPath: "path"},
+		{in: "bucket/path/subdir", wantBucket: "bucket", wantPath: "path/subdir"},
+	} {
+		gotBucket, gotPath := Split(test.in)
+		assert.Equal(t, test.wantBucket, gotBucket, test.in)
+		assert.Equal(t, test.wantPath, gotPath, test.in)
+	}
+}
+
+func TestCache(t *testing.T) {
+	c := NewCache()
+	errBoom := errors.New("boom")
+
+	assert.Equal(t, 0, len(c.status))
+
+	// IsDeleted before creation
+	assert.False(t, c.IsDeleted("bucket"))
+
+	// MarkOK
+
+	c.MarkOK("")
+	assert.Equal(t, 0, len(c.status))
+
+	// MarkOK again
+
+	c.MarkOK("bucket")
+	assert.Equal(t, map[string]bool{"bucket": true}, c.status)
+
+	// MarkDeleted
+
+	c.MarkDeleted("bucket")
+	assert.Equal(t, map[string]bool{"bucket": false}, c.status)
+
+	// MarkOK again
+
+	c.MarkOK("bucket")
+	assert.Equal(t, map[string]bool{"bucket": true}, c.status)
+
+	// IsDeleted after creation
+	assert.False(t, c.IsDeleted("bucket"))
+
+	// Create from root
+
+	err := c.Create("", nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true}, c.status)
+
+	// Create bucket that is already OK
+
+	err = c.Create("bucket", nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true}, c.status)
+
+	// Create new bucket
+
+	err = c.Create("bucket2", func() error {
+		return nil
+	}, func() (bool, error) {
+		return true, nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true}, c.status)
+
+	// Create bucket that has been deleted with error
+
+	c.status["bucket2"] = false // mark bucket deleted
+	err = c.Create("bucket2", nil, func() (bool, error) {
+		return false, errBoom
+	})
+	assert.Equal(t, errBoom, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": false}, c.status)
+
+	// Create bucket that has been deleted with no error
+
+	err = c.Create("bucket2", nil, func() (bool, error) {
+		return true, nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true}, c.status)
+
+	// Create a new bucket with no exists function
+
+	err = c.Create("bucket3", func() error {
+		return nil
+	}, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": true}, c.status)
+
+	// Create a new bucket with no exists function with an error
+
+	err = c.Create("bucket4", func() error {
+		return errBoom
+	}, nil)
+	assert.Equal(t, errBoom, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": true}, c.status)
+
+	// Remove root
+
+	err = c.Remove("", func() error {
+		return nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": true}, c.status)
+
+	// Remove existing bucket
+
+	err = c.Remove("bucket3", func() error {
+		return nil
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": false}, c.status)
+
+	// IsDeleted after removal
+	assert.True(t, c.IsDeleted("bucket3"))
+
+	// Remove it again
+
+	err = c.Remove("bucket3", func() error {
+		return errBoom
+	})
+	assert.Equal(t, ErrAlreadyDeleted, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": false}, c.status)
+
+	// Remove bucket with error
+
+	err = c.Remove("bucket2", func() error {
+		return errBoom
+	})
+	assert.Equal(t, errBoom, err)
+	assert.Equal(t, map[string]bool{"bucket": true, "bucket2": true, "bucket3": false}, c.status)
+}