cache: cache lists using batch writes

backend/cache/cache.go

@@ -725,6 +725,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	fs.Debugf(dir, "list: source entries: %v", entries)
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
+	var batchDirectories []*Directory
 	for _, entry := range entries {
 		switch o := entry.(type) {
 		case fs.Object:
@@ -746,18 +747,19 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			cdd := DirectoryFromOriginal(f, o)
 			// check if the dir isn't expired and add it in cache if it isn't
 			if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
-				err := f.cache.AddDir(cdd)
+				batchDirectories = append(batchDirectories, cdd)
-				if err != nil {
-					fs.Errorf(dir, "list: error caching dir from listing %v", o)
-				} else {
-					fs.Debugf(dir, "list: cached dir: %v", cdd)
-				}
 			}
 			cachedEntries = append(cachedEntries, cdd)
 		default:
 			fs.Debugf(entry, "list: Unknown object type %T", entry)
 		}
 	}
+	err = f.cache.AddBatchDir(batchDirectories)
+	if err != nil {
+		fs.Errorf(dir, "list: error caching directories from listing %v", dir)
+	} else {
+		fs.Debugf(dir, "list: cached directories: %v", len(batchDirectories))
+	}
 
 	// cache dir meta
 	t := time.Now()

backend/cache/storage_persistent.go

@@ -192,20 +192,36 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
 
 // AddDir will update a CachedDirectory metadata and all its entries
 func (b *Persistent) AddDir(cachedDir *Directory) error {
+	return b.AddBatchDir([]*Directory{cachedDir})
+}
+
+// AddBatchDir will update a list of CachedDirectory metadata and all their entries
+func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
+	if len(cachedDirs) == 0 {
+		return nil
+	}
+
 	return b.db.Update(func(tx *bolt.Tx) error {
-		bucket := b.getBucket(cachedDir.abs(), true, tx)
+		bucket := b.getBucket(cachedDirs[0].Dir, true, tx)
 		if bucket == nil {
-			return errors.Errorf("couldn't open bucket (%v)", cachedDir)
+			return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
+		}
+
+		for _, cachedDir := range cachedDirs {
+			b, err := bucket.CreateBucketIfNotExists([]byte(cachedDir.Name))
+			if err != nil {
+				return err
 			}
 
 			encoded, err := json.Marshal(cachedDir)
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
 			}
-		err = bucket.Put([]byte("."), encoded)
+			err = b.Put([]byte("."), encoded)
 			if err != nil {
 				return err
 			}
+		}
 		return nil
 	})
 }