added examples of using repository API

examples/repository/main.go

@@ -0,0 +1,40 @@
+// Command repository_api demonstrates the use of Kopia's Repository API.
+package main
+
+import (
+	"context"
+	"os"
+
+	"github.com/kopia/kopia/repo"
+	colorable "github.com/mattn/go-colorable"
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+)
+
+func main() {
+	ctx := context.Background()
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: colorable.NewColorableStderr()}).With().Timestamp().Logger()
+
+	if err := setupRepositoryAndConnect(ctx); err != nil {
+		log.Error().Msgf("unable to set up repository: %v", err)
+		os.Exit(1)
+	}
+
+	r, err := repo.Open(ctx, configFile, &repo.Options{})
+	if err != nil {
+		log.Error().Msgf("unable to open repository: %v", err)
+		os.Exit(1)
+	}
+	defer r.Close(ctx) //nolint:errcheck
+
+	uploadAndDownloadObjects(ctx, r)
+
+	blks, err := r.Blocks.ListBlocks("")
+	if err != nil {
+		log.Fatal().Msgf("err: %v")
+	}
+
+	for _, b := range blks {
+		log.Printf("found block: %v with length %v", b.BlockID, b.Length)
+	}
+}

examples/repository/setup_repository.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/kopia/kopia/auth"
+	"github.com/kopia/kopia/block"
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/storage/gcs"
+	"github.com/kopia/kopia/storage/logging"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	masterPassword = "my-password$!@#!@"
+	bucketName     = "kopia-example"
+	configFile     = "/tmp/kopia-example/config"
+	cacheDirectory = "/tmp/kopia-example/cache"
+)
+
+func setupRepositoryAndConnect(ctx context.Context) error {
+	// set up credentials
+	creds, err := auth.Password(masterPassword)
+	if err != nil {
+		return fmt.Errorf("invalid password: %v", err)
+	}
+
+	st, err := gcs.New(ctx, &gcs.Options{
+		BucketName: bucketName,
+	})
+	if err != nil {
+		return fmt.Errorf("unable to connect to storage: %v", err)
+	}
+
+	// set up logging so we can see what's going on
+	st = logging.NewWrapper(st)
+
+	// see if we already have the config file, if not connect.
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		// initialize repository
+		if err := repo.Initialize(ctx, st, &repo.NewRepositoryOptions{}, creds); err != nil {
+			log.Printf("unable to initialize repository: %v", err)
+		}
+
+		// now establish connection to repository and create configuration file.
+		if err := repo.Connect(ctx, configFile, st, creds, repo.ConnectOptions{
+			PersistCredentials: true,
+			CachingOptions: block.CachingOptions{
+				CacheDirectory:    cacheDirectory,
+				MaxCacheSizeBytes: 100000000,
+			},
+		}); err != nil {
+			return fmt.Errorf("unable to connect to repository: %v", err)
+		}
+	}
+
+	return nil
+}

examples/repository/upload_download_objects.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+
+	"github.com/kopia/kopia/object"
+	"github.com/kopia/kopia/repo"
+	"github.com/rs/zerolog/log"
+)
+
+func uploadRandomObject(ctx context.Context, r *repo.Repository, length int) (object.ID, error) {
+	w := r.Objects.NewWriter(ctx, object.WriterOptions{})
+	defer w.Close() //nolint:errcheck
+
+	buf := make([]byte, 256*1024)
+	for length > 0 {
+		todo := length
+		if todo > len(buf) {
+			todo = len(buf)
+		}
+		rand.Read(buf[0:todo]) //nolint:errcheck
+		if _, err := w.Write(buf[0:todo]); err != nil {
+			return object.NullID, err
+		}
+		length -= todo
+	}
+	return w.Result()
+}
+
+func downloadObject(ctx context.Context, r *repo.Repository, oid object.ID) ([]byte, error) {
+	rd, err := r.Objects.Open(ctx, oid)
+	if err != nil {
+		return nil, err
+	}
+	defer rd.Close() //nolint:errcheck
+
+	return ioutil.ReadAll(rd)
+}
+
+func uploadAndDownloadObjects(ctx context.Context, r *repo.Repository) {
+	var oids []object.ID
+
+	for size := 100; size < 100000000; size *= 2 {
+		log.Printf("uploading file with %v bytes", size)
+		oid, err := uploadRandomObject(ctx, r, size)
+		if err != nil {
+			log.Error().Msgf("unable to upload: %v", err)
+			os.Exit(1)
+		}
+		log.Printf("uploaded %v bytes as %v", size, oid)
+		oids = append(oids, oid)
+	}
+
+	for _, oid := range oids {
+		log.Info().Msgf("downloading %q", oid)
+		b, err := downloadObject(ctx, r, oid)
+		if err != nil {
+			log.Error().Msgf("unable to read object: %v", err)
+		}
+		log.Printf("downloaded %v", len(b))
+	}
+}