mirror of
https://github.com/kopia/kopia.git
synced 2026-01-05 13:07:53 -05:00
0f7253eb66
This is a breaking change to users who might be using Kopia as a library.
### Log Format
```json
{"t":"<timestamp-rfc-3389-microseconds>", "span:T1":"V1", "span:T2":"V2", "n":"<source>", "m":"<message>", /*parameters*/}
```
Where each record is associated with one or more spans that describe its scope:
* `"span:client": "<hash-of-username@hostname>"`
* `"span:repo": "<random>"` - random identifier of a repository connection (from `repo.Open`)
* `"span:maintenance": "<random>"` - random identifier of a maintenance session
* `"span:upload": "<hash-of-username@host:/path>"` - uniquely identifies upload session of a given directory
* `"span:checkpoint": "<random>"` - encapsulates each checkpoint operation during Upload
* `"span:server-session": "<random>"` -single client connection to the server
* `"span:flush": "<random>"` - encapsulates each Flush session
* `"span:maintenance": "<random>"` - encapsulates each maintenance operation
* `"span:loadIndex" : "<random>"` - encapsulates index loading operation
* `"span:emr" : "<random>"` - encapsulates epoch manager refresh
* `"span:writePack": "<pack-blob-ID>"` - encapsulates pack blob preparation and writing
(plus additional minor spans for various phases of the maintenance).
Notable points:
- Used internal zero allocation JSON writer for reduced memory usage.
- renamed `--disable-internal-log` to `--disable-repository-log` (controls saving blobs to repository)
- added `--disable-content-log` (controls writing of `content-log` files)
- all storage operations are also logged in a structural way and associated with the corresponding spans.
- all content IDs are logged in a truncated format (since first N bytes that are usually enough to be unique) to improve compressibility of logs (blob IDs are frequently repeated but content IDs usually appear just once).
This format should make it possible to recreate the journey of any single content throughout pack blobs, indexes and compaction events.
94 lines
2.4 KiB
Go
94 lines
2.4 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"os"
|
|
"sort"
|
|
|
|
"github.com/pkg/errors"
|
|
|
|
"github.com/kopia/kopia/internal/ospath"
|
|
"github.com/kopia/kopia/internal/serverapi"
|
|
"github.com/kopia/kopia/repo"
|
|
"github.com/kopia/kopia/snapshot"
|
|
"github.com/kopia/kopia/snapshot/policy"
|
|
)
|
|
|
|
func handleSourcesList(_ context.Context, rc requestContext) (any, *apiError) {
|
|
_, multiUser := rc.rep.(repo.DirectRepository)
|
|
|
|
resp := &serverapi.SourcesResponse{
|
|
Sources: []*serverapi.SourceStatus{},
|
|
LocalHost: rc.rep.ClientOptions().Hostname,
|
|
LocalUsername: rc.rep.ClientOptions().Username,
|
|
MultiUser: multiUser,
|
|
}
|
|
|
|
for src, v := range rc.srv.snapshotAllSourceManagers() {
|
|
if sourceMatchesURLFilter(src, rc.req.URL.Query()) {
|
|
resp.Sources = append(resp.Sources, v.Status())
|
|
}
|
|
}
|
|
|
|
sort.Slice(resp.Sources, func(i, j int) bool {
|
|
return resp.Sources[i].Source.String() < resp.Sources[j].Source.String()
|
|
})
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
func handleSourcesCreate(ctx context.Context, rc requestContext) (any, *apiError) {
|
|
var req serverapi.CreateSnapshotSourceRequest
|
|
|
|
if err := json.Unmarshal(rc.body, &req); err != nil {
|
|
return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body")
|
|
}
|
|
|
|
if req.Path == "" {
|
|
return nil, requestError(serverapi.ErrorMalformedRequest, "missing path")
|
|
}
|
|
|
|
if req.Policy == nil {
|
|
return nil, requestError(serverapi.ErrorMalformedRequest, "missing policy")
|
|
}
|
|
|
|
req.Path = ospath.ResolveUserFriendlyPath(req.Path, true)
|
|
|
|
_, err := os.Stat(req.Path)
|
|
if os.IsNotExist(err) {
|
|
return nil, requestError(serverapi.ErrorPathNotFound, "path does not exist")
|
|
}
|
|
|
|
if err != nil {
|
|
return nil, internalServerError(err)
|
|
}
|
|
|
|
sourceInfo := snapshot.SourceInfo{
|
|
UserName: rc.rep.ClientOptions().Username,
|
|
Host: rc.rep.ClientOptions().Hostname,
|
|
Path: req.Path,
|
|
}
|
|
|
|
resp := &serverapi.CreateSnapshotSourceResponse{}
|
|
|
|
if err = repo.WriteSession(ctx, rc.rep, repo.WriteSessionOptions{
|
|
Purpose: "handleSourcesCreate",
|
|
}, func(ctx context.Context, w repo.RepositoryWriter) error {
|
|
return policy.SetPolicy(ctx, w, sourceInfo, req.Policy)
|
|
}); err != nil {
|
|
return nil, internalServerError(errors.Wrap(err, "unable to set initial policy"))
|
|
}
|
|
|
|
manager := rc.srv.getOrCreateSourceManager(ctx, sourceInfo)
|
|
|
|
if req.CreateSnapshot {
|
|
resp.SnapshotStarted = true
|
|
|
|
userLog(ctx).Debugf("scheduling snapshot of %v immediately...", sourceInfo)
|
|
manager.scheduleSnapshotNow()
|
|
}
|
|
|
|
return resp, nil
|
|
}
|