mirror/opencloud
1
0
Fork 0
mirror of https://github.com/opencloud-eu/opencloud.git synced 2026-05-18 13:35:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

add more upload session filters (#7933)

Browse Source 
* add more upload session filters

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* default descriptions

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* update readme

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* use tablewriter, --json flag

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

* update readme

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>

---------

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2024-01-26 09:21:32 +01:00
committed by GitHub
parent 6cf8546bf8
commit 3ed0424735
2 changed files with 204 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
services/storage-users/README.md
View File

@@ -36,10 +36,13 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo
Example cases for expired uploads
* When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared.
* In the final step the upload blob is moved from the upload area to the final blobstore (e.g. S3).
* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed.
There are two commands available to manage unfinished uploads
The admin can restart the postprocessing for this with the postprocessing cli.
The storage users service can only list and clean upload sessions:
```bash
ocis storage-users uploads <command>
@@ -47,21 +50,38 @@ ocis storage-users uploads <command>
```plaintext
COMMANDS:
list Print a list of all incomplete uploads
clean Clean up leftovers from expired uploads
sessions Print a list of upload sessions
clean Clean up leftovers from expired uploads
list Print a list of all incomplete uploads (deprecated)
```
#### Command Examples
Command to identify incomplete uploads
Command to list ongoing upload sessions
```bash
ocis storage-users uploads list
ocis storage-users sessions --expired=false
```
```plaintext
Incomplete uploads:
- 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
Not expired sessions:
+--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
| Space | Upload Id | Name | Offset | Size | Executant | Owner | Expires | Processing |
+--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 5e387954-7313-4223-a904-bf996da6ec0b | foo.txt | 0 | 1234 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:04:31+01:00 | false |
| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f066244d-97b2-48e7-a30d-b40fcb60cec6 | bar.txt | 0 | 4321 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:18:47+01:00 | false |
+--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+
```
The sessions command can also output json
```bash
ocis storage-users sessions --expired=false --json
```
```json
{"id":"5e387954-7313-4223-a904-bf996da6ec0b","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"foo.txt","offset":0,"size":1234,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:04:31+01:00","processing":false}
{"id":"f066244d-97b2-48e7-a30d-b40fcb60cec6","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"bar.txt","offset":0,"size":4321,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:18:47+01:00","processing":false}
```
Command to clear expired uploads
@@ -74,6 +94,17 @@ Cleaned uploads:
- 455bd640-cd08-46e8-a5a0-9304908bd40a (Filename: file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
```
Deprecated list command to identify unfinished uploads
```bash
ocis storage-users uploads list
```
```plaintext
Incomplete uploads:
- 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00)
```
### Purge Expired Space Trash-Bins Items
<!-- referencing: https://github.com/owncloud/ocis/pull/5500 -->

168
services/storage-users/pkg/command/uploads.go
View File

@@ -1,12 +1,18 @@
package command
import (
"encoding/json"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
tw "github.com/olekukonko/tablewriter"
"github.com/urfave/cli/v2"
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
"github.com/cs3org/reva/v2/pkg/storage"
"github.com/cs3org/reva/v2/pkg/storage/fs/registry"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
@@ -22,6 +28,7 @@ func Uploads(cfg *config.Config) *cli.Command {
Usage: "manage unfinished uploads",
Subcommands: []*cli.Command{
ListUploads(cfg),
ListUploadSessions(cfg),
PurgeExpiredUploads(cfg),
},
}
@@ -31,7 +38,7 @@ func Uploads(cfg *config.Config) *cli.Command {
func ListUploads(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "list",
Usage: "Print a list of all incomplete uploads",
Usage: "Print a list of all incomplete uploads (deprecated, use sessions)",
Before: func(c *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
@@ -50,7 +57,7 @@ func ListUploads(cfg *config.Config) *cli.Command {
managingFS, ok := fs.(storage.UploadSessionLister)
if !ok {
fmt.Fprintf(os.Stderr, "'%s' storage does not support listing expired uploads\n", cfg.Driver)
fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver)
os.Exit(1)
}
expired := false
@@ -69,7 +76,162 @@ func ListUploads(cfg *config.Config) *cli.Command {
}
}
// PurgeExpiredUploads is the entry point for the server command.
// ListUploadSessions prints a list of upload sessiens
func ListUploadSessions(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "sessions",
Usage: "Print a list of upload sessions",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "id",
DefaultText: "unset",
Usage: "filter sessions by upload session id",
},
&cli.BoolFlag{
Name: "processing",
DefaultText: "unset",
Usage: "filter sessions by processing status",
},
&cli.BoolFlag{
Name: "expired",
DefaultText: "unset",
Usage: "filter sessions by expired status",
},
&cli.BoolFlag{
Name: "json",
Usage: "output as json",
},
},
Before: func(c *cli.Context) error {
return configlog.ReturnFatal(parser.ParseConfig(cfg))
},
Action: func(c *cli.Context) error {
f, ok := registry.NewFuncs[cfg.Driver]
if !ok {
fmt.Fprintf(os.Stderr, "Unknown filesystem driver '%s'\n", cfg.Driver)
os.Exit(1)
}
drivers := revaconfig.StorageProviderDrivers(cfg)
fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil)
if err != nil {
fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver)
return err
}
managingFS, ok := fs.(storage.UploadSessionLister)
if !ok {
fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver)
os.Exit(1)
}
var b strings.Builder
filter := storage.UploadSessionFilter{}
if c.IsSet("processing") {
processingValue := c.Bool("processing")
filter.Processing = &processingValue
if !processingValue {
b.WriteString("Not ")
}
if b.Len() == 0 {
b.WriteString("Processing ")
} else {
b.WriteString("processing ")
}
}
if c.IsSet("expired") {
expiredValue := c.Bool("expired")
filter.Expired = &expiredValue
if !expiredValue {
if b.Len() == 0 {
b.WriteString("Not ")
} else {
b.WriteString(", not ")
}
}
if b.Len() == 0 {
b.WriteString("Expired ")
} else {
b.WriteString("expired ")
}
}
if b.Len() == 0 {
b.WriteString("Sessions")
} else {
b.WriteString("sessions")
}
if c.IsSet("id") {
idValue := c.String("id")
filter.ID = &idValue
b.WriteString(" with id '" + idValue + "'")
}
b.WriteString(":")
uploads, err := managingFS.ListUploadSessions(c.Context, filter)
if err != nil {
return err
}
var table *tw.Table
if c.Bool("json") {
for _, u := range uploads {
ref := u.Reference()
s := struct {
ID string `json:"id"`
Space string `json:"space"`
Filename string `json:"filename"`
Offset int64 `json:"offset"`
Size int64 `json:"size"`
Executant userpb.UserId `json:"executant"`
SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"`
Expires time.Time `json:"expires"`
Processing bool `json:"processing"`
}{
Space: ref.GetResourceId().GetSpaceId(),
ID: u.ID(),
Filename: u.Filename(),
Offset: u.Offset(),
Size: u.Size(),
Executant: u.Executant(),
SpaceOwner: u.SpaceOwner(),
Expires: u.Expires(),
Processing: u.IsProcessing(),
}
j, err := json.Marshal(s)
if err != nil {
fmt.Println(err)
}
fmt.Println(string(j))
}
} else {
// Print what the user requested
fmt.Println(b.String())
// start a table
table = tw.NewWriter(os.Stdout)
table.SetHeader([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing"})
table.SetAutoFormatHeaders(false)
for _, u := range uploads {
table.Append([]string{
u.Reference().ResourceId.GetSpaceId(),
u.ID(),
u.Filename(),
strconv.FormatInt(u.Offset(), 10),
strconv.FormatInt(u.Size(), 10),
u.Executant().OpaqueId,
u.SpaceOwner().GetOpaqueId(),
u.Expires().Format(time.RFC3339),
strconv.FormatBool(u.IsProcessing()),
})
}
table.Render()
}
return nil
},
}
}
// PurgeExpiredUploads is the entry point for the clean command
func PurgeExpiredUploads(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "clean",