mirror of
https://github.com/opencloud-eu/opencloud.git
synced 2026-11-08 21:43:00 -05:00
build: bump reva to workpackages
Pins reva to the workpackages branch (2d4db7d27), which carries the upload coordinator and the other WP1 storage changes on top of reva main. Once workpackages lands in reva main this becomes a regular bump to main. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
1 parent
b2903fc9e5
commit
de3f0f80a8
130 files changed
+4183
-17911
No files matched your search
@@ -33,7 +33,7 @@ require (
|
||||
github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20240726082623-6831adfdcdc4
|
||||
github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0
|
||||
github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry v1.2.0
|
||||
github.com/go-playground/validator/v10 v10.30.3
|
||||
github.com/go-playground/validator/v10 v10.30.5
|
||||
github.com/go-resty/resty/v2 v2.17.2
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0
|
||||
github.com/gofrs/flock v0.13.1
|
||||
@@ -68,7 +68,7 @@ require (
|
||||
github.com/open-policy-agent/opa v1.21.1
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346
|
||||
github.com/opencloud-eu/reva/v2 v2.52.1
|
||||
github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc
|
||||
github.com/opensearch-project/opensearch-go/v4 v4.7.3
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
@@ -218,7 +218,7 @@ require (
|
||||
github.com/go-micro/plugins/v4/store/nats-js v1.2.1 // indirect
|
||||
github.com/go-micro/plugins/v4/store/redis v1.2.1 // indirect
|
||||
github.com/go-ole/go-ole v1.3.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/locales v0.14.2 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.2 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5 // indirect
|
||||
github.com/go-sql-driver/mysql v1.10.1 // indirect
|
||||
@@ -259,7 +259,7 @@ require (
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/kovidgoyal/go-parallel v1.1.1 // indirect
|
||||
github.com/kovidgoyal/go-shm v1.0.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/leodido/go-urn v1.5.0 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
|
||||
github.com/lestrrat-go/dsig v1.4.0 // indirect
|
||||
github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect
|
||||
|
||||
@@ -373,12 +373,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
|
||||
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/locales v0.14.2 h1:d8UmcrM6Nip0hfGZKLGpAvZH37XB4TS0xzK9B56YNCY=
|
||||
github.com/go-playground/locales v0.14.2/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.2 h1:LCsMLC9RzmbUMNUPVYD15dmcjwYAJhmX8mPZRW4rAVU=
|
||||
github.com/go-playground/universal-translator v0.18.2/go.mod h1:67VZIMp5lQpDWlnStOct22q1bkdJGJqHghbOtmkawxk=
|
||||
github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8=
|
||||
github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc=
|
||||
github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4=
|
||||
github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ=
|
||||
github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
|
||||
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
|
||||
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
|
||||
@@ -610,8 +610,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0=
|
||||
github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE=
|
||||
github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69 h1:ZLo0bXX17iUZpb/Sp7N+1uK4Jb7dl0fm5x5ETfzYSvY=
|
||||
github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69/go.mod h1:ksG5iXViKefoupjy+0qQjAVoaDnylnQ1ejWl9g14wh8=
|
||||
github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA=
|
||||
@@ -761,8 +761,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
|
||||
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 h1:3ptbiFhSK4fzY/rC3idcYUveiwepMoOov1sBpc7Rg3k=
|
||||
github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
|
||||
github.com/opencloud-eu/reva/v2 v2.52.1 h1:c4tjmqtbJeqDjztTNq4Jvl3dWWSMHhXqV1JixKLKbT4=
|
||||
github.com/opencloud-eu/reva/v2 v2.52.1/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8=
|
||||
github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc h1:ltmetYT21ZUBhP3cZD/l3cQ+TSHctayzSJpIyG2XdLI=
|
||||
github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc/go.mod h1:649GUlGdmY6ukU+7G7fHw+EQlZoLSLhztahgqI9TAe4=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
|
||||
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
|
||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||
|
||||
+7
-5
@@ -139,11 +139,13 @@ func main() {
|
||||
fmt.Println(l.CardinalPluralRule(1.0, 1))
|
||||
fmt.Println(l.CardinalPluralRule(3, 0))
|
||||
|
||||
// Ordinal Plural Rules
|
||||
fmt.Println(l.OrdinalPluralRule(21, 0)) // 21st
|
||||
fmt.Println(l.OrdinalPluralRule(22, 0)) // 22nd
|
||||
fmt.Println(l.OrdinalPluralRule(33, 0)) // 33rd
|
||||
fmt.Println(l.OrdinalPluralRule(34, 0)) // 34th
|
||||
// Ordinal Plural Rules. The method returns the locales/plurals.Rule
|
||||
// (e.g. One, Two, Few, Other) for the number; converting that rule into
|
||||
// the actual suffix ("st", "nd", "rd", "th") is left to the caller.
|
||||
fmt.Println(l.OrdinalPluralRule(21, 0)) // One (-> "21st")
|
||||
fmt.Println(l.OrdinalPluralRule(22, 0)) // Two (-> "22nd")
|
||||
fmt.Println(l.OrdinalPluralRule(33, 0)) // Few (-> "33rd")
|
||||
fmt.Println(l.OrdinalPluralRule(34, 0)) // Other (-> "34th")
|
||||
|
||||
// Range Plural Rules
|
||||
fmt.Println(l.RangePluralRule(1, 0, 1, 0)) // 1-1
|
||||
|
||||
+12
-16
@@ -498,10 +498,9 @@ func (en *en) FmtTimeShort(t time.Time) string {
|
||||
|
||||
b := make([]byte, 0, 32)
|
||||
|
||||
h := t.Hour()
|
||||
|
||||
if h > 12 {
|
||||
h -= 12
|
||||
h := t.Hour() % 12
|
||||
if h == 0 {
|
||||
h = 12
|
||||
}
|
||||
|
||||
b = strconv.AppendInt(b, int64(h), 10)
|
||||
@@ -528,10 +527,9 @@ func (en *en) FmtTimeMedium(t time.Time) string {
|
||||
|
||||
b := make([]byte, 0, 32)
|
||||
|
||||
h := t.Hour()
|
||||
|
||||
if h > 12 {
|
||||
h -= 12
|
||||
h := t.Hour() % 12
|
||||
if h == 0 {
|
||||
h = 12
|
||||
}
|
||||
|
||||
b = strconv.AppendInt(b, int64(h), 10)
|
||||
@@ -565,10 +563,9 @@ func (en *en) FmtTimeLong(t time.Time) string {
|
||||
|
||||
b := make([]byte, 0, 32)
|
||||
|
||||
h := t.Hour()
|
||||
|
||||
if h > 12 {
|
||||
h -= 12
|
||||
h := t.Hour() % 12
|
||||
if h == 0 {
|
||||
h = 12
|
||||
}
|
||||
|
||||
b = strconv.AppendInt(b, int64(h), 10)
|
||||
@@ -607,10 +604,9 @@ func (en *en) FmtTimeFull(t time.Time) string {
|
||||
|
||||
b := make([]byte, 0, 32)
|
||||
|
||||
h := t.Hour()
|
||||
|
||||
if h > 12 {
|
||||
h -= 12
|
||||
h := t.Hour() % 12
|
||||
if h == 0 {
|
||||
h = 12
|
||||
}
|
||||
|
||||
b = strconv.AppendInt(b, int64(h), 10)
|
||||
|
||||
+13
-10
@@ -98,7 +98,7 @@ type Translator interface {
|
||||
// returns the locales narrow weekday given the 'weekday' provided
|
||||
WeekdayNarrow(weekday time.Weekday) string
|
||||
|
||||
// WeekdaysNarrowreturns the locales narrow weekdays
|
||||
// WeekdaysNarrow returns the locales narrow weekdays
|
||||
WeekdaysNarrow() []string
|
||||
|
||||
// returns the locales short weekday given the 'weekday' provided
|
||||
@@ -113,7 +113,7 @@ type Translator interface {
|
||||
// returns the locales wide weekdays
|
||||
WeekdaysWide() []string
|
||||
|
||||
// The following Functions are common Formatting functionsfor the Translator's Locale
|
||||
// The following Functions are common Formatting functions for the Translator's Locale
|
||||
|
||||
// returns 'num' with digits/precision of 'v' for locale and handles both Whole and Real numbers based on 'v'
|
||||
FmtNumber(num float64, v uint64) string
|
||||
@@ -228,14 +228,14 @@ func W(n float64, v uint64) (w int64) {
|
||||
|
||||
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
|
||||
|
||||
// with either be '0' or '0.xxxx', so if 1 then w will be zero
|
||||
// will either be '0' or '0.xxxx', so if 1 then w will be zero
|
||||
// otherwise need to parse
|
||||
if len(s) != 1 {
|
||||
|
||||
s = s[2:]
|
||||
end := len(s) + 1
|
||||
end := 0
|
||||
|
||||
for i := end; i >= 0; i-- {
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
if s[i] != '0' {
|
||||
end = i + 1
|
||||
break
|
||||
@@ -253,7 +253,7 @@ func F(n float64, v uint64) (f int64) {
|
||||
|
||||
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
|
||||
|
||||
// with either be '0' or '0.xxxx', so if 1 then f will be zero
|
||||
// will either be '0' or '0.xxxx', so if 1 then f will be zero
|
||||
// otherwise need to parse
|
||||
if len(s) != 1 {
|
||||
|
||||
@@ -270,14 +270,14 @@ func T(n float64, v uint64) (t int64) {
|
||||
|
||||
s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64)
|
||||
|
||||
// with either be '0' or '0.xxxx', so if 1 then t will be zero
|
||||
// will either be '0' or '0.xxxx', so if 1 then t will be zero
|
||||
// otherwise need to parse
|
||||
if len(s) != 1 {
|
||||
|
||||
s = s[2:]
|
||||
end := len(s) + 1
|
||||
end := 0
|
||||
|
||||
for i := end; i >= 0; i-- {
|
||||
for i := len(s) - 1; i >= 0; i-- {
|
||||
if s[i] != '0' {
|
||||
end = i + 1
|
||||
break
|
||||
@@ -286,8 +286,11 @@ func T(n float64, v uint64) (t int64) {
|
||||
|
||||
// ignoring error, because it can't fail as we generated
|
||||
// the string internally from a real number
|
||||
t, _ = strconv.ParseInt(s[:end], 10, 64)
|
||||
if end > 0 {
|
||||
t, _ = strconv.ParseInt(s[:end], 10, 64)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
_obj
|
||||
_test
|
||||
bin
|
||||
.gocache/
|
||||
.gomodcache/
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ linters:
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- exhaustruct
|
||||
- exhaustruct_v5
|
||||
- forbidigo
|
||||
- forcetypeassert
|
||||
- funlen
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ GOCMD=go
|
||||
linters-install:
|
||||
@golangci-lint --version >/dev/null 2>&1 || { \
|
||||
echo "installing linting tools..."; \
|
||||
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.0.2; \
|
||||
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.13.1; \
|
||||
}
|
||||
|
||||
lint: linters-install
|
||||
|
||||
+2
-1
@@ -85,7 +85,7 @@ validate := validator.New(validator.WithRequiredStructEnabled())
|
||||
| eqcsfield | Field Equals Another Field (relative)|
|
||||
| eqfield | Field Equals Another Field |
|
||||
| fieldcontains | Check the indicated characters are present in the Field |
|
||||
| fieldexcludes | Check the indicated characters are not present in the field |
|
||||
| fieldexcludes | Current field does not contain another field's value |
|
||||
| gtcsfield | Field Greater Than Another Relative Field |
|
||||
| gtecsfield | Field Greater Than or Equal To Another Relative Field |
|
||||
| gtefield | Field Greater Than or Equal To Another Field |
|
||||
@@ -132,6 +132,7 @@ validate := validator.New(validator.WithRequiredStructEnabled())
|
||||
| origin | Web origin (URL with HTTP(S) scheme and host, but no path/query/fragment) |
|
||||
| url_encoded | URL Encoded |
|
||||
| urn_rfc2141 | Urn RFC 2141 String |
|
||||
| urn_rfc8141 | Urn RFC 8141 String |
|
||||
|
||||
### Strings:
|
||||
|
||||
|
||||
+33
-4
@@ -143,6 +143,7 @@ var (
|
||||
"https_url": isHttpsURL,
|
||||
"uri": isURI,
|
||||
"origin": isOrigin,
|
||||
"urn_rfc8141": isUrnRFC8141, // RFC 8141
|
||||
"urn_rfc2141": isUrnRFC2141, // RFC 2141
|
||||
"file": isFile,
|
||||
"filepath": isFilePath,
|
||||
@@ -1695,6 +1696,26 @@ func isHttpsURL(fl FieldLevel) bool {
|
||||
panic(fmt.Sprintf("Bad field type %s", field.Type()))
|
||||
}
|
||||
|
||||
// isUrnRFC8141 is the validation function for validating if the current field's value is a valid URN as per RFC 8141.
|
||||
func isUrnRFC8141(fl FieldLevel) bool {
|
||||
field := fl.Field()
|
||||
|
||||
switch field.Kind() {
|
||||
case reflect.String:
|
||||
|
||||
str := field.String()
|
||||
if str == "" {
|
||||
return false
|
||||
}
|
||||
|
||||
_, match := urn.Parse([]byte(str), urn.WithParsingMode(urn.RFC8141Only))
|
||||
|
||||
return match
|
||||
}
|
||||
|
||||
panic(fmt.Sprintf("Bad field type %T", field.Interface()))
|
||||
}
|
||||
|
||||
// isUrnRFC2141 is the validation function for validating if the current field's value is a valid URN as per RFC 2141.
|
||||
func isUrnRFC2141(fl FieldLevel) bool {
|
||||
field := fl.Field()
|
||||
@@ -2896,7 +2917,7 @@ func isHostnameRFC1123(fl FieldLevel) bool {
|
||||
func isFQDN(fl FieldLevel) bool {
|
||||
val := fl.Field().String()
|
||||
|
||||
if val == "" {
|
||||
if val == "" || len(strings.TrimSuffix(val, ".")) > 253 {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -3024,9 +3045,18 @@ func isHostnamePort(fl FieldLevel) bool {
|
||||
|
||||
// IsPort validates if the current field's value represents a valid port
|
||||
func isPort(fl FieldLevel) bool {
|
||||
val := fl.Field().Uint()
|
||||
field := fl.Field()
|
||||
|
||||
return val >= 1 && val <= 65535
|
||||
switch field.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
val := field.Int()
|
||||
return val >= 1 && val <= 65535
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
val := field.Uint()
|
||||
return val >= 1 && val <= 65535
|
||||
default:
|
||||
panic(fmt.Sprintf("Bad field type %s", field.Type()))
|
||||
}
|
||||
}
|
||||
|
||||
// isLowercase is the validation function for validating if the current field's value is a lowercase string.
|
||||
@@ -3510,4 +3540,3 @@ var (
|
||||
errMethodReturnNoValues = errors.New(`method return o values (void)`)
|
||||
errMethodReturnInvalidType = errors.New(`method should return invalid type`)
|
||||
)
|
||||
|
||||
+14
-6
@@ -803,13 +803,14 @@ other types.
|
||||
|
||||
Usage: containsfield=InnerStructField.Field
|
||||
|
||||
# Field Excludes Another Field
|
||||
# Field Excludes Another Field's Value
|
||||
|
||||
This does the same as excludes except for struct fields. It should only be used
|
||||
with string types. See the behavior of reflect.Value.String() for behavior on
|
||||
other types.
|
||||
This validates that the current field's string value does not contain the
|
||||
string value of the field named by the parameter. It should only be used with
|
||||
string types. If the referenced field cannot be resolved, validation succeeds.
|
||||
See the behavior of reflect.Value.String() for behavior on other types.
|
||||
|
||||
Usage: excludesfield=InnerStructField.Field
|
||||
Usage: fieldexcludes=InnerStructField.Field
|
||||
|
||||
# Unique
|
||||
|
||||
@@ -1026,6 +1027,13 @@ This will accept any uri the golang request uri accepts
|
||||
|
||||
Usage: uri
|
||||
|
||||
# Urn RFC 8141 String
|
||||
|
||||
This validataes that a string value contains a valid URN
|
||||
according to the RFC 8141 spec.
|
||||
|
||||
Usage: urn_rfc8141
|
||||
|
||||
# Urn RFC 2141 String
|
||||
|
||||
This validates that a string value contains a valid URN
|
||||
@@ -1437,7 +1445,7 @@ can be used to validate fields typically passed to sockets and connections.
|
||||
|
||||
# Port
|
||||
|
||||
This validates that the value falls within the valid port number range of 1 to 65,535.
|
||||
This validates that an integer value (signed or unsigned) falls within the valid port range of 1-65,535.
|
||||
|
||||
Usage: port
|
||||
|
||||
|
||||
+59
-3
@@ -310,7 +310,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er
|
||||
return
|
||||
}
|
||||
|
||||
if err = ut.Add("max-items", "{0} must contain at maximum {1}", false); err != nil {
|
||||
if err = ut.Add("max-items", "{0} must contain at most {1}", false); err != nil {
|
||||
return
|
||||
}
|
||||
if err = ut.AddCardinal("max-items-item", "{0} item", locales.PluralRuleOne, false); err != nil {
|
||||
@@ -548,7 +548,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er
|
||||
{
|
||||
tag: "lte",
|
||||
customRegisFunc: func(ut ut.Translator) (err error) {
|
||||
if err = ut.Add("lte-string", "{0} must be at maximum {1} in length", false); err != nil {
|
||||
if err = ut.Add("lte-string", "{0} must be at most {1} in length", false); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -564,7 +564,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er
|
||||
return
|
||||
}
|
||||
|
||||
if err = ut.Add("lte-items", "{0} must contain at maximum {1}", false); err != nil {
|
||||
if err = ut.Add("lte-items", "{0} must contain at most {1}", false); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1187,6 +1187,62 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er
|
||||
return t
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: "startswith",
|
||||
translation: "{0} must start with '{1}'",
|
||||
override: false,
|
||||
customTransFunc: func(ut ut.Translator, fe validator.FieldError) string {
|
||||
t, err := ut.T(fe.Tag(), fe.Field(), fe.Param())
|
||||
if err != nil {
|
||||
log.Printf("warning: error translating FieldError: %#v", fe)
|
||||
return fe.(error).Error()
|
||||
}
|
||||
|
||||
return t
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: "endswith",
|
||||
translation: "{0} must end with '{1}'",
|
||||
override: false,
|
||||
customTransFunc: func(ut ut.Translator, fe validator.FieldError) string {
|
||||
t, err := ut.T(fe.Tag(), fe.Field(), fe.Param())
|
||||
if err != nil {
|
||||
log.Printf("warning: error translating FieldError: %#v", fe)
|
||||
return fe.(error).Error()
|
||||
}
|
||||
|
||||
return t
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: "startsnotwith",
|
||||
translation: "{0} must not start with '{1}'",
|
||||
override: false,
|
||||
customTransFunc: func(ut ut.Translator, fe validator.FieldError) string {
|
||||
t, err := ut.T(fe.Tag(), fe.Field(), fe.Param())
|
||||
if err != nil {
|
||||
log.Printf("warning: error translating FieldError: %#v", fe)
|
||||
return fe.(error).Error()
|
||||
}
|
||||
|
||||
return t
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: "endsnotwith",
|
||||
translation: "{0} must not end with '{1}'",
|
||||
override: false,
|
||||
customTransFunc: func(ut ut.Translator, fe validator.FieldError) string {
|
||||
t, err := ut.T(fe.Tag(), fe.Field(), fe.Param())
|
||||
if err != nil {
|
||||
log.Printf("warning: error translating FieldError: %#v", fe)
|
||||
return fe.(error).Error()
|
||||
}
|
||||
|
||||
return t
|
||||
},
|
||||
},
|
||||
{
|
||||
tag: "excludes",
|
||||
translation: "{0} cannot contain the text '{1}'",
|
||||
|
||||
+5
-1
@@ -24,6 +24,7 @@ var (
|
||||
err8141QComponentStart = "expecting only one q-component (starting with the ?= sequence) [col %d]"
|
||||
err8141MalformedRComp = "expecting a non-empty r-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]"
|
||||
err8141MalformedQComp = "expecting a non-empty q-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]"
|
||||
errInvalidSyntax = "invalid URN syntax [col %d]"
|
||||
)
|
||||
var _toStateActions []byte = []byte{
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
@@ -4358,7 +4359,7 @@ func (m *machine) Parse(input []byte) (*URN, error) {
|
||||
|
||||
if m.parsingMode != RFC8141Only {
|
||||
// Throw an error when:
|
||||
// - we are entering here matching the the prefix in the namespace identifier part
|
||||
// - we are entering here matching the prefix in the namespace identifier part
|
||||
// - looking ahead (3 chars) we find a colon
|
||||
if pos := m.p + 3; pos < m.pe && m.data[pos] == 58 && output.prefix != "" {
|
||||
m.err = fmt.Errorf(errNoUrnWithinID, pos)
|
||||
@@ -5026,6 +5027,9 @@ func (m *machine) Parse(input []byte) (*URN, error) {
|
||||
}
|
||||
|
||||
if m.cs < firstFinal || m.cs == enFail {
|
||||
if m.err == nil {
|
||||
m.err = fmt.Errorf(errInvalidSyntax, m.p)
|
||||
}
|
||||
return nil, m.err
|
||||
}
|
||||
|
||||
|
||||
+6
-2
@@ -24,6 +24,7 @@ var (
|
||||
err8141QComponentStart = "expecting only one q-component (starting with the ?= sequence) [col %d]"
|
||||
err8141MalformedRComp = "expecting a non-empty r-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]"
|
||||
err8141MalformedQComp = "expecting a non-empty q-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]"
|
||||
errInvalidSyntax = "invalid URN syntax [col %d]"
|
||||
)
|
||||
|
||||
%%{
|
||||
@@ -48,7 +49,7 @@ action set_pre {
|
||||
action throw_pre_urn_err {
|
||||
if m.parsingMode != RFC8141Only {
|
||||
// Throw an error when:
|
||||
// - we are entering here matching the the prefix in the namespace identifier part
|
||||
// - we are entering here matching the prefix in the namespace identifier part
|
||||
// - looking ahead (3 chars) we find a colon
|
||||
if pos := m.p + 3; pos < m.pe && m.data[pos] == 58 && output.prefix != "" {
|
||||
m.err = fmt.Errorf(errNoUrnWithinID, pos)
|
||||
@@ -366,6 +367,9 @@ func (m *machine) Parse(input []byte) (*URN, error) {
|
||||
%% write exec;
|
||||
|
||||
if m.cs < first_final || m.cs == en_fail {
|
||||
if m.err == nil {
|
||||
m.err = fmt.Errorf(errInvalidSyntax, m.p)
|
||||
}
|
||||
return nil, m.err
|
||||
}
|
||||
|
||||
@@ -383,4 +387,4 @@ func (m *machine) WithParsingMode(x ParsingMode) {
|
||||
m.startParsingAt = en_scim_only
|
||||
}
|
||||
m.parsingModeSet = true
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
const errInvalidURN = "invalid URN: %s"
|
||||
|
||||
// URN represents an Uniform Resource Name.
|
||||
// URN represents a Uniform Resource Name.
|
||||
//
|
||||
// The general form represented is:
|
||||
//
|
||||
@@ -32,7 +32,7 @@ type URN struct {
|
||||
|
||||
// Normalize turns the receiving URN into its norm version.
|
||||
//
|
||||
// Which means: lowercase prefix, lowercase namespace identifier, and immutate namespace specific string chars (except <hex> tokens which are lowercased).
|
||||
// Which means: lowercase prefix, lowercase namespace identifier, and leave namespace-specific string chars unchanged (except <hex> tokens which are lowercased).
|
||||
func (u *URN) Normalize() *URN {
|
||||
return &URN{
|
||||
prefix: "urn",
|
||||
@@ -82,10 +82,10 @@ func (u *URN) String() string {
|
||||
return res
|
||||
}
|
||||
|
||||
// Parse is responsible to create an URN instance from a byte array matching the correct URN syntax (RFC 2141).
|
||||
// Parse is responsible for creating a URN instance from a byte array matching the correct URN syntax (RFC 2141).
|
||||
func Parse(u []byte, options ...Option) (*URN, bool) {
|
||||
urn, err := NewMachine(options...).Parse(u)
|
||||
if err != nil {
|
||||
if err != nil || urn == nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
|
||||
Generated
Vendored
+25
-8
@@ -47,6 +47,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
@@ -71,6 +72,7 @@ type config struct {
|
||||
MountID string `mapstructure:"mount_id"`
|
||||
UploadExpiration int64 `mapstructure:"upload_expiration" docs:"0;Duration for how long uploads will be valid."`
|
||||
Events eventconfig `mapstructure:"events" docs:"0;Event stream configuration"`
|
||||
UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."`
|
||||
}
|
||||
|
||||
type eventconfig struct {
|
||||
@@ -106,6 +108,7 @@ func (c *config) init() {
|
||||
type Service struct {
|
||||
conf *config
|
||||
Storage storage.FS
|
||||
Coordinator upload.Coordinator
|
||||
dataServerURL *url.URL
|
||||
availableXS []*provider.ResourceChecksumPriority
|
||||
}
|
||||
@@ -175,7 +178,14 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc.
|
||||
|
||||
c.init()
|
||||
|
||||
fs, err := getFS(c, log)
|
||||
// One stream for both the driver and the coordinator: a second one would open a
|
||||
// second nats connection for the same events.
|
||||
evstream, err := estreamFromConfig(c.Events)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fs, err := getFS(c, evstream, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -202,9 +212,16 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// storageprovider only initiates uploads; the data path assembles chunks, so no chunking here.
|
||||
coord, err := upload.NewCoordinatorFromConfig(c.UploadDirectory, c.Drivers[c.Driver], fs, evstream, log, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storageprovider: %w", err)
|
||||
}
|
||||
|
||||
service := &Service{
|
||||
conf: c,
|
||||
Storage: fs,
|
||||
Coordinator: coord,
|
||||
dataServerURL: u,
|
||||
availableXS: xsTypes,
|
||||
}
|
||||
@@ -414,7 +431,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate
|
||||
metadata["expires"] = strconv.Itoa(int(expirationTimestamp.Seconds))
|
||||
}
|
||||
|
||||
uploadIDs, err := s.Storage.InitiateUpload(ctx, req.Ref, uploadLength, metadata)
|
||||
uploadIDs, err := s.Coordinator.InitiateUpload(ctx, req.Ref, uploadLength, metadata)
|
||||
if err != nil {
|
||||
var st *rpc.Status
|
||||
switch err.(type) {
|
||||
@@ -582,6 +599,10 @@ func (s *Service) ListStorageSpaces(ctx context.Context, req *provider.ListStora
|
||||
}, nil
|
||||
}
|
||||
|
||||
caps := storage.FullCapabilities()
|
||||
if cp, ok := s.Storage.(storage.CapabilityProvider); ok {
|
||||
caps = cp.Capabilities(ctx)
|
||||
}
|
||||
for _, sp := range spaces {
|
||||
if sp.GetId() == nil || sp.GetId().GetOpaqueId() == "" {
|
||||
log.Error().Str("service", "storageprovider").Str("driver", s.conf.Driver).Interface("space", sp).Msg("space is missing space id and root id")
|
||||
@@ -589,6 +610,7 @@ func (s *Service) ListStorageSpaces(ctx context.Context, req *provider.ListStora
|
||||
}
|
||||
|
||||
s.addMissingStorageProviderID(sp.GetRoot(), sp.GetId())
|
||||
sp.Opaque = utils.AppendJSONToOpaque(sp.Opaque, storage.CapabilitiesOpaqueKey, caps)
|
||||
}
|
||||
|
||||
return &provider.ListStorageSpacesResponse{
|
||||
@@ -1330,12 +1352,7 @@ func (s *Service) addMissingStorageProviderID(resourceID *provider.ResourceId, s
|
||||
}
|
||||
}
|
||||
|
||||
func getFS(c *config, log *zerolog.Logger) (storage.FS, error) {
|
||||
evstream, err := estreamFromConfig(c.Events)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func getFS(c *config, evstream events.Stream, log *zerolog.Logger) (storage.FS, error) {
|
||||
if f, ok := registry.NewFuncs[c.Driver]; ok {
|
||||
driverConf := c.Drivers[c.Driver]
|
||||
driverConf["mount_id"] = c.MountID // pass the mount id to the driver
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -52,5 +52,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re
|
||||
// fall back to hostname if not configured
|
||||
realm = r.Host
|
||||
}
|
||||
w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm))
|
||||
w.Header().Add("WWW-Authenticate", auth.Challenge("Basic", realm))
|
||||
}
|
||||
Generated
Vendored
+1
-1
@@ -63,5 +63,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re
|
||||
// fall back to hostname if not configured
|
||||
realm = r.Host
|
||||
}
|
||||
w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, realm))
|
||||
w.Header().Add("WWW-Authenticate", auth.Challenge("Bearer", realm))
|
||||
}
|
||||
Generated
Vendored
+19
-3
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -51,6 +52,7 @@ type config struct {
|
||||
NatsEnableTLS bool `mapstructure:"nats_enable_tls"`
|
||||
NatsUsername string `mapstructure:"nats_username"`
|
||||
NatsPassword string `mapstructure:"nats_password"`
|
||||
UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."`
|
||||
}
|
||||
|
||||
func (c *config) init() {
|
||||
@@ -104,7 +106,21 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dataTXs, err := getDataTXs(conf, fs, evstream, log)
|
||||
// the data path assembles chunks, so enable chunking
|
||||
coord, err := upload.NewCoordinatorFromConfig(conf.UploadDirectory, conf.Drivers[conf.Driver], fs, evstream, log, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dataprovider: %w", err)
|
||||
}
|
||||
|
||||
// only the data path consumes postprocessing results: one consumer group gets
|
||||
// one copy of each event, so a second subscriber would take half of them
|
||||
if ac := upload.AsyncConfFromDriverConf(conf.Drivers[conf.Driver]); ac.Enabled {
|
||||
if err := coord.StartPostprocessing(evstream, ac); err != nil {
|
||||
return nil, fmt.Errorf("dataprovider: could not start postprocessing: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
dataTXs, err := getDataTXs(conf, coord, fs, evstream, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -126,7 +142,7 @@ func getFS(c *config, stream events.Stream, log *zerolog.Logger) (storage.FS, er
|
||||
return nil, fmt.Errorf("driver not found: %s", c.Driver)
|
||||
}
|
||||
|
||||
func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) {
|
||||
func getDataTXs(c *config, coord upload.Coordinator, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) {
|
||||
if c.DataTXs == nil {
|
||||
c.DataTXs = make(map[string]map[string]interface{})
|
||||
}
|
||||
@@ -146,7 +162,7 @@ func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerol
|
||||
for t := range c.DataTXs {
|
||||
if f, ok := datatxregistry.NewFuncs[t]; ok {
|
||||
if tx, err := f(c.DataTXs[t], publisher, log); err == nil {
|
||||
if handler, err := tx.Handler(fs); err == nil {
|
||||
if handler, err := tx.Handler(coord, fs); err == nil {
|
||||
txs[t] = handler
|
||||
}
|
||||
}
|
||||
|
||||
Generated
Vendored
+2
-2
@@ -20,7 +20,6 @@ package ocdav
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -35,6 +34,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/errors"
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/auth"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/router"
|
||||
@@ -438,7 +438,7 @@ func handleBasicAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta
|
||||
}
|
||||
|
||||
func authChallenge(scheme, realm string) string {
|
||||
return fmt.Sprintf("%s realm=\"%s\", charset=\"UTF-8\"", scheme, realm)
|
||||
return auth.Challenge(scheme, realm, `charset="UTF-8"`)
|
||||
}
|
||||
|
||||
func handleSignatureAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta1.GatewayAPIClient], token, sig, expiration string) (*gatewayv1beta1.AuthenticateResponse, error) {
|
||||
|
||||
Generated
Vendored
+2
@@ -1538,6 +1538,8 @@ func mdToPropResponse(ctx context.Context, pf *XML, md *provider.ResourceInfo, p
|
||||
types.WriteString("<oc:share-type>" + strconv.Itoa(int(conversions.ShareTypeUser)) + "</oc:share-type>")
|
||||
case "2": // provider.GranteeType_GRANTEE_TYPE_GROUP
|
||||
types.WriteString("<oc:share-type>" + strconv.Itoa(int(conversions.ShareTypeGroup)) + "</oc:share-type>")
|
||||
case "4": // conversions.ShareTypeGuest, guest shares are stored as user grants
|
||||
types.WriteString("<oc:share-type>" + strconv.Itoa(int(conversions.ShareTypeGuest)) + "</oc:share-type>")
|
||||
default:
|
||||
sublog.Debug().Interface("shareType", shareType).Msg("unknown share type, ignoring")
|
||||
}
|
||||
|
||||
Generated
Vendored
+2
-1
@@ -337,7 +337,8 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
sReq.Ref.Path = uReq.Ref.GetPath()
|
||||
sReq.Ref.ResourceId = nil
|
||||
} else {
|
||||
if resid, err := storagespace.ParseID(httpRes.Header.Get(net.HeaderOCFileID)); err == nil {
|
||||
// new files have no node id yet; keep the path-based ref instead
|
||||
if resid, err := storagespace.ParseID(httpRes.Header.Get(net.HeaderOCFileID)); err == nil && resid.GetOpaqueId() != "" {
|
||||
sReq.Ref = &provider.Reference{
|
||||
ResourceId: &resid,
|
||||
}
|
||||
|
||||
Generated
Vendored
+10
-2
@@ -31,6 +31,7 @@ type Handler struct {
|
||||
c ocs.CapabilitiesData
|
||||
defaultUploadProtocol string
|
||||
userAgentChunkingMap map[string]string
|
||||
gatewayAddr string
|
||||
}
|
||||
|
||||
// Init initializes this and any contained handlers
|
||||
@@ -38,6 +39,7 @@ func (h *Handler) Init(c *config.Config) {
|
||||
h.c = c.Capabilities
|
||||
h.defaultUploadProtocol = c.DefaultUploadProtocol
|
||||
h.userAgentChunkingMap = c.UserAgentChunkingMap
|
||||
h.gatewayAddr = c.GatewaySvc
|
||||
|
||||
// capabilities
|
||||
if h.c.Capabilities == nil {
|
||||
@@ -120,8 +122,9 @@ func (h *Handler) Init(c *config.Config) {
|
||||
if h.c.Capabilities.Dav == nil {
|
||||
h.c.Capabilities.Dav = &ocs.CapabilitiesDav{}
|
||||
}
|
||||
if h.c.Capabilities.Dav.Trashbin == "" {
|
||||
h.c.Capabilities.Dav.Trashbin = "1.0"
|
||||
// Trashbin is deprecated for clients, but we keep announcing it until they read the providers section.
|
||||
if h.c.Capabilities.Dav.Trashbin == "" { //nolint:staticcheck // SA1019: still announced for older clients
|
||||
h.c.Capabilities.Dav.Trashbin = "1.0" //nolint:staticcheck // SA1019: still announced for older clients
|
||||
}
|
||||
if h.c.Capabilities.Dav.Reports == nil {
|
||||
h.c.Capabilities.Dav.Reports = []string{}
|
||||
@@ -241,5 +244,10 @@ func (h *Handler) Init(c *config.Config) {
|
||||
// Handler renders the capabilities
|
||||
func (h *Handler) GetCapabilities(w http.ResponseWriter, r *http.Request) {
|
||||
c := h.getCapabilitiesForUserAgent(r.UserAgent())
|
||||
if providers := h.resolveProviders(r.Context()); len(providers) > 0 && c.Capabilities != nil {
|
||||
caps := *c.Capabilities
|
||||
caps.Providers = providers
|
||||
c.Capabilities = &caps
|
||||
}
|
||||
response.WriteOCSSuccess(w, r, c)
|
||||
}
|
||||
Generated
Vendored
+53
@@ -0,0 +1,53 @@
|
||||
package capabilities
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// resolveProviders builds the per-provider capability map from the spaces visible
|
||||
// to the current user, keyed by provider ID. Returns nil on no user context or an
|
||||
// unreachable gateway, so the response omits the section rather than failing.
|
||||
func (h *Handler) resolveProviders(ctx context.Context) map[string]*ocs.ProviderCapabilities {
|
||||
log := appctx.GetLogger(ctx)
|
||||
|
||||
gc, err := pool.GetGatewayServiceClient(h.gatewayAddr)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("capabilities: error getting gateway client")
|
||||
return nil
|
||||
}
|
||||
|
||||
res, err := gc.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{})
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("capabilities: error listing storage spaces")
|
||||
return nil
|
||||
}
|
||||
if res.GetStatus().GetCode() != rpc.Code_CODE_OK {
|
||||
return nil
|
||||
}
|
||||
|
||||
providers := map[string]*ocs.ProviderCapabilities{}
|
||||
for _, sp := range res.GetStorageSpaces() {
|
||||
id := sp.GetRoot().GetStorageId()
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
if _, done := providers[id]; done {
|
||||
continue
|
||||
}
|
||||
var caps storage.Capabilities
|
||||
if err := utils.ReadJSONFromOpaque(sp.GetOpaque(), storage.CapabilitiesOpaqueKey, &caps); err != nil {
|
||||
continue
|
||||
}
|
||||
providers[id] = ocs.NewProviderCapabilities(caps)
|
||||
}
|
||||
return providers
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package auth
|
||||
|
||||
import "strings"
|
||||
|
||||
var quotedStringEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`)
|
||||
|
||||
// Challenge builds a WWW-Authenticate challenge for the given scheme with the
|
||||
// realm escaped as a quoted-string. Additional params are appended without escaping.
|
||||
func Challenge(scheme, realm string, params ...string) string {
|
||||
return strings.Join(append([]string{scheme + ` realm="` + quotedStringEscaper.Replace(realm) + `"`}, params...), ", ")
|
||||
}
|
||||
+3
@@ -61,6 +61,9 @@ const (
|
||||
// ShareTypeSpaceMembershipGroup represents an action regarding group type space members
|
||||
ShareTypeSpaceMembershipGroup ShareType = 8
|
||||
|
||||
// ShareTypeGuest represents a guest share
|
||||
ShareTypeGuest ShareType = 4
|
||||
|
||||
// ShareWithUserTypeUser represents a normal user
|
||||
ShareWithUserTypeUser ShareWithUserType = 0
|
||||
|
||||
|
||||
+2
@@ -103,6 +103,7 @@ type PostprocessingStepFinished struct {
|
||||
UploadID string
|
||||
ExecutingUser *user.User
|
||||
Filename string
|
||||
ResourceID *provider.ResourceId
|
||||
|
||||
FinishedStep Postprocessingstep // name of the step
|
||||
Result interface{} // result information see VirusscanResult for example
|
||||
@@ -145,6 +146,7 @@ type VirusscanResult struct {
|
||||
type PostprocessingFinished struct {
|
||||
UploadID string
|
||||
Filename string
|
||||
ResourceID *provider.ResourceId
|
||||
SpaceOwner *user.UserId
|
||||
ExecutingUser *user.User
|
||||
Result map[Postprocessingstep]interface{} // it is a map[step]Event
|
||||
|
||||
+7
@@ -33,6 +33,7 @@ var mimeTypes = map[string]string{
|
||||
"3g2": "video/3gpp2",
|
||||
"3gp": "video/3gpp",
|
||||
"3gpp": "video/3gpp",
|
||||
"3mf": "model/3mf",
|
||||
"7z": "application/x-7z-compressed",
|
||||
"aab": "application/x-authorware-bin",
|
||||
"aac": "audio/x-aac",
|
||||
@@ -82,6 +83,7 @@ var mimeTypes = map[string]string{
|
||||
"azs": "application/vnd.airzip.filesecure.azs",
|
||||
"azv": "image/vnd.airzip.accelerator.azv",
|
||||
"azw": "application/vnd.amazon.ebook",
|
||||
"azw3": "application/vnd.amazon.mobi8-ebook",
|
||||
"bat": "application/x-msdownload",
|
||||
"bcpio": "application/x-bcpio",
|
||||
"bdf": "application/x-font-bdf",
|
||||
@@ -91,6 +93,7 @@ var mimeTypes = map[string]string{
|
||||
"bh2": "application/vnd.fujitsu.oasysprs",
|
||||
"bin": "application/octet-stream",
|
||||
"blb": "application/x-blorb",
|
||||
"blend": "application/x-blender",
|
||||
"blorb": "application/x-blorb",
|
||||
"bmi": "application/vnd.bmi",
|
||||
"bmp": "image/x-ms-bmp",
|
||||
@@ -276,6 +279,7 @@ var mimeTypes = map[string]string{
|
||||
"f4v": "video/x-f4v",
|
||||
"f77": "text/x-fortran",
|
||||
"f90": "text/x-fortran",
|
||||
"fb2": "application/x-fictionbook+xml",
|
||||
"fbs": "image/vnd.fastbidsheet",
|
||||
"fcdt": "application/vnd.adobe.formscentral.fcdt",
|
||||
"fcs": "application/vnd.isac.fcs",
|
||||
@@ -407,6 +411,7 @@ var mimeTypes = map[string]string{
|
||||
"iota": "application/vnd.astraea-software.iota",
|
||||
"ipfix": "application/ipfix",
|
||||
"ipk": "application/vnd.shana.informed.package",
|
||||
"ipynb": "application/x-ipynb+json",
|
||||
"irm": "application/vnd.ibm.rights-management",
|
||||
"irp": "application/vnd.irepository.package+xml",
|
||||
"iso": "application/x-iso9660-image",
|
||||
@@ -456,6 +461,7 @@ var mimeTypes = map[string]string{
|
||||
"kpr": "application/vnd.kde.kpresenter",
|
||||
"kpt": "application/vnd.kde.kpresenter",
|
||||
"kpxx": "application/vnd.ds-keypoint",
|
||||
"kra": "application/x-krita",
|
||||
"ksp": "application/vnd.kde.kspread",
|
||||
"ktr": "application/vnd.kahootz",
|
||||
"ktx": "image/ktx",
|
||||
@@ -495,6 +501,7 @@ var mimeTypes = map[string]string{
|
||||
"m3u": "audio/x-mpegurl",
|
||||
"m3u8": "application/vnd.apple.mpegurl",
|
||||
"m4a": "audio/x-m4a",
|
||||
"m4b": "audio/x-m4b",
|
||||
"m4p": "application/mp4",
|
||||
"m4u": "video/vnd.mpegurl",
|
||||
"m4v": "video/x-m4v",
|
||||
|
||||
+28
-3
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/studio-b12/gowebdav"
|
||||
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/mime"
|
||||
@@ -218,7 +219,7 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error {
|
||||
}
|
||||
|
||||
func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error {
|
||||
client, _, rel, err := d.webdavClient(ctx, nil, ref)
|
||||
client, _, rel, err := d.serviceWebdavClient(ctx, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -226,7 +227,7 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error {
|
||||
}
|
||||
|
||||
func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
|
||||
client, _, rel, err := d.webdavClient(ctx, nil, ref)
|
||||
client, _, rel, err := d.serviceWebdavClient(ctx, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -327,7 +328,19 @@ func extractChecksum(props gowebdav.Props) *provider.ResourceChecksum {
|
||||
}
|
||||
|
||||
func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, _ []string) (*provider.ResourceInfo, error) {
|
||||
client, share, rel, err := d.webdavClient(ctx, nil, ref)
|
||||
var (
|
||||
client *gowebdav.Client
|
||||
share *ocmpb.ReceivedShare
|
||||
rel string
|
||||
err error
|
||||
)
|
||||
if token, ok := ctxpkg.ContextGetToken(ctx); ok && token != "" {
|
||||
client, share, rel, err = d.webdavClient(ctx, nil, ref)
|
||||
} else {
|
||||
// the upload coordinator stats an upload it finishes with the context of the
|
||||
// upload's executant, whose request, and with it the token, is gone by then
|
||||
client, share, rel, err = d.serviceWebdavClient(ctx, ref)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -494,6 +507,10 @@ func (d *driver) GetLock(ctx context.Context, ref *provider.Reference) (*provide
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// gowebdav formats a missing lockdiscovery property as "<nil>"
|
||||
if token == "" || token == "<nil>" {
|
||||
return nil, errtypes.NotFound("no lock found")
|
||||
}
|
||||
|
||||
return &provider.Lock{LockId: token, Type: provider.LockType_LOCK_TYPE_EXCL}, nil
|
||||
}
|
||||
@@ -617,3 +634,11 @@ func (d *driver) AddLabel(ctx context.Context, ref *provider.Reference, userID *
|
||||
func (d *driver) RemoveLabel(ctx context.Context, ref *provider.Reference, userID *userpb.UserId, label string) error {
|
||||
return errtypes.NotSupported("RemoveLabel not implemented")
|
||||
}
|
||||
|
||||
func (d *driver) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) {
|
||||
return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil
|
||||
}
|
||||
|
||||
func (d *driver) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error {
|
||||
return nil
|
||||
}
|
||||
+24
-366
@@ -20,399 +20,57 @@ package ocm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"hash/adler32"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
"github.com/studio-b12/gowebdav"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
var defaultFilePerm = os.FileMode(0664)
|
||||
|
||||
func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
|
||||
return []storage.UploadSession{}, nil
|
||||
}
|
||||
func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
|
||||
shareID, rel := shareInfoFromReference(ref)
|
||||
p := getPathFromShareIDAndRelPath(shareID, rel)
|
||||
|
||||
info := tusd.FileInfo{
|
||||
MetaData: tusd.MetaData{
|
||||
"filename": filepath.Base(p),
|
||||
"dir": filepath.Dir(p),
|
||||
},
|
||||
Size: uploadLength,
|
||||
}
|
||||
|
||||
upload, err := d.NewUpload(ctx, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info, _ = upload.GetInfo(ctx)
|
||||
|
||||
return map[string]string{
|
||||
"simple": info.ID,
|
||||
"tus": info.ID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// MarkProcessing is a no-op: the file lives on the remote instance, so there is no
|
||||
// local node to flag while postprocessing runs.
|
||||
func (d *driver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error {
|
||||
return errtypes.NotSupported("op not supported")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *driver) CommitUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.UploadSource) error {
|
||||
return errtypes.NotSupported("op not supported")
|
||||
}
|
||||
|
||||
func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) {
|
||||
shareID, _ := shareInfoFromReference(req.Ref)
|
||||
u, err := d.GetUpload(ctx, shareID.OpaqueId)
|
||||
// CommitUpload streams the staged bytes to the remote instance.
|
||||
func (d *driver) CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source storage.UploadSource) error {
|
||||
client, _, rel, err := d.serviceWebdavClient(ctx, ref)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
return err
|
||||
}
|
||||
|
||||
info, err := u.GetInfo(ctx)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
|
||||
defer cleanup(&upload{Info: info})
|
||||
|
||||
client, _, rel, err := d.webdavClient(ctx, nil, &provider.Reference{
|
||||
Path: filepath.Join(info.MetaData["dir"], info.MetaData["filename"]),
|
||||
})
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
client.SetInterceptor(func(method string, rq *http.Request) {
|
||||
// Set the content length on the request struct directly instead of the header.
|
||||
// The content-length header gets reset by the golang http library before
|
||||
// sendind out the request, resulting in chunked encoding to be used which
|
||||
// sending out the request, resulting in chunked encoding to be used which
|
||||
// breaks the quota checks in ocdav.
|
||||
if method == "PUT" {
|
||||
rq.ContentLength = req.Length
|
||||
rq.ContentLength = source.Length
|
||||
}
|
||||
})
|
||||
|
||||
locktoken, _ := ctxpkg.ContextGetLockID(ctx)
|
||||
return &provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0, locktoken)
|
||||
// the coordinator owns source.Body and closes it after this returns
|
||||
lockToken, _ := ctxpkg.ContextGetLockID(ctx)
|
||||
return client.WriteStream(rel, source.Body, 0, lockToken)
|
||||
}
|
||||
|
||||
// UseIn tells the tus upload middleware which extensions it supports.
|
||||
func (d *driver) UseIn(composer *tusd.StoreComposer) {
|
||||
composer.UseCore(d)
|
||||
composer.UseTerminater(d)
|
||||
composer.UseConcater(d)
|
||||
composer.UseLengthDeferrer(d)
|
||||
}
|
||||
|
||||
// AsTerminatableUpload returns a TerminatableUpload
|
||||
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
|
||||
// the storage needs to implement AsTerminatableUpload
|
||||
func (d *driver) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
|
||||
return up.(*upload)
|
||||
}
|
||||
|
||||
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
|
||||
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
|
||||
// the storage needs to implement AsLengthDeclarableUpload
|
||||
func (d *driver) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
|
||||
return up.(*upload)
|
||||
}
|
||||
|
||||
// AsConcatableUpload returns a ConcatableUpload
|
||||
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
|
||||
// the storage needs to implement AsConcatableUpload
|
||||
func (d *driver) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
|
||||
return up.(*upload)
|
||||
}
|
||||
|
||||
// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol
|
||||
// - the storage needs to implement NewUpload and GetUpload
|
||||
// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload
|
||||
|
||||
// NewUpload returns a new tus Upload instance
|
||||
func (d *driver) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) {
|
||||
return NewUpload(ctx, d, d.c.StorageRoot, info)
|
||||
}
|
||||
|
||||
// GetUpload returns the Upload for the given upload id
|
||||
func (d *driver) GetUpload(ctx context.Context, id string) (tusd.Upload, error) {
|
||||
return GetUpload(ctx, d, d.c.StorageRoot, id)
|
||||
}
|
||||
func NewUpload(ctx context.Context, d *driver, storageRoot string, info tusd.FileInfo) (tusd.Upload, error) {
|
||||
if info.MetaData["filename"] == "" {
|
||||
return nil, errors.New("decomposedfs: missing filename in metadata")
|
||||
// serviceWebdavClient builds a webdav client without needing a request auth token:
|
||||
// it authenticates as the service account and looks up the share on behalf of the
|
||||
// upload's executant.
|
||||
func (d *driver) serviceWebdavClient(ctx context.Context, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) {
|
||||
serviceUserCtx, err := utils.GetServiceUserContextWithContext(context.Background(), d.gateway, d.c.ServiceAccountID, d.c.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
if info.MetaData["dir"] == "" {
|
||||
return nil, errors.New("decomposedfs: missing dir in metadata")
|
||||
}
|
||||
|
||||
uploadRoot := filepath.Join(storageRoot, "uploads")
|
||||
info.ID = uuid.New().String()
|
||||
|
||||
user, ok := ctxpkg.ContextGetUser(ctx)
|
||||
executant, ok := ctxpkg.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
return nil, errors.New("no user in context")
|
||||
return nil, nil, "", errtypes.BadRequest("ocm: no executant in context")
|
||||
}
|
||||
info.MetaData["user"] = user.GetId().GetOpaqueId()
|
||||
info.MetaData["idp"] = user.GetId().GetIdp()
|
||||
|
||||
info.Storage = map[string]string{
|
||||
"Type": "OCM",
|
||||
"Path": uploadRoot,
|
||||
}
|
||||
|
||||
u := &upload{
|
||||
Info: info,
|
||||
Ctx: ctx,
|
||||
d: d,
|
||||
}
|
||||
|
||||
err := os.MkdirAll(uploadRoot, 0755)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(u.BinPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
err = u.Persist()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func GetUpload(ctx context.Context, d *driver, storageRoot string, id string) (tusd.Upload, error) {
|
||||
info := tusd.FileInfo{}
|
||||
data, err := os.ReadFile(filepath.Join(storageRoot, "uploads", id+".info"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = json.Unmarshal(data, &info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
upload := &upload{
|
||||
Info: info,
|
||||
Ctx: ctx,
|
||||
d: d,
|
||||
}
|
||||
return upload, nil
|
||||
}
|
||||
|
||||
type upload struct {
|
||||
Info tusd.FileInfo
|
||||
Ctx context.Context
|
||||
|
||||
d *driver
|
||||
}
|
||||
|
||||
func (u *upload) InfoPath() string {
|
||||
return filepath.Join(u.Info.Storage["Path"], u.Info.ID+".info")
|
||||
}
|
||||
|
||||
func (u *upload) BinPath() string {
|
||||
return filepath.Join(u.Info.Storage["Path"], u.Info.ID)
|
||||
}
|
||||
|
||||
func (u *upload) Persist() error {
|
||||
data, err := json.Marshal(u.Info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(u.InfoPath(), data, defaultFilePerm)
|
||||
}
|
||||
|
||||
func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
|
||||
file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum
|
||||
// TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ...
|
||||
// It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used
|
||||
// but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ...
|
||||
n, err := io.Copy(file, src)
|
||||
|
||||
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
|
||||
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
|
||||
// However, for the ocm driver it's not important whether the stream has ended
|
||||
// on purpose or accidentally.
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
return n, err
|
||||
}
|
||||
|
||||
u.Info.Offset += n
|
||||
return n, u.Persist()
|
||||
}
|
||||
|
||||
func (u *upload) GetInfo(ctx context.Context) (tusd.FileInfo, error) {
|
||||
return u.Info, nil
|
||||
}
|
||||
|
||||
func (u *upload) GetReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
return os.Open(u.BinPath())
|
||||
}
|
||||
|
||||
func (u *upload) FinishUpload(ctx context.Context) error {
|
||||
log := appctx.GetLogger(u.Ctx)
|
||||
|
||||
// calculate the checksum of the written bytes
|
||||
// they will all be written to the metadata later, so we cannot omit any of them
|
||||
// TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present
|
||||
// TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ...
|
||||
sha1h := sha1.New()
|
||||
md5h := md5.New()
|
||||
adler32h := adler32.New()
|
||||
{
|
||||
f, err := os.Open(u.BinPath())
|
||||
if err != nil {
|
||||
// we can continue if no oc checksum header is set
|
||||
log.Info().Err(err).Str("binPath", u.BinPath()).Msg("error opening binPath")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r1 := io.TeeReader(f, sha1h)
|
||||
r2 := io.TeeReader(r1, md5h)
|
||||
|
||||
_, err = io.Copy(adler32h, r2)
|
||||
if err != nil {
|
||||
log.Info().Err(err).Msg("error copying checksums")
|
||||
}
|
||||
}
|
||||
|
||||
defer cleanup(u)
|
||||
// compare if they match the sent checksum
|
||||
// TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads
|
||||
if u.Info.MetaData["checksum"] != "" {
|
||||
var err error
|
||||
parts := strings.SplitN(u.Info.MetaData["checksum"], " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
|
||||
}
|
||||
switch parts[0] {
|
||||
case "sha1":
|
||||
err = u.checkHash(parts[1], sha1h)
|
||||
case "md5":
|
||||
err = u.checkHash(parts[1], md5h)
|
||||
case "adler32":
|
||||
err = u.checkHash(parts[1], adler32h)
|
||||
default:
|
||||
err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// send to the remote storage via webdav
|
||||
// shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"])
|
||||
// p := getPathFromShareIDAndRelPath(shareID, rel)
|
||||
|
||||
serviceUserCtx, err := utils.GetServiceUserContextWithContext(context.Background(), u.d.gateway, u.d.c.ServiceAccountID, u.d.c.ServiceAccountSecret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
client, _, rel, err := u.d.webdavClient(serviceUserCtx, &userpb.UserId{
|
||||
OpaqueId: u.Info.MetaData["user"],
|
||||
Idp: u.Info.MetaData["idp"],
|
||||
}, &provider.Reference{
|
||||
Path: filepath.Join(u.Info.MetaData["dir"], u.Info.MetaData["filename"]),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client.SetInterceptor(func(method string, rq *http.Request) {
|
||||
// Set the content length on the request struct directly instead of the header.
|
||||
// The content-length header gets reset by the golang http library before
|
||||
// sendind out the request, resulting in chunked encoding to be used which
|
||||
// breaks the quota checks in ocdav.
|
||||
if method == "PUT" {
|
||||
rq.ContentLength = u.Info.Size
|
||||
}
|
||||
})
|
||||
|
||||
f, err := os.Open(u.BinPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return client.WriteStream(rel, f, 0, "")
|
||||
}
|
||||
|
||||
func (u *upload) Terminate(ctx context.Context) error {
|
||||
cleanup(u)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) error {
|
||||
file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
for _, partialUpload := range uploads {
|
||||
fileUpload := partialUpload.(*upload)
|
||||
|
||||
src, err := os.Open(fileUpload.BinPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
if _, err := io.Copy(file, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *upload) DeclareLength(ctx context.Context, length int64) error {
|
||||
u.Info.Size = length
|
||||
u.Info.SizeIsDeferred = false
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *upload) checkHash(expected string, h hash.Hash) error {
|
||||
if expected != hex.EncodeToString(h.Sum(nil)) {
|
||||
return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", u.Info.MetaData["checksum"], h.Sum(nil)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanup(u *upload) {
|
||||
if u == nil {
|
||||
return
|
||||
}
|
||||
_ = os.Remove(u.BinPath())
|
||||
_ = os.Remove(u.InfoPath())
|
||||
return d.webdavClient(serviceUserCtx, executant.GetId(), ref)
|
||||
}
|
||||
+70
-5
@@ -20,6 +20,10 @@ package ocs
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"maps"
|
||||
"slices"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
)
|
||||
|
||||
// ocsBool implements the xml/json Marshaler interface. The OCS API inconsistency require us to parse boolean values
|
||||
@@ -62,6 +66,63 @@ type Capabilities struct {
|
||||
Theme *CapabilitiesTheme `json:"theme,omitempty" xml:"theme,omitempty" mapstructure:"theme"`
|
||||
Notifications *CapabilitiesNotifications `json:"notifications,omitempty" xml:"notifications,omitempty"`
|
||||
Groupware *CapabilitiesGroupware `json:"groupware,omitempty" xml:"groupware,omitempty" mapstructure:"groupware"`
|
||||
// Providers is the per-provider capability section, keyed by provider ID. It
|
||||
// supersedes the deprecated global storage keys under Files/Dav; there is no
|
||||
// fallback between them.
|
||||
Providers ProviderCapabilitiesMap `json:"providers,omitempty" xml:"providers,omitempty" mapstructure:"providers"`
|
||||
}
|
||||
|
||||
// ProviderCapabilitiesMap is the per-provider section keyed by provider ID.
|
||||
type ProviderCapabilitiesMap map[string]*ProviderCapabilities
|
||||
|
||||
// MarshalXML renders the map as <providers><provider id="...">...</provider>...</providers>,
|
||||
// ordered by id. encoding/xml cannot marshal a Go map, so XML output is produced
|
||||
// explicitly; JSON uses the default map encoding.
|
||||
func (m ProviderCapabilitiesMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := e.EncodeToken(start); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range slices.Sorted(maps.Keys(m)) {
|
||||
el := xml.StartElement{
|
||||
Name: xml.Name{Local: "provider"},
|
||||
Attr: []xml.Attr{{Name: xml.Name{Local: "id"}, Value: id}},
|
||||
}
|
||||
if err := e.EncodeElement(m[id], el); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return e.EncodeToken(start.End())
|
||||
}
|
||||
|
||||
// ProviderCapabilities is the wire form of a provider's declared capabilities.
|
||||
type ProviderCapabilities struct {
|
||||
Upload ocsBool `json:"upload" xml:"upload"`
|
||||
CreateContainer ocsBool `json:"create_container" xml:"create_container"`
|
||||
Delete ocsBool `json:"delete" xml:"delete"`
|
||||
Move ocsBool `json:"move" xml:"move"`
|
||||
Versioning ocsBool `json:"versioning" xml:"versioning"`
|
||||
Trash ocsBool `json:"trash" xml:"trash"`
|
||||
Locking ocsBool `json:"locking" xml:"locking"`
|
||||
Sharing ocsBool `json:"sharing" xml:"sharing"`
|
||||
ArbitraryMetadata ocsBool `json:"arbitrary_metadata" xml:"arbitrary_metadata"`
|
||||
}
|
||||
|
||||
// NewProviderCapabilities maps a driver's declaration onto the OCS response type.
|
||||
func NewProviderCapabilities(c storage.Capabilities) *ProviderCapabilities {
|
||||
return &ProviderCapabilities{
|
||||
Upload: ocsBool(c.Upload),
|
||||
CreateContainer: ocsBool(c.CreateContainer),
|
||||
Delete: ocsBool(c.Delete),
|
||||
Move: ocsBool(c.Move),
|
||||
Versioning: ocsBool(c.Versioning),
|
||||
Trash: ocsBool(c.Trash),
|
||||
Locking: ocsBool(c.Locking),
|
||||
Sharing: ocsBool(c.Sharing),
|
||||
ArbitraryMetadata: ocsBool(c.ArbitraryMetadata),
|
||||
}
|
||||
}
|
||||
|
||||
// CapabilitiesSearch holds the search capabilities
|
||||
@@ -192,10 +253,13 @@ type CapabilitiesAppProvider struct {
|
||||
|
||||
// CapabilitiesFiles TODO this is storage specific, not global. What effect do these options have on the clients?
|
||||
type CapabilitiesFiles struct {
|
||||
PrivateLinks ocsBool `json:"privateLinks" xml:"privateLinks" mapstructure:"private_links"`
|
||||
BigFileChunking ocsBool `json:"bigfilechunking" xml:"bigfilechunking"`
|
||||
Undelete ocsBool `json:"undelete" xml:"undelete"`
|
||||
Versioning ocsBool `json:"versioning" xml:"versioning"`
|
||||
PrivateLinks ocsBool `json:"privateLinks" xml:"privateLinks" mapstructure:"private_links"`
|
||||
BigFileChunking ocsBool `json:"bigfilechunking" xml:"bigfilechunking"`
|
||||
// Deprecated: use capabilities.providers[<id>].trash.
|
||||
Undelete ocsBool `json:"undelete" xml:"undelete"`
|
||||
// Deprecated: use capabilities.providers[<id>].versioning.
|
||||
Versioning ocsBool `json:"versioning" xml:"versioning"`
|
||||
// Deprecated: use capabilities.providers[<id>].arbitrary_metadata.
|
||||
Favorites ocsBool `json:"favorites" xml:"favorites"`
|
||||
FullTextSearch ocsBool `json:"full_text_search" xml:"full_text_search" mapstructure:"full_text_search"`
|
||||
Tags ocsBool `json:"tags" xml:"tags"`
|
||||
@@ -207,7 +271,8 @@ type CapabilitiesFiles struct {
|
||||
|
||||
// CapabilitiesDav holds dav endpoint config
|
||||
type CapabilitiesDav struct {
|
||||
Chunking string `json:"chunking" xml:"chunking"`
|
||||
Chunking string `json:"chunking" xml:"chunking"`
|
||||
// Deprecated: use capabilities.providers[<id>].trash.
|
||||
Trashbin string `json:"trashbin" xml:"trashbin"`
|
||||
Reports []string `json:"reports" xml:"reports>element" mapstructure:"reports"`
|
||||
ChunkingParallelUploadDisabled bool `json:"chunkingParallelUploadDisabled" xml:"chunkingParallelUploadDisabled"`
|
||||
|
||||
+4
-1
@@ -28,12 +28,15 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// DataTX provides an abstraction around various data transfer protocols.
|
||||
type DataTX interface {
|
||||
Handler(fs storage.FS) (http.Handler, error)
|
||||
// Handler serves the protocol's data path. Uploads go through coord, which
|
||||
// owns the upload lifecycle for every driver; downloads read from driver.
|
||||
Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error)
|
||||
}
|
||||
|
||||
// EmitFileUploadedEvent is a helper function which publishes a FileUploaded event
|
||||
|
||||
+4
-3
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
@@ -78,7 +79,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
func (m *manager) Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sublog := m.log.With().Str("path", r.URL.Path).Logger()
|
||||
r = r.WithContext(appctx.WithLogger(r.Context(), &sublog))
|
||||
@@ -92,7 +93,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
metrics.DownloadsActive.Sub(1)
|
||||
}()
|
||||
}
|
||||
download.GetOrHeadFile(w, r, fs, "")
|
||||
download.GetOrHeadFile(w, r, driver, "")
|
||||
case "PUT":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
@@ -114,7 +115,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
ctx = ctxpkg.ContextSetLockID(ctx, lockID)
|
||||
}
|
||||
|
||||
info, err := fs.Upload(ctx, storage.UploadRequest{
|
||||
info, err := coord.Upload(ctx, upload.Request{
|
||||
Ref: ref,
|
||||
Body: r.Body,
|
||||
Length: r.ContentLength,
|
||||
|
||||
+4
-3
@@ -42,6 +42,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
@@ -80,7 +81,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
func (m *manager) Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error) {
|
||||
h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var spaceID string
|
||||
spaceID, r.URL.Path = router.ShiftPath(r.URL.Path)
|
||||
@@ -97,7 +98,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
metrics.DownloadsActive.Sub(1)
|
||||
}()
|
||||
}
|
||||
download.GetOrHeadFile(w, r, fs, spaceID)
|
||||
download.GetOrHeadFile(w, r, driver, spaceID)
|
||||
case "PUT":
|
||||
metrics.UploadsActive.Add(1)
|
||||
defer func() {
|
||||
@@ -117,7 +118,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
Path: fn,
|
||||
}
|
||||
var info *provider.ResourceInfo
|
||||
info, err = fs.Upload(ctx, storage.UploadRequest{
|
||||
info, err = coord.Upload(ctx, upload.Request{
|
||||
Ref: ref,
|
||||
Body: r.Body,
|
||||
Length: r.ContentLength,
|
||||
|
||||
+55
-54
@@ -35,13 +35,13 @@ import (
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/upload"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -89,20 +89,19 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
composable, ok := fs.(storage.ComposableFS)
|
||||
if !ok {
|
||||
return nil, errtypes.NotSupported("file system does not support the tus protocol")
|
||||
}
|
||||
func (m *manager) Handler(coord upload.Coordinator, fs storage.FS) (http.Handler, error) {
|
||||
// such a driver creates a new file under the id minted at initiate, so that id is
|
||||
// already the file's
|
||||
nc, ok := fs.(storage.NodeCreator)
|
||||
createsNode := ok && nc.PrepareCreatesNode()
|
||||
|
||||
// A storage backend for tusd may consist of multiple different parts which
|
||||
// handle upload creation, locking, termination and so on. The composer is a
|
||||
// place where all those separated pieces are joined together. In this example
|
||||
// we only use the file store but you may plug in multiple.
|
||||
// place where all those separated pieces are joined together.
|
||||
composer := tusd.NewStoreComposer()
|
||||
|
||||
// let the composable storage tell tus which extensions it supports
|
||||
composable.UseIn(composer)
|
||||
// the coordinator serves the tus protocol on behalf of every driver
|
||||
coord.UseIn(composer)
|
||||
|
||||
config := tusd.Config{
|
||||
StoreComposer: composer,
|
||||
@@ -110,7 +109,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
Logger: slog.New(tusdLogger{log: m.log}),
|
||||
// Add the finalized resource's etag and permissions to the last chunk's response
|
||||
// (see preFinishResponseCallback). https://github.com/opencloud-eu/opencloud/issues/2409
|
||||
PreFinishResponseCallback: m.preFinishResponseCallback(fs),
|
||||
PreFinishResponseCallback: m.preFinishResponseCallback(coord),
|
||||
}
|
||||
|
||||
if m.conf.CorsEnabled {
|
||||
@@ -135,33 +134,29 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if usl, ok := fs.(storage.UploadSessionLister); ok {
|
||||
// We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info
|
||||
go func() {
|
||||
for {
|
||||
ev := <-handler.CompleteUploads
|
||||
// We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files
|
||||
// so we create a Progress instance here that is used to read the correct properties
|
||||
ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID})
|
||||
if err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session")
|
||||
} else {
|
||||
if len(ups) < 1 {
|
||||
appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found")
|
||||
continue
|
||||
}
|
||||
up := ups[0]
|
||||
executant := up.Executant()
|
||||
ref := up.Reference()
|
||||
if m.publisher != nil {
|
||||
if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event")
|
||||
}
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
ev := <-handler.CompleteUploads
|
||||
// tus erases its info files, so read the session back for the event's properties
|
||||
ups, err := coord.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID})
|
||||
if err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session")
|
||||
continue
|
||||
}
|
||||
if len(ups) < 1 {
|
||||
appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found")
|
||||
continue
|
||||
}
|
||||
up := ups[0]
|
||||
executant := up.Executant()
|
||||
ref := up.Reference()
|
||||
if m.publisher != nil {
|
||||
if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil {
|
||||
appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event")
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
sublog := m.log.With().Str("uploadid", r.URL.Path).Logger()
|
||||
@@ -179,7 +174,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
// set etag, mtime and file id
|
||||
setHeaders(fs, w, r)
|
||||
setHeaders(coord, createsNode, w, r)
|
||||
handler.PostFile(w, r)
|
||||
case "HEAD":
|
||||
handler.HeadFile(w, r)
|
||||
@@ -189,7 +184,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
metrics.UploadsActive.Sub(1)
|
||||
}()
|
||||
// set etag, mtime and file id
|
||||
setHeaders(fs, w, r)
|
||||
setHeaders(coord, createsNode, w, r)
|
||||
handler.PatchFile(w, r)
|
||||
case "DELETE":
|
||||
handler.DelFile(w, r)
|
||||
@@ -214,16 +209,16 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) {
|
||||
// the final response is written, it looks up the finalized resource and attaches its etag and
|
||||
// WebDAV permissions to the response headers, so clients do not need a follow-up PROPFIND after
|
||||
// the last chunk. It mirrors the etag and permissions the ocdav gateway already produces for the
|
||||
// creation-with-upload path. The storage driver resolves the resource as the upload's executant
|
||||
// (the data path's transfer token carries no user identity), so this layer does not reconstruct
|
||||
// the user. It is best effort: it logs any lookup failure and still completes the upload (the
|
||||
// creation-with-upload path. The upload coordinator resolves the resource as the upload's
|
||||
// executant (the data path's transfer token carries no user identity), so this layer does not
|
||||
// reconstruct the user. It is best effort: it logs any lookup failure and still completes the upload (the
|
||||
// client falls back to a PROPFIND). https://github.com/opencloud-eu/opencloud/issues/2409
|
||||
func (m *manager) preFinishResponseCallback(fs storage.FS) func(tusd.HookEvent) (tusd.HTTPResponse, error) {
|
||||
resolver, ok := fs.(storage.UploadSessionResolver)
|
||||
func (m *manager) preFinishResponseCallback(r any) func(tusd.HookEvent) (tusd.HTTPResponse, error) {
|
||||
resolver, ok := r.(storage.UploadSessionResolver)
|
||||
return func(hook tusd.HookEvent) (tusd.HTTPResponse, error) {
|
||||
resp := tusd.HTTPResponse{Header: tusd.HTTPHeader{}}
|
||||
if !ok {
|
||||
// the storage driver cannot resolve the upload; the client falls back to a PROPFIND
|
||||
// nothing resolves the upload; the client falls back to a PROPFIND
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@@ -246,15 +241,10 @@ func (m *manager) preFinishResponseCallback(fs storage.FS) func(tusd.HookEvent)
|
||||
}
|
||||
}
|
||||
|
||||
func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) {
|
||||
func setHeaders(coord upload.Coordinator, createsNode bool, w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
id := path.Base(r.URL.Path)
|
||||
datastore, ok := fs.(tusd.DataStore)
|
||||
if !ok {
|
||||
appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore")
|
||||
return
|
||||
}
|
||||
upload, err := datastore.GetUpload(ctx, id)
|
||||
upload, err := coord.GetUpload(ctx, id)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage")
|
||||
return
|
||||
@@ -268,12 +258,23 @@ func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) {
|
||||
if expires != "" {
|
||||
w.Header().Set(net.HeaderTusUploadExpires, expires)
|
||||
}
|
||||
resourceid := &provider.ResourceId{
|
||||
if id := fileID(info, createsNode); id != "" {
|
||||
w.Header().Set(net.HeaderOCFileID, id)
|
||||
}
|
||||
}
|
||||
|
||||
// fileID returns the id of the file an upload writes, or "" while it is not known yet:
|
||||
// a new file's id is only valid once the upload commits, unless the driver creates the
|
||||
// file under the id minted at initiate.
|
||||
func fileID(info tusd.FileInfo, createsNode bool) string {
|
||||
if info.Storage["NodeExists"] != "true" && !createsNode {
|
||||
return ""
|
||||
}
|
||||
return storagespace.FormatResourceID(&provider.ResourceId{
|
||||
StorageId: info.MetaData["providerID"],
|
||||
SpaceId: info.Storage["SpaceRoot"],
|
||||
OpaqueId: info.Storage["NodeId"],
|
||||
}
|
||||
w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(resourceid))
|
||||
})
|
||||
}
|
||||
|
||||
// tusdLogger is a logger implementation (slog) for tusd that uses zerolog.
|
||||
|
||||
+17
-10
@@ -1,4 +1,4 @@
|
||||
// Package metrics provides prometheus metrics for the data managers..
|
||||
// Package metrics provides prometheus metrics for the data managers.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
@@ -25,36 +25,43 @@ var (
|
||||
// UploadSessionsInitiated is the number of upload sessions that have been initiated
|
||||
UploadSessionsInitiated = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_initiated",
|
||||
Help: "Number of uploads sessions that were initiated",
|
||||
Help: "Number of upload sessions that were initiated",
|
||||
})
|
||||
// UploadSessionsBytesReceived is the number of upload sessions that have received all bytes
|
||||
// UploadSessionsBytesReceived counts upload sessions whose data has been fully received and is
|
||||
// safely stored, before postprocessing decides their outcome; it counts sessions, not bytes,
|
||||
// despite its name (kept for compatibility, see https://github.com/opencloud-eu/reva/issues/930)
|
||||
UploadSessionsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_bytes_received",
|
||||
Help: "Number of uploads sessions that have received all bytes",
|
||||
Help: "Number of upload sessions whose data has been fully received, before postprocessing (counts sessions, not bytes)",
|
||||
})
|
||||
// UploadSessionsFinalized is the number of upload sessions that have received all bytes
|
||||
// UploadSessionsFinalized is the number of upload sessions that have successfully completed
|
||||
UploadSessionsFinalized = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_finalized",
|
||||
Help: "Number of uploads sessions that have successfully completed",
|
||||
Help: "Number of upload sessions that have successfully completed",
|
||||
})
|
||||
// UploadSessionsAborted is the number of upload sessions that have been aborted
|
||||
UploadSessionsAborted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_aborted",
|
||||
Help: "Number of uploads sessions that have aborted by postprocessing",
|
||||
Help: "Number of upload sessions that have been aborted by postprocessing",
|
||||
})
|
||||
// UploadSessionsCommitFailed is the number of upload sessions whose blob commit failed after retries and were reverted
|
||||
UploadSessionsCommitFailed = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_commit_failed",
|
||||
Help: "Number of upload sessions whose blob commit failed after retries and were reverted",
|
||||
})
|
||||
// UploadSessionsDeleted is the number of upload sessions that have been deleted
|
||||
UploadSessionsDeleted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_deleted",
|
||||
Help: "Number of uploads sessions that have been deleted by postprocessing",
|
||||
Help: "Number of upload sessions that have been deleted by postprocessing",
|
||||
})
|
||||
// UploadSessionsRestarted is the number of upload sessions that have been restarted
|
||||
UploadSessionsRestarted = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_restarted",
|
||||
Help: "Number of uploads sessions that have been restarted by postprocessing",
|
||||
Help: "Number of upload sessions that have been restarted by postprocessing",
|
||||
})
|
||||
// UploadSessionsScanned is the number of upload sessions that have been scanned by antivirus
|
||||
UploadSessionsScanned = promauto.NewCounter(prometheus.CounterOpts{
|
||||
Name: "reva_upload_sessions_scanned",
|
||||
Help: "Number of uploads sessions that have been scanned by antivirus",
|
||||
Help: "Number of upload sessions that have been scanned by antivirus",
|
||||
})
|
||||
)
|
||||
Generated
Vendored
+1
-1
@@ -33,7 +33,7 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -32,7 +32,7 @@ import (
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -35,7 +35,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
+8
-10
@@ -75,16 +75,6 @@ func (fs *hellofs) Move(ctx context.Context, oldRef, newRef *provider.Reference)
|
||||
return nil, errtypes.NotSupported("unimplemented")
|
||||
}
|
||||
|
||||
// Upload creates or updates a resource of type file with a new revision
|
||||
func (fs *hellofs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) {
|
||||
return nil, errtypes.NotSupported("hellofs: upload not supported")
|
||||
}
|
||||
|
||||
// InitiateUpload returns a list of protocols with urls that can be used to append bytes to a new upload session
|
||||
func (fs *hellofs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
|
||||
return nil, errtypes.NotSupported("hellofs: initiate upload not supported")
|
||||
}
|
||||
|
||||
// MarkProcessing toggles a processing flag on the resource.
|
||||
func (fs *hellofs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error {
|
||||
return errtypes.NotSupported("hellofs: mark processing not supported")
|
||||
@@ -95,6 +85,14 @@ func (fs *hellofs) CommitUpload(_ context.Context, _ *provider.Reference, _ stri
|
||||
return errtypes.NotSupported("hellofs: commit upload not supported")
|
||||
}
|
||||
|
||||
func (fs *hellofs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) {
|
||||
return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil
|
||||
}
|
||||
|
||||
func (fs *hellofs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// grants
|
||||
|
||||
// DenyGrant marks a resource as denied for a recipient
|
||||
|
||||
-2
@@ -31,9 +31,7 @@ import (
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/local"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/localhome"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix"
|
||||
_ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng"
|
||||
// Add your own here
|
||||
)
|
||||
+30
-48
@@ -21,7 +21,6 @@ package nextcloud
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -271,7 +270,20 @@ func (nc *StorageDriver) CreateDir(ctx context.Context, ref *provider.Reference)
|
||||
|
||||
// TouchFile as defined in the storage.FS interface
|
||||
func (nc *StorageDriver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error {
|
||||
return fmt.Errorf("unimplemented: TouchFile")
|
||||
type paramsObj struct {
|
||||
Ref *provider.Reference `json:"ref"`
|
||||
MarkProcessing bool `json:"markprocessing"`
|
||||
MTime string `json:"mtime"`
|
||||
}
|
||||
bodyStr, err := json.Marshal(¶msObj{Ref: ref, MarkProcessing: markprocessing, MTime: mtime})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Info().Msgf("TouchFile %s", bodyStr)
|
||||
|
||||
_, _, err = nc.do(ctx, Action{"TouchFile", string(bodyStr)})
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete as defined in the storage.FS interface
|
||||
@@ -372,58 +384,28 @@ func (nc *StorageDriver) ListFolder(ctx context.Context, ref *provider.Reference
|
||||
return pointers, err
|
||||
}
|
||||
|
||||
// InitiateUpload as defined in the storage.FS interface
|
||||
func (nc *StorageDriver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
|
||||
type paramsObj struct {
|
||||
Ref *provider.Reference `json:"ref"`
|
||||
UploadLength int64 `json:"uploadLength"`
|
||||
Metadata map[string]string `json:"metadata"`
|
||||
}
|
||||
bodyObj := ¶msObj{
|
||||
Ref: ref,
|
||||
UploadLength: uploadLength,
|
||||
Metadata: metadata,
|
||||
}
|
||||
bodyStr, _ := json.Marshal(bodyObj)
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Info().Msgf("InitiateUpload %s", bodyStr)
|
||||
|
||||
_, respBody, err := nc.do(ctx, Action{"InitiateUpload", string(bodyStr)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
respMap := make(map[string]string)
|
||||
err = json.Unmarshal(respBody, &respMap)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return respMap, err
|
||||
}
|
||||
|
||||
// MarkProcessing as defined in the storage.FS interface
|
||||
func (nc *StorageDriver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error {
|
||||
return errtypes.NotSupported("nextcloud: mark processing not supported")
|
||||
// MarkProcessing as defined in the storage.FS interface.
|
||||
// No sciencemesh endpoint toggles the flag alone. A no-op, not NotSupported:
|
||||
// the coordinator treats a failed mark as fatal.
|
||||
func (nc *StorageDriver) MarkProcessing(_ context.Context, _ *provider.Reference, _ bool, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CommitUpload as defined in the storage.FS interface
|
||||
func (nc *StorageDriver) CommitUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.UploadSource) error {
|
||||
return errtypes.NotSupported("nextcloud: commit upload not supported")
|
||||
func (nc *StorageDriver) CommitUpload(ctx context.Context, ref *provider.Reference, _ string, source storage.UploadSource) error {
|
||||
if source.Body == nil {
|
||||
// A nil body would PUT an empty request and truncate the file.
|
||||
return errtypes.BadRequest("nextcloud: source body is nil")
|
||||
}
|
||||
return nc.doUpload(ctx, ref.GetPath(), source.Body)
|
||||
}
|
||||
|
||||
// Upload as defined in the storage.FS interface
|
||||
func (nc *StorageDriver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) {
|
||||
err := nc.doUpload(ctx, req.Ref.Path, req.Body)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
func (nc *StorageDriver) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) {
|
||||
return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil
|
||||
}
|
||||
|
||||
// return id, etag and mtime
|
||||
ri, err := nc.GetMD(ctx, req.Ref, []string{}, []string{"id", "etag", "mtime"})
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
|
||||
return ri, nil
|
||||
func (nc *StorageDriver) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download as defined in the storage.FS interface
|
||||
|
||||
Generated
Vendored
+19
-1
@@ -74,6 +74,11 @@ var responses = map[string]Response{
|
||||
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/EmptyRecycle `: {200, ``, serverStateEmpty},
|
||||
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetQuota `: {200, `{"totalBytes":456,"usedBytes":123}`, serverStateEmpty},
|
||||
|
||||
// the parent of an upload target, stated for its permissions and its id
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":[]} EMPTY`: {200, `{"opaque":{},"type":2,"id":{"opaque_id":"fileid-/"},"checksum":{},"etag":"deadbeef","mime_type":"httpd/unix-directory","mtime":{"seconds":1234567890},"path":"/","permission_set":{"initiate_file_upload":true,"stat":true,"list_container":true},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty},
|
||||
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":null} EMPTY`: {404, ``, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":null} HOME`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateHome},
|
||||
|
||||
@@ -109,9 +114,21 @@ var responses = map[string]Response{
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/versionedFile"},"mdKeys":null} EMPTY`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":2,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/versionedFile"},"mdKeys":null} FILE-RESTORED`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":1,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateFileRestored},
|
||||
|
||||
// The coordinator resolves the upload target itself, so it stats the file with the
|
||||
// full reference before every upload. Reporting the file as existing keeps it on the
|
||||
// overwrite path, which needs neither a parent stat nor a TouchFile (whose mtime is
|
||||
// wall-clock derived and could never match this table's exact-body keys).
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"mdKeys":[]}`: {200, `{"opaque":{},"type":1,"id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","opaque_id":"fileid-/versionedFile"},"parent_id":{"opaque_id":"fileid-/"},"name":"versionedFile","checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{"initiate_file_upload":true,"stat":true,"list_container":true,"list_file_versions":true,"restore_file_version":true},"size":1,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty},
|
||||
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetPathByID {"storage_id":"00000000-0000-0000-0000-000000000000","opaque_id":"fileid-/some/path"} EMPTY`: {200, "/subdir", serverStateEmpty},
|
||||
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty},
|
||||
// the coordinator resolves the upload target itself: the file does not exist yet,
|
||||
// so it stats the parent for the permissions and the parent id
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":[]}`: {404, ``, serverStateEmpty},
|
||||
// a zero-length upload finishes at once, so the node is created and read back here
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/TouchFile {"ref":{"resource_id":{"opaque_id":"fileid-/"},"path":"file"},"markprocessing":false,"mtime":"1234567890"}`: {200, ``, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"resource_id":{"opaque_id":"fileid-/"},"path":"file"},"mdKeys":null}`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/file"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/file","permission_set":{},"size":0,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"path":"/file"},"uploadLength":0,"metadata":{"providerID":""}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":1,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":2,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty},
|
||||
@@ -160,6 +177,7 @@ var responses = map[string]Response{
|
||||
`POST /apps/sciencemesh/~tester/api/storage/ListFolder {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some"},"mdKeys":["val1","val2","val3"]}`: {200, `[{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/some/path","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}]`, serverStateEmpty},
|
||||
// `POST /apps/sciencemesh/~tester/api/storage/ListFolder {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some"},"mdKeys":["val1","val2","val3"]}`: {200, `[{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/path","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}]`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~tester/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some/path"},"uploadLength":12345,"metadata":{"key1":"val1","key2":"val2","key3":"val3"}}`: {200, `{ "not":"sure", "what": "should be", "returned": "here" }`, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~tester/api/storage/TouchFile {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"some/file/path.txt"},"markprocessing":false,"mtime":"1234567890"}`: {200, ``, serverStateEmpty},
|
||||
`PUT /apps/sciencemesh/~tester/api/storage/Upload/some/file/path.txt shiny!`: {200, ``, serverStateEmpty},
|
||||
`POST /apps/sciencemesh/~tester/api/storage/GetMD {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"some/file/path.txt"},"mdKeys":[]}`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"key1":"val1","key2":"val2","key3":"val3"}}}`, serverStateEmpty},
|
||||
`GET /apps/sciencemesh/~tester/api/storage/Download/some/file/path.txt `: {200, `the contents of the file`, serverStateEmpty},
|
||||
|
||||
Generated
Vendored
-199
@@ -1,199 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package blobstore
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// ErrBlobIDEmpty is returned when the BlobID is empty
|
||||
var ErrBlobIDEmpty = fmt.Errorf("blobstore: BlobID is empty")
|
||||
|
||||
// Blobstore provides an interface to an filesystem based blobstore
|
||||
type Blobstore struct {
|
||||
root string
|
||||
}
|
||||
|
||||
// New returns a new Blobstore
|
||||
func New(root string) (*Blobstore, error) {
|
||||
err := os.MkdirAll(root, 0700)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Blobstore{
|
||||
root: root,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload stores some data in the blobstore under the given key
|
||||
func (bs *Blobstore) Upload(node *node.Node, source string) (err error) {
|
||||
if node.BlobID == "" {
|
||||
return ErrBlobIDEmpty
|
||||
}
|
||||
|
||||
dest := bs.Path(node)
|
||||
|
||||
// ensure parent path exists
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob")
|
||||
}
|
||||
|
||||
if err := os.Rename(source, dest); err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Rename failed, file needs to be copied.
|
||||
file, err := os.Open(source)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload")
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0700)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not open blob '%s' for writing", dest)
|
||||
}
|
||||
defer func() {
|
||||
if cerr := f.Close(); cerr != nil && err == nil {
|
||||
err = errors.Wrapf(cerr, "could not close blob '%s'", dest)
|
||||
}
|
||||
}()
|
||||
|
||||
w := bufio.NewWriter(f)
|
||||
if _, err = w.ReadFrom(file); err != nil {
|
||||
return errors.Wrapf(err, "could not write blob '%s'", dest)
|
||||
}
|
||||
|
||||
if err = w.Flush(); err != nil {
|
||||
return errors.Wrapf(err, "could not flush blob '%s'", dest)
|
||||
}
|
||||
|
||||
// Ensure data is synced to disk before returning
|
||||
if err = f.Sync(); err != nil {
|
||||
return errors.Wrapf(err, "could not sync blob '%s' to disk", dest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFromReader stores data from a reader in the blobstore under the given key.
|
||||
// It buffers to a temp file on the same filesystem as the destination so that the
|
||||
// final placement is an atomic os.Rename.
|
||||
func (bs *Blobstore) UploadFromReader(node *node.Node, r io.Reader, size int64) error {
|
||||
if node.BlobID == "" {
|
||||
return ErrBlobIDEmpty
|
||||
}
|
||||
|
||||
dest := bs.Path(node)
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob")
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp(filepath.Dir(dest), "reva-blobstore-upload-*")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating temp file for upload")
|
||||
}
|
||||
tmpName := tmp.Name()
|
||||
defer func() {
|
||||
tmp.Close()
|
||||
os.Remove(tmpName)
|
||||
}()
|
||||
|
||||
w := bufio.NewWriter(tmp)
|
||||
if _, err = io.Copy(w, r); err != nil {
|
||||
return errors.Wrapf(err, "could not write blob to temp file '%s'", tmpName)
|
||||
}
|
||||
if err = w.Flush(); err != nil {
|
||||
return errors.Wrapf(err, "could not flush blob temp file '%s'", tmpName)
|
||||
}
|
||||
if err = tmp.Sync(); err != nil {
|
||||
return errors.Wrapf(err, "could not sync blob temp file '%s'", tmpName)
|
||||
}
|
||||
tmp.Close()
|
||||
|
||||
if err = os.Rename(tmpName, dest); err != nil {
|
||||
return errors.Wrapf(err, "could not rename blob temp file to '%s'", dest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download retrieves a blob from the blobstore for reading
|
||||
func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) {
|
||||
if node.BlobID == "" {
|
||||
return nil, ErrBlobIDEmpty
|
||||
}
|
||||
|
||||
dest := bs.Path(node)
|
||||
file, err := os.Open(dest)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not read blob '%s'", dest)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
// Delete deletes a blob from the blobstore
|
||||
func (bs *Blobstore) Delete(node *node.Node) error {
|
||||
if node.BlobID == "" {
|
||||
return ErrBlobIDEmpty
|
||||
}
|
||||
dest := bs.Path(node)
|
||||
if err := utils.RemoveItem(dest); err != nil {
|
||||
return errors.Wrapf(err, "could not delete blob '%s'", dest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists all blobs in the Blobstore
|
||||
func (bs *Blobstore) List() ([]*node.Node, error) {
|
||||
dirs, err := filepath.Glob(filepath.Join(bs.root, "spaces", "*", "*", "blobs", "*", "*", "*", "*", "*"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobids := make([]*node.Node, 0, len(dirs))
|
||||
for _, d := range dirs {
|
||||
_, s, _ := strings.Cut(d, "spaces")
|
||||
spaceraw, blobraw, _ := strings.Cut(s, "blobs")
|
||||
blobids = append(blobids, &node.Node{
|
||||
SpaceID: strings.ReplaceAll(spaceraw, "/", ""),
|
||||
BlobID: strings.ReplaceAll(blobraw, "/", ""),
|
||||
})
|
||||
}
|
||||
return blobids, nil
|
||||
}
|
||||
|
||||
func (bs *Blobstore) Path(node *node.Node) string {
|
||||
return filepath.Join(
|
||||
bs.root,
|
||||
filepath.Clean(filepath.Join(
|
||||
"/", "spaces", lookup.Pathify(node.SpaceID, 1, 2), "blobs", lookup.Pathify(node.BlobID, 4, 2)),
|
||||
),
|
||||
)
|
||||
}
|
||||
-51
@@ -1,51 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package ocis
|
||||
|
||||
import (
|
||||
"path"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registry.Register("ocis", New)
|
||||
}
|
||||
|
||||
// New returns an implementation to of the storage.FS interface that talk to
|
||||
// a local filesystem.
|
||||
func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) {
|
||||
o, err := options.New(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
bs, err := blobstore.New(path.Join(o.Root))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return decomposedfs.NewDefault(m, bs, stream, log)
|
||||
}
|
||||
+3
-519
@@ -20,155 +20,12 @@ package owncloudsql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
iofs "io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/conversions"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/logger"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/mime"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
)
|
||||
|
||||
var defaultFilePerm = os.FileMode(0664)
|
||||
|
||||
func (fs *owncloudsqlfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) {
|
||||
upload, err := fs.GetUpload(ctx, req.Ref.GetPath())
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error retrieving upload")
|
||||
}
|
||||
|
||||
uploadInfo := upload.(*fileUpload)
|
||||
|
||||
p := uploadInfo.info.Storage["InternalDestination"]
|
||||
if chunking.IsChunked(p) {
|
||||
var assembledFile string
|
||||
p, assembledFile, err = fs.chunkHandler.WriteChunk(p, req.Body)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
if p == "" {
|
||||
if err = uploadInfo.Terminate(ctx); err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error removing auxiliary files")
|
||||
}
|
||||
return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String())
|
||||
}
|
||||
uploadInfo.info.Storage["InternalDestination"] = p
|
||||
fd, err := os.Open(assembledFile)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error opening assembled file")
|
||||
}
|
||||
defer fd.Close()
|
||||
defer os.RemoveAll(assembledFile)
|
||||
req.Body = fd
|
||||
}
|
||||
|
||||
if _, err := uploadInfo.WriteChunk(ctx, 0, req.Body); err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error writing to binary file")
|
||||
}
|
||||
|
||||
if err := uploadInfo.FinishUpload(ctx); err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
|
||||
if uff != nil {
|
||||
info := uploadInfo.info
|
||||
uploadRef := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: info.MetaData["providerID"],
|
||||
SpaceId: info.Storage["SpaceRoot"],
|
||||
OpaqueId: info.Storage["SpaceRoot"],
|
||||
},
|
||||
Path: utils.MakeRelativePath(filepath.Join(info.MetaData["dir"], info.MetaData["filename"])),
|
||||
}
|
||||
owner, ok := ctxpkg.ContextGetUser(uploadInfo.ctx)
|
||||
if !ok {
|
||||
return &provider.ResourceInfo{}, errtypes.PreconditionFailed("error getting user from uploadinfo context")
|
||||
}
|
||||
// spaces support in localfs needs to be revisited:
|
||||
// * info.Storage["SpaceRoot"] is never set
|
||||
// * there is no space owner or manager that could be passed to the UploadFinishedFunc
|
||||
uff(owner.Id, owner.Id, uploadRef)
|
||||
}
|
||||
|
||||
ri := &provider.ResourceInfo{
|
||||
// fill with at least fileid, mtime and etag
|
||||
Id: &provider.ResourceId{
|
||||
StorageId: uploadInfo.info.MetaData["providerID"],
|
||||
SpaceId: uploadInfo.info.Storage["StorageId"],
|
||||
OpaqueId: uploadInfo.info.Storage["fileid"],
|
||||
},
|
||||
Etag: uploadInfo.info.MetaData["etag"],
|
||||
}
|
||||
|
||||
if mtime, err := utils.MTimeToTS(uploadInfo.info.MetaData["mtime"]); err == nil {
|
||||
ri.Mtime = &mtime
|
||||
}
|
||||
|
||||
return ri, nil
|
||||
}
|
||||
|
||||
// InitiateUpload returns upload ids corresponding to different protocols it supports
|
||||
// TODO read optional content for small files in this request
|
||||
func (fs *owncloudsqlfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
|
||||
ip, err := fs.resolve(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "owncloudsql: error resolving reference")
|
||||
}
|
||||
|
||||
// permissions are checked in NewUpload below
|
||||
|
||||
p := fs.toStoragePath(ctx, ip)
|
||||
|
||||
info := tusd.FileInfo{
|
||||
MetaData: tusd.MetaData{
|
||||
"filename": filepath.Base(p),
|
||||
"dir": filepath.Dir(p),
|
||||
"mtime": strconv.FormatInt(time.Now().Unix(), 10),
|
||||
},
|
||||
Size: uploadLength,
|
||||
}
|
||||
|
||||
if metadata != nil {
|
||||
info.MetaData["providerID"] = metadata["providerID"]
|
||||
if metadata["mtime"] != "" {
|
||||
info.MetaData["mtime"] = metadata["mtime"]
|
||||
}
|
||||
if _, ok := metadata["sizedeferred"]; ok {
|
||||
info.SizeIsDeferred = true
|
||||
}
|
||||
}
|
||||
|
||||
upload, err := fs.NewUpload(ctx, info)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info, _ = upload.GetInfo(ctx)
|
||||
|
||||
return map[string]string{
|
||||
"simple": info.ID,
|
||||
"tus": info.ID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (fs *owncloudsqlfs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error {
|
||||
return errtypes.NotSupported("op not supported")
|
||||
}
|
||||
@@ -177,383 +34,10 @@ func (fs *owncloudsqlfs) CommitUpload(_ context.Context, _ *provider.Reference,
|
||||
return errtypes.NotSupported("op not supported")
|
||||
}
|
||||
|
||||
// UseIn tells the tus upload middleware which extensions it supports.
|
||||
func (fs *owncloudsqlfs) UseIn(composer *tusd.StoreComposer) {
|
||||
composer.UseCore(fs)
|
||||
composer.UseTerminater(fs)
|
||||
composer.UseConcater(fs)
|
||||
composer.UseLengthDeferrer(fs)
|
||||
func (fs *owncloudsqlfs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) {
|
||||
return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil
|
||||
}
|
||||
|
||||
// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol
|
||||
// - the storage needs to implement NewUpload and GetUpload
|
||||
// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload
|
||||
|
||||
func (fs *owncloudsqlfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) {
|
||||
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Debug().Interface("info", info).Msg("owncloudsql: NewUpload")
|
||||
|
||||
if info.MetaData["filename"] == "" {
|
||||
return nil, errors.New("owncloudsql: missing filename in metadata")
|
||||
}
|
||||
info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"])
|
||||
|
||||
dir := info.MetaData["dir"]
|
||||
if dir == "" {
|
||||
return nil, errors.New("owncloudsql: missing dir in metadata")
|
||||
}
|
||||
info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"])
|
||||
|
||||
ip := fs.toInternalPath(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"]))
|
||||
|
||||
// check permissions
|
||||
var perm *provider.ResourcePermissions
|
||||
var perr error
|
||||
var fsInfo iofs.FileInfo
|
||||
// if destination exists
|
||||
if fsInfo, err = os.Stat(ip); err == nil {
|
||||
// check permissions of file to be overwritten
|
||||
perm, perr = fs.readPermissions(ctx, ip)
|
||||
} else {
|
||||
// check permissions of parent folder
|
||||
perm, perr = fs.readPermissions(ctx, filepath.Dir(ip))
|
||||
}
|
||||
if perr == nil {
|
||||
if !perm.InitiateFileUpload {
|
||||
return nil, errtypes.PermissionDenied("")
|
||||
}
|
||||
} else {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, errtypes.NotFound(fs.toStoragePath(ctx, filepath.Dir(ip)))
|
||||
}
|
||||
return nil, errors.Wrap(err, "owncloudsql: error reading permissions")
|
||||
}
|
||||
|
||||
// if we are trying to overwriting a folder with a file
|
||||
if fsInfo != nil && fsInfo.IsDir() {
|
||||
return nil, errtypes.PreconditionFailed("resource is not a file")
|
||||
}
|
||||
|
||||
log.Debug().Interface("info", info).Msg("owncloudsql: resolved filename")
|
||||
|
||||
info.ID = uuid.New().String()
|
||||
|
||||
binPath, err := fs.getUploadPath(ctx, info.ID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "owncloudsql: error resolving upload path")
|
||||
}
|
||||
usr := ctxpkg.ContextMustGetUser(ctx)
|
||||
storageID, err := fs.getStorage(ctx, ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info.Storage = map[string]string{
|
||||
"Type": "OwnCloudStore",
|
||||
"BinPath": binPath,
|
||||
"InternalDestination": ip,
|
||||
"Permissions": strconv.Itoa((int)(conversions.RoleFromResourcePermissions(perm, false).OCSPermissions())),
|
||||
|
||||
"Idp": usr.Id.Idp,
|
||||
"UserId": usr.Id.OpaqueId,
|
||||
"UserName": usr.Username,
|
||||
|
||||
"LogLevel": log.GetLevel().String(),
|
||||
|
||||
"StorageId": strconv.Itoa(storageID),
|
||||
}
|
||||
// Create binary file in the upload folder with no content
|
||||
log.Debug().Interface("info", info).Msg("owncloudsql: built storage info")
|
||||
file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
u := &fileUpload{
|
||||
info: info,
|
||||
binPath: binPath,
|
||||
infoPath: filepath.Join(fs.c.UploadInfoDir, info.ID+".info"),
|
||||
fs: fs,
|
||||
ctx: ctx,
|
||||
}
|
||||
|
||||
// writeInfo creates the file by itself if necessary
|
||||
err = u.writeInfo()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (fs *owncloudsqlfs) getUploadPath(ctx context.Context, uploadID string) (string, error) {
|
||||
u, ok := ctxpkg.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")
|
||||
return "", err
|
||||
}
|
||||
layout := templates.WithUser(u, fs.c.UserLayout)
|
||||
return filepath.Join(fs.c.DataDirectory, layout, "uploads", uploadID), nil
|
||||
}
|
||||
|
||||
// GetUpload returns the Upload for the given upload id
|
||||
func (fs *owncloudsqlfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) {
|
||||
infoPath := filepath.Join(fs.c.UploadInfoDir, id+".info")
|
||||
|
||||
info := tusd.FileInfo{}
|
||||
data, err := os.ReadFile(infoPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// Interpret os.ErrNotExist as 404 Not Found
|
||||
err = tusd.ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stat, err := os.Stat(info.Storage["BinPath"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info.Offset = stat.Size()
|
||||
|
||||
u := &userpb.User{
|
||||
Id: &userpb.UserId{
|
||||
Idp: info.Storage["Idp"],
|
||||
OpaqueId: info.Storage["UserId"],
|
||||
},
|
||||
Username: info.Storage["UserName"],
|
||||
}
|
||||
|
||||
ctx = ctxpkg.ContextSetUser(ctx, u)
|
||||
// TODO configure the logger the same way ... store and add traceid in file info
|
||||
|
||||
opts := []logger.Option{
|
||||
logger.WithLevel(info.Storage["LogLevel"]),
|
||||
logger.WithWriter(os.Stderr, logger.ConsoleMode),
|
||||
}
|
||||
l := logger.New(opts...)
|
||||
|
||||
sub := l.With().Int("pid", os.Getpid()).Logger()
|
||||
|
||||
ctx = appctx.WithLogger(ctx, &sub)
|
||||
|
||||
return &fileUpload{
|
||||
info: info,
|
||||
binPath: info.Storage["BinPath"],
|
||||
infoPath: infoPath,
|
||||
fs: fs,
|
||||
ctx: ctx,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type fileUpload struct {
|
||||
// info stores the current information about the upload
|
||||
info tusd.FileInfo
|
||||
// infoPath is the path to the .info file
|
||||
infoPath string
|
||||
// binPath is the path to the binary file (which has no extension)
|
||||
binPath string
|
||||
// only fs knows how to handle metadata and versions
|
||||
fs *owncloudsqlfs
|
||||
// a context with a user
|
||||
// TODO add logger as well?
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// GetInfo returns the FileInfo
|
||||
func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) {
|
||||
return upload.info, nil
|
||||
}
|
||||
|
||||
// WriteChunk writes the stream from the reader to the given offset of the upload
|
||||
func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
|
||||
file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
n, err := io.Copy(file, src)
|
||||
|
||||
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
|
||||
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
|
||||
// However, for OwnCloudStore it's not important whether the stream has ended
|
||||
// on purpose or accidentally.
|
||||
if err != nil {
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
return n, err
|
||||
}
|
||||
}
|
||||
|
||||
upload.info.Offset += n
|
||||
err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk
|
||||
|
||||
return n, err
|
||||
}
|
||||
|
||||
// GetReader returns an io.Reader for the upload
|
||||
func (upload *fileUpload) GetReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
return os.Open(upload.binPath)
|
||||
}
|
||||
|
||||
// writeInfo updates the entire information. Everything will be overwritten.
|
||||
func (upload *fileUpload) writeInfo() error {
|
||||
log.Debug().Str("path", upload.infoPath).Msg("Writing info file")
|
||||
data, err := json.Marshal(upload.info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(upload.infoPath, data, defaultFilePerm)
|
||||
}
|
||||
|
||||
// FinishUpload finishes an upload and moves the file to the internal destination
|
||||
func (upload *fileUpload) FinishUpload(ctx context.Context) error {
|
||||
|
||||
ip := upload.info.Storage["InternalDestination"]
|
||||
|
||||
// if destination exists
|
||||
// TODO check etag with If-Match header
|
||||
if _, err := os.Stat(ip); err == nil {
|
||||
// create revision
|
||||
if err := upload.fs.archiveRevision(upload.ctx, upload.fs.getVersionsPath(upload.ctx, ip), ip); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
sha1h, md5h, adler32h, err := upload.fs.HashFile(upload.binPath)
|
||||
if err != nil {
|
||||
log.Err(err).Msg("owncloudsql: could not open file for checksumming")
|
||||
}
|
||||
|
||||
err = os.Rename(upload.binPath, ip)
|
||||
if err != nil {
|
||||
log.Err(err).Interface("info", upload.info).
|
||||
Str("binPath", upload.binPath).
|
||||
Str("ipath", ip).
|
||||
Msg("owncloudsql: could not rename")
|
||||
return err
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
fi, err = os.Stat(ip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
perms, err := strconv.Atoi(upload.info.Storage["Permissions"])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if upload.info.MetaData["mtime"] == "" {
|
||||
upload.info.MetaData["mtime"] = fmt.Sprintf("%d", fi.ModTime().Unix())
|
||||
}
|
||||
if upload.info.MetaData["etag"] == "" {
|
||||
upload.info.MetaData["etag"] = calcEtag(upload.ctx, fi)
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
"path": upload.fs.toDatabasePath(ip),
|
||||
"checksum": fmt.Sprintf("SHA1:%032x MD5:%032x ADLER32:%032x", sha1h, md5h, adler32h),
|
||||
"etag": upload.info.MetaData["etag"],
|
||||
"size": upload.info.Size,
|
||||
"mimetype": mime.Detect(false, ip),
|
||||
"permissions": perms,
|
||||
"mtime": upload.info.MetaData["mtime"],
|
||||
"storage_mtime": upload.info.MetaData["mtime"],
|
||||
}
|
||||
var fileid int
|
||||
fileid, err = upload.fs.filecache.InsertOrUpdate(ctx, upload.info.Storage["StorageId"], data, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
upload.info.Storage["fileid"] = fmt.Sprintf("%d", fileid)
|
||||
|
||||
// only delete the upload if it was successfully written to the storage
|
||||
if err := os.Remove(upload.infoPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Err(err).Interface("info", upload.info).Msg("owncloudsql: could not delete upload info")
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return upload.fs.propagate(upload.ctx, ip)
|
||||
}
|
||||
|
||||
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
|
||||
// - the storage needs to implement AsTerminatableUpload
|
||||
// - the upload needs to implement Terminate
|
||||
|
||||
// AsTerminatableUpload returns a TerminatableUpload
|
||||
func (fs *owncloudsqlfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload {
|
||||
return upload.(*fileUpload)
|
||||
}
|
||||
|
||||
// Terminate terminates the upload
|
||||
func (upload *fileUpload) Terminate(ctx context.Context) error {
|
||||
if err := os.Remove(upload.infoPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.Remove(upload.binPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
func (fs *owncloudsqlfs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
|
||||
// - the storage needs to implement AsLengthDeclarableUpload
|
||||
// - the upload needs to implement DeclareLength
|
||||
|
||||
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
|
||||
func (fs *owncloudsqlfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload {
|
||||
return upload.(*fileUpload)
|
||||
}
|
||||
|
||||
// DeclareLength updates the upload length information
|
||||
func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error {
|
||||
upload.info.Size = length
|
||||
upload.info.SizeIsDeferred = false
|
||||
return upload.writeInfo()
|
||||
}
|
||||
|
||||
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
|
||||
// - the storage needs to implement AsConcatableUpload
|
||||
// - the upload needs to implement ConcatUploads
|
||||
|
||||
// AsConcatableUpload returns a ConcatableUpload
|
||||
func (fs *owncloudsqlfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload {
|
||||
return upload.(*fileUpload)
|
||||
}
|
||||
|
||||
// ConcatUploads concatenates multiple uploads
|
||||
func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) {
|
||||
file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
for _, partialUpload := range uploads {
|
||||
fileUpload := partialUpload.(*fileUpload)
|
||||
|
||||
src, err := os.Open(fileUpload.binPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(file, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Generated
Vendored
+11
-3
@@ -67,7 +67,10 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if bs.canUseRenameForUpload {
|
||||
// a source staged in the tmp dir already is the temp file (see UploadFromReader)
|
||||
staged := filepath.Clean(source) == tempName
|
||||
|
||||
if !staged && bs.canUseRenameForUpload {
|
||||
err := os.Rename(source, tempName)
|
||||
switch {
|
||||
case err == nil:
|
||||
@@ -80,7 +83,7 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error {
|
||||
}
|
||||
}
|
||||
|
||||
if !bs.canUseRenameForUpload {
|
||||
if !staged && !bs.canUseRenameForUpload {
|
||||
sourceFile, err := os.Open(source)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open source file '%s': %v", source, err)
|
||||
@@ -193,7 +196,12 @@ func (bs *Blobstore) UploadFromReader(n *node.Node, r io.Reader, size int64) err
|
||||
if n.SpaceRoot == nil {
|
||||
return errors.New("node has no space root")
|
||||
}
|
||||
f, err := os.CreateTemp(n.SpaceRoot.InternalPath(), ".upload-*")
|
||||
// stage in the space's tmp dir, which the watcher ignores
|
||||
tmpDir := filepath.Join(n.SpaceRoot.InternalPath(), TMPDir)
|
||||
if err := os.MkdirAll(tmpDir, 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.CreateTemp(tmpDir, ".upload-*")
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not create temporary upload file")
|
||||
}
|
||||
|
||||
+14
-58
@@ -28,9 +28,7 @@ import (
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/rs/zerolog"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
@@ -49,7 +47,6 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware"
|
||||
"github.com/pkg/errors"
|
||||
@@ -179,13 +176,12 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache.
|
||||
}
|
||||
|
||||
aspects := aspects.Aspects{
|
||||
Lookup: lu,
|
||||
Tree: tp,
|
||||
Permissions: p,
|
||||
EventStream: stream,
|
||||
UserMapper: um,
|
||||
DisableVersioning: o.DisableVersioning,
|
||||
Trashbin: trashbin,
|
||||
Lookup: lu,
|
||||
Tree: tp,
|
||||
Permissions: p,
|
||||
EventStream: stream,
|
||||
UserMapper: um,
|
||||
Trashbin: trashbin,
|
||||
}
|
||||
|
||||
dfs, err := decomposedfs.New(&o.Options, aspects, log)
|
||||
@@ -245,54 +241,14 @@ func (fs *posixFS) WarmupIDCache(root string, assimilate, onlyDirty bool) error
|
||||
return fs.tree.WarmupIDCache(root, assimilate, onlyDirty)
|
||||
}
|
||||
|
||||
// ListUploadSessions returns the upload sessions matching the given filter
|
||||
func (fs *posixFS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
|
||||
return fs.FS.(storage.UploadSessionLister).ListUploadSessions(ctx, filter)
|
||||
// PrepareCreatesNode reports whether the wrapped driver's PrepareUpload creates a
|
||||
// new file's node.
|
||||
func (fs *posixFS) PrepareCreatesNode() bool {
|
||||
nc, ok := fs.FS.(storage.NodeCreator)
|
||||
return ok && nc.PrepareCreatesNode()
|
||||
}
|
||||
|
||||
// ResolveUpload resolves a finished upload to the resource it produced, read as the upload's
|
||||
// executant. It is best effort: if the wrapped storage does not resolve uploads it returns
|
||||
// NotSupported rather than panicking, so the tus finalize falls back to no headers.
|
||||
func (fs *posixFS) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) {
|
||||
r, ok := fs.FS.(storage.UploadSessionResolver)
|
||||
if !ok {
|
||||
return nil, ctx, errtypes.NotSupported("storage does not resolve upload sessions")
|
||||
}
|
||||
return r.ResolveUpload(ctx, info)
|
||||
}
|
||||
|
||||
// UseIn tells the tus upload middleware which extensions it supports.
|
||||
func (fs *posixFS) UseIn(composer *tusd.StoreComposer) {
|
||||
fs.FS.(storage.ComposableFS).UseIn(composer)
|
||||
}
|
||||
|
||||
// NewUpload returns a new tus Upload instance
|
||||
func (fs *posixFS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) {
|
||||
return fs.FS.(tusd.DataStore).NewUpload(ctx, info)
|
||||
}
|
||||
|
||||
// NewUpload returns a new tus Upload instance
|
||||
func (fs *posixFS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) {
|
||||
return fs.FS.(tusd.DataStore).GetUpload(ctx, id)
|
||||
}
|
||||
|
||||
// AsTerminatableUpload returns a TerminatableUpload
|
||||
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
|
||||
// the storage needs to implement AsTerminatableUpload
|
||||
func (fs *posixFS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
}
|
||||
|
||||
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
|
||||
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
|
||||
// the storage needs to implement AsLengthDeclarableUpload
|
||||
func (fs *posixFS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
}
|
||||
|
||||
// AsConcatableUpload returns a ConcatableUpload
|
||||
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
|
||||
// the storage needs to implement AsConcatableUpload
|
||||
func (fs *posixFS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable.
|
||||
func (fs *posixFS) IsOrphaned(ctx context.Context, ref *provider.Reference) bool {
|
||||
return fs.FS.(storage.OrphanChecker).IsOrphaned(ctx, ref)
|
||||
}
|
||||
+45
-9
@@ -41,8 +41,10 @@ import (
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
|
||||
@@ -414,6 +416,9 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
|
||||
return err
|
||||
}
|
||||
|
||||
if storage.SkipTouchPropagation(ctx) {
|
||||
return nil
|
||||
}
|
||||
return t.Propagate(ctx, n, 0)
|
||||
}
|
||||
|
||||
@@ -788,8 +793,12 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
|
||||
|
||||
_, span := tracer.Start(ctx, "InitNewNode")
|
||||
defer span.End()
|
||||
|
||||
// the node may not be cached yet, so its path comes from the parent and the name
|
||||
nodePath := filepath.Join(n.ParentPath(), n.Name)
|
||||
|
||||
// create folder structure (if needed)
|
||||
if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(nodePath), 0700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -800,36 +809,63 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
|
||||
}
|
||||
|
||||
// we also need to touch the actual node file here it stores the mtime of the resource
|
||||
h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600)
|
||||
h, err := os.OpenFile(nodePath, os.O_CREATE|os.O_EXCL, 0600)
|
||||
if err != nil {
|
||||
// the file is not ours: leave it, and its cache entry, alone
|
||||
releaseNewNodeLock(ctx, n, unlock)
|
||||
if os.IsExist(err) {
|
||||
return unlock, errtypes.AlreadyExists(n.InternalPath())
|
||||
// the name only: the message reaches the client, the path is the server's
|
||||
return nil, errtypes.AlreadyExists(n.Name)
|
||||
}
|
||||
return unlock, err
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = h.Close() }()
|
||||
|
||||
// from here the file is ours. A failure removes it and our cache entry before
|
||||
// unlocking, so nothing can open the node in between.
|
||||
fail := func(err error) (metadata.UnlockFunc, error) {
|
||||
if rErr := os.Remove(nodePath); rErr != nil {
|
||||
t.log.Error().Err(rErr).Str("nodeid", n.ID).Msg("could not remove new node file")
|
||||
}
|
||||
if dErr := t.lookup.IDCache.Delete(ctx, n.SpaceID, n.ID); dErr != nil {
|
||||
t.log.Error().Err(dErr).Str("nodeid", n.ID).Msg("could not remove new node from the id cache")
|
||||
}
|
||||
releaseNewNodeLock(ctx, n, unlock)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// cached only now that the name is ours, so a losing upload cannot point it at its id
|
||||
if err := t.lookup.CacheID(ctx, n.SpaceID, n.ID, nodePath); err != nil {
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
// Set known mtime from filesystem to metadata to preven re-assimilation
|
||||
fi, err := h.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return fail(err)
|
||||
}
|
||||
mtime := fi.ModTime()
|
||||
err = n.SetXattrsWithContext(ctx, map[string][]byte{
|
||||
prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)),
|
||||
})
|
||||
if err != nil {
|
||||
t.log.Error().Err(err).Str("path", n.InternalPath()).Msg("could not set mtime attribute on new node")
|
||||
t.log.Error().Err(err).Str("path", nodePath).Msg("could not set mtime attribute on new node")
|
||||
}
|
||||
|
||||
_ = h.Close()
|
||||
|
||||
if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil {
|
||||
return unlock, err
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
// releaseNewNodeLock releases the lock of a node InitNewNode failed to create.
|
||||
func releaseNewNodeLock(ctx context.Context, n *node.Node, unlock metadata.UnlockFunc) {
|
||||
if err := unlock(); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not release new node lock")
|
||||
}
|
||||
}
|
||||
|
||||
// TODO check if node exists?
|
||||
func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) {
|
||||
if t.Ignorer.IsIgnored(filepath.Join(n.ParentPath(), n.Name)) {
|
||||
|
||||
Generated
Vendored
-179
@@ -1,179 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package blobstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Blobstore provides an interface to an s3 compatible blobstore
|
||||
type Blobstore struct {
|
||||
client *minio.Client
|
||||
|
||||
defaultPutOptions Options
|
||||
|
||||
bucket string
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
DisableContentSha256 bool
|
||||
DisableMultipart bool
|
||||
SendContentMd5 bool
|
||||
ConcurrentStreamParts bool
|
||||
NumThreads uint
|
||||
PartSize uint64
|
||||
}
|
||||
|
||||
// New returns a new Blobstore
|
||||
func New(endpoint, region, bucket, accessKey, secretKey string, defaultPutOptions Options) (*Blobstore, error) {
|
||||
u, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to parse s3 endpoint")
|
||||
}
|
||||
|
||||
useSSL := u.Scheme != "http"
|
||||
client, err := minio.New(u.Host, &minio.Options{
|
||||
Region: region,
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: useSSL,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to setup s3 client")
|
||||
}
|
||||
|
||||
return &Blobstore{
|
||||
client: client,
|
||||
bucket: bucket,
|
||||
defaultPutOptions: defaultPutOptions,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Upload stores some data in the blobstore under the given key
|
||||
// TODO(OCISDEV-901): remove once FinishUploadDecomposed is replaced by the upload coordinator (OCISDEV-900).
|
||||
func (bs *Blobstore) Upload(node *node.Node, source string) error {
|
||||
reader, err := os.Open(source)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "can not open source file to upload")
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
_, err = bs.client.PutObject(context.Background(), bs.bucket, bs.Path(node), reader, node.Blobsize, minio.PutObjectOptions{
|
||||
ContentType: "application/octet-stream",
|
||||
SendContentMd5: bs.defaultPutOptions.SendContentMd5,
|
||||
ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts,
|
||||
NumThreads: bs.defaultPutOptions.NumThreads,
|
||||
PartSize: bs.defaultPutOptions.PartSize,
|
||||
DisableMultipart: bs.defaultPutOptions.DisableMultipart,
|
||||
DisableContentSha256: bs.defaultPutOptions.DisableContentSha256,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.Path(node), bs.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadFromReader stores data from a reader directly into S3 without local buffering.
|
||||
func (bs *Blobstore) UploadFromReader(node *node.Node, r io.Reader, size int64) error {
|
||||
_, err := bs.client.PutObject(context.Background(), bs.bucket, bs.Path(node), r, size, minio.PutObjectOptions{
|
||||
ContentType: "application/octet-stream",
|
||||
SendContentMd5: bs.defaultPutOptions.SendContentMd5,
|
||||
ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts,
|
||||
NumThreads: bs.defaultPutOptions.NumThreads,
|
||||
PartSize: bs.defaultPutOptions.PartSize,
|
||||
DisableMultipart: bs.defaultPutOptions.DisableMultipart,
|
||||
DisableContentSha256: bs.defaultPutOptions.DisableContentSha256,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.Path(node), bs.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Download retrieves a blob from the blobstore for reading
|
||||
func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) {
|
||||
reader, err := bs.client.GetObject(context.Background(), bs.bucket, bs.Path(node), minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", bs.Path(node), bs.bucket)
|
||||
}
|
||||
|
||||
stat, err := reader.Stat()
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "blob path: %s", bs.Path(node))
|
||||
}
|
||||
|
||||
if node.Blobsize != stat.Size {
|
||||
return nil, fmt.Errorf("blob has unexpected size. %d bytes expected, got %d bytes", node.Blobsize, stat.Size)
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// Delete deletes a blob from the blobstore
|
||||
func (bs *Blobstore) Delete(node *node.Node) error {
|
||||
err := bs.client.RemoveObject(context.Background(), bs.bucket, bs.Path(node), minio.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", bs.Path(node), bs.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List lists all blobs in the Blobstore
|
||||
func (bs *Blobstore) List() ([]*node.Node, error) {
|
||||
ch := bs.client.ListObjects(context.Background(), bs.bucket, minio.ListObjectsOptions{Recursive: true})
|
||||
|
||||
var err error
|
||||
ids := make([]*node.Node, 0)
|
||||
for oi := range ch {
|
||||
if oi.Err != nil {
|
||||
err = oi.Err
|
||||
continue
|
||||
}
|
||||
spaceid, blobid, _ := strings.Cut(oi.Key, "/")
|
||||
ids = append(ids, &node.Node{
|
||||
SpaceID: strings.ReplaceAll(spaceid, "/", ""),
|
||||
BlobID: strings.ReplaceAll(blobid, "/", ""),
|
||||
})
|
||||
}
|
||||
return ids, err
|
||||
}
|
||||
|
||||
func (bs *Blobstore) Path(node *node.Node) string {
|
||||
// https://aws.amazon.com/de/premiumsupport/knowledge-center/s3-prefix-nested-folders-difference/
|
||||
// Prefixes are used to partion a bucket. A prefix is everything except the filename.
|
||||
// For a file `BucketName/foo/bar/lorem.ipsum`, `BucketName/foo/bar/` is the prefix.
|
||||
// There are request limits per prefix, therefore you should have many prefixes.
|
||||
// There are no limits to prefixes per bucket, so in general it's better to have more then less.
|
||||
//
|
||||
// Since the spaceID is always the same for a space, we don't need to pathify that, because it would
|
||||
// not yield any performance gains
|
||||
return filepath.Clean(filepath.Join(node.SpaceID, lookup.Pathify(node.BlobID, 4, 2)))
|
||||
}
|
||||
-93
@@ -1,93 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package s3ng
|
||||
|
||||
import (
|
||||
"github.com/go-viper/mapstructure/v2"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// Option defines a single option function.
|
||||
type Option func(o *Options)
|
||||
|
||||
// Options defines the available options for this package.
|
||||
type Options struct {
|
||||
|
||||
// Endpoint of the s3 blobstore
|
||||
S3Endpoint string `mapstructure:"s3.endpoint"`
|
||||
|
||||
// Region of the s3 blobstore
|
||||
S3Region string `mapstructure:"s3.region"`
|
||||
|
||||
// Bucket of the s3 blobstore
|
||||
S3Bucket string `mapstructure:"s3.bucket"`
|
||||
|
||||
// Access key for the s3 blobstore
|
||||
S3AccessKey string `mapstructure:"s3.access_key"`
|
||||
|
||||
// Secret key for the s3 blobstore
|
||||
S3SecretKey string `mapstructure:"s3.secret_key"`
|
||||
|
||||
// disable sending content sha256
|
||||
DisableContentSha256 bool `mapstructure:"s3.disable_content_sha254"`
|
||||
|
||||
// disable multipart uploads
|
||||
DisableMultipart bool `mapstructure:"s3.disable_multipart"`
|
||||
|
||||
// enable sending content md5, defaults to true if unset
|
||||
SendContentMd5 bool `mapstructure:"s3.send_content_md5"`
|
||||
|
||||
// use concurrent stream parts
|
||||
ConcurrentStreamParts bool `mapstructure:"s3.concurrent_stream_parts"`
|
||||
|
||||
// number of concurrent uploads
|
||||
NumThreads uint `mapstructure:"s3.num_threads"`
|
||||
|
||||
// part size for concurrent uploads
|
||||
PartSize uint64 `mapstructure:"s3.part_size"`
|
||||
}
|
||||
|
||||
// S3ConfigComplete return true if all required s3 fields are set
|
||||
func (o *Options) S3ConfigComplete() bool {
|
||||
return o.S3Endpoint != "" &&
|
||||
o.S3Region != "" &&
|
||||
o.S3Bucket != "" &&
|
||||
o.S3AccessKey != "" &&
|
||||
o.S3SecretKey != ""
|
||||
}
|
||||
|
||||
func parseConfig(m map[string]interface{}) (*Options, error) {
|
||||
o := &Options{}
|
||||
if err := mapstructure.Decode(m, o); err != nil {
|
||||
err = errors.Wrap(err, "error decoding conf")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if unset we set these defaults
|
||||
if m["s3.send_content_md5"] == nil {
|
||||
o.SendContentMd5 = true
|
||||
}
|
||||
if m["s3.concurrent_stream_parts"] == nil {
|
||||
o.ConcurrentStreamParts = true
|
||||
}
|
||||
if m["s3.num_threads"] == nil {
|
||||
o.NumThreads = 4
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
-63
@@ -1,63 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package s3ng
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registry.Register("s3ng", New)
|
||||
}
|
||||
|
||||
// New returns an implementation to of the storage.FS interface that talk to
|
||||
// a local filesystem.
|
||||
func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) {
|
||||
o, err := parseConfig(m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !o.S3ConfigComplete() {
|
||||
return nil, fmt.Errorf("S3 configuration incomplete")
|
||||
}
|
||||
|
||||
defaultPutOptions := blobstore.Options{
|
||||
DisableContentSha256: o.DisableContentSha256,
|
||||
DisableMultipart: o.DisableMultipart,
|
||||
SendContentMd5: o.SendContentMd5,
|
||||
ConcurrentStreamParts: o.ConcurrentStreamParts,
|
||||
NumThreads: o.NumThreads,
|
||||
PartSize: o.PartSize,
|
||||
}
|
||||
|
||||
bs, err := blobstore.New(o.S3Endpoint, o.S3Region, o.S3Bucket, o.S3AccessKey, o.S3SecretKey, defaultPutOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return decomposedfs.NewDefault(m, bs, stream, log)
|
||||
}
|
||||
Generated
Vendored
+6
-7
@@ -28,11 +28,10 @@ import (
|
||||
|
||||
// Aspects holds dependencies for handling aspects of the decomposedfs
|
||||
type Aspects struct {
|
||||
Lookup node.PathLookup
|
||||
Tree node.Tree
|
||||
Trashbin trashbin.Trashbin
|
||||
Permissions permissions.Permissions
|
||||
EventStream events.Stream
|
||||
DisableVersioning bool
|
||||
UserMapper usermapper.Mapper
|
||||
Lookup node.PathLookup
|
||||
Tree node.Tree
|
||||
Trashbin trashbin.Trashbin
|
||||
Permissions permissions.Permissions
|
||||
EventStream events.Stream
|
||||
UserMapper usermapper.Mapper
|
||||
}
|
||||
Generated
Vendored
+144
-10
@@ -19,6 +19,8 @@ package decomposedfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
@@ -27,6 +29,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
)
|
||||
@@ -62,20 +65,24 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere
|
||||
return n.RemoveXattr(ctx, prefixes.StatusPrefix)
|
||||
}
|
||||
|
||||
if n.IsProcessing(ctx) {
|
||||
return errtypes.ResourceProcessing(ref.String())
|
||||
}
|
||||
// a newer upload takes over the mark, the last writer wins
|
||||
return n.SetXattrsWithContext(ctx, node.Attributes{
|
||||
prefixes.StatusPrefix: []byte(node.ProcessingStatus + sessionID),
|
||||
})
|
||||
}
|
||||
|
||||
// CommitUpload writes the staged bytes from source to the resource at ref.
|
||||
// sessionID is used to identify the correct blob slot prepared before postprocessing.
|
||||
// sessionID identifies the upload that PrepareUpload wrote onto the node.
|
||||
// Caller owns source.Body and must close it after CommitUpload returns.
|
||||
//
|
||||
// The bytes are written against the real node, so drivers that keep the content
|
||||
// in the node itself (posix) can replace it atomically. If a newer upload took
|
||||
// over the node in the meantime, the bytes go to the revision that upload created
|
||||
// for this session instead. Without such a revision (versioning disabled) they are
|
||||
// discarded: nothing references them anymore.
|
||||
func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source storage.UploadSource) error {
|
||||
if source.Body == nil {
|
||||
return errtypes.BadRequest("Decomposedfs: source body is nil")
|
||||
if source.Body == nil && source.Path == "" {
|
||||
return errtypes.BadRequest("Decomposedfs: source has neither body nor path")
|
||||
}
|
||||
if sessionID == "" {
|
||||
return errtypes.BadRequest("Decomposedfs: sessionID is empty")
|
||||
@@ -89,17 +96,75 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc
|
||||
if !n.Exists {
|
||||
return errtypes.NotFound(ref.String())
|
||||
}
|
||||
n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
blobNode := node.New(n.SpaceID, n.ID, "", "", source.Length, sessionID,
|
||||
provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu)
|
||||
unlock, err := fs.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not lock node")
|
||||
}
|
||||
defer func() {
|
||||
_ = unlock()
|
||||
}()
|
||||
|
||||
if err := fs.tp.WriteBlobFromReader(blobNode, source.Body, source.Length); err != nil {
|
||||
if derr := fs.tp.DeleteBlob(blobNode); derr != nil {
|
||||
// re-read the blob id while holding the lock, a newer upload may have taken over.
|
||||
// read the attribute itself: the posix lookup's ReadBlobIDAndSizeAttr reports no blob id
|
||||
blobID, err := fs.lu.MetadataBackend().Get(ctx, n, prefixes.BlobIDAttr)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not read blob id")
|
||||
}
|
||||
n.BlobID = string(blobID)
|
||||
|
||||
target := n
|
||||
if n.BlobID != sessionID {
|
||||
revision, revisionUnlock, err := fs.uploadRevision(ctx, n, sessionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if revision == nil {
|
||||
appctx.GetLogger(ctx).Info().Str("nodeid", n.ID).Str("sessionid", sessionID).Msg("upload was superseded and has no revision, discarding its bytes")
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
_ = revisionUnlock()
|
||||
}()
|
||||
target = revision
|
||||
}
|
||||
|
||||
if source.Path != "" {
|
||||
err = fs.tp.WriteBlob(target, source.Path)
|
||||
} else {
|
||||
err = fs.tp.WriteBlobFromReader(target, source.Body, source.Length)
|
||||
}
|
||||
if err != nil {
|
||||
if derr := fs.tp.DeleteBlob(target); derr != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(derr).Str("nodeid", n.ID).Str("blobid", sessionID).Msg("could not clean up orphaned blob after failed write")
|
||||
}
|
||||
return errors.Wrap(err, "Decomposedfs: failed to write blob")
|
||||
}
|
||||
|
||||
// on the node that holds the content, not the session: the session is deleted when the
|
||||
// upload finishes
|
||||
if !source.ScanDate.IsZero() {
|
||||
if err := target.SetScanData(ctx, source.ScanResult, source.ScanDate); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", target.ID).Msg("could not set scan results")
|
||||
}
|
||||
}
|
||||
|
||||
if target != n {
|
||||
// the revision kept this upload's processing mark for a rollback of the newer
|
||||
// upload; this upload is done, so restoring it must not mark the node again
|
||||
if err := target.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", target.ID).Msg("could not remove the processing mark from the revision")
|
||||
}
|
||||
} else {
|
||||
// revisions kept only for a rollback of this upload served their purpose; versions
|
||||
// stay, also the ones created before versioning was disabled
|
||||
fs.removeRevisions(ctx, n)
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if p, err := n.Parent(ctx); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not read parent for etag propagation")
|
||||
@@ -112,3 +177,72 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeRevisions deletes the revisions of n that were kept only for a rollback, with their
|
||||
// blobs, while the caller holds the node lock: the ones of uploads that were rolled back,
|
||||
// and the snapshots taken with versioning disabled.
|
||||
func (fs *Decomposedfs) removeRevisions(ctx context.Context, n *node.Node) {
|
||||
revPrefix := fs.lu.VersionPath(n.SpaceID, n.ID, "")
|
||||
revisions, err := filepath.Glob(fs.lu.VersionPath(n.SpaceID, n.ID, "*"))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not list revisions")
|
||||
return
|
||||
}
|
||||
for _, rev := range revisions {
|
||||
if strings.HasSuffix(rev, ".mpk") || strings.HasSuffix(rev, ".mlock") {
|
||||
continue
|
||||
}
|
||||
revisionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+strings.TrimPrefix(rev, revPrefix), fs.lu)
|
||||
if !fs.keptForRollback(ctx, revisionNode) {
|
||||
continue
|
||||
}
|
||||
blobID, err := fs.lu.MetadataBackend().Get(ctx, revisionNode, prefixes.BlobIDAttr)
|
||||
if err == nil && len(blobID) > 0 && string(blobID) != n.BlobID {
|
||||
blobNode := node.New(n.SpaceID, revisionNode.ID, "", "", 0, string(blobID), provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu)
|
||||
if err := fs.tp.DeleteBlob(blobNode); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("revisionid", revisionNode.ID).Msg("could not delete the blob of a superseded revision")
|
||||
}
|
||||
}
|
||||
removeRevision(ctx, fs.lu, revisionNode)
|
||||
}
|
||||
}
|
||||
|
||||
// keptForRollback reports whether a revision exists only for a rollback, rather than as a
|
||||
// version.
|
||||
func (fs *Decomposedfs) keptForRollback(ctx context.Context, revision metadata.MetadataNode) bool {
|
||||
for _, attr := range []string{purgeOnRestoreAttr, uploadSnapshotAttr} {
|
||||
if _, err := fs.lu.MetadataBackend().Get(ctx, revision, attr); err == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// uploadRevision returns the locked revision of n that holds the upload sessionID,
|
||||
// or nil if there is none.
|
||||
func (fs *Decomposedfs) uploadRevision(ctx context.Context, n *node.Node, sessionID string) (*node.Node, metadata.UnlockFunc, error) {
|
||||
revPrefix := fs.lu.VersionPath(n.SpaceID, n.ID, "")
|
||||
revisions, err := filepath.Glob(fs.lu.VersionPath(n.SpaceID, n.ID, "*"))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, rev := range revisions {
|
||||
if strings.HasSuffix(rev, ".mpk") || strings.HasSuffix(rev, ".mlock") {
|
||||
continue
|
||||
}
|
||||
revisionID := n.ID + node.RevisionIDDelimiter + strings.TrimPrefix(rev, revPrefix)
|
||||
blobID, err := fs.lu.MetadataBackend().Get(ctx, node.NewBaseNode(n.SpaceID, revisionID, fs.lu), prefixes.BlobIDAttr)
|
||||
if err != nil || string(blobID) != sessionID {
|
||||
continue
|
||||
}
|
||||
revision, unlock, err := node.LockAndReadNode(ctx, fs.lu, n.SpaceID, revisionID, "", false, n.SpaceRoot, false)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// ReadNode caches the attributes of the node, not of the revision. The posix
|
||||
// blobstore copies them onto the file it writes, so read the revision's own.
|
||||
revision.ResetXattrsCache()
|
||||
return revision, unlock, nil
|
||||
}
|
||||
return nil, nil, nil
|
||||
}
|
||||
Generated
Vendored
+34
-373
@@ -36,18 +36,17 @@ import (
|
||||
"github.com/jellydator/ttlcache/v2"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
microstore "go-micro.dev/v4/store"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
|
||||
@@ -60,9 +59,7 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
@@ -80,11 +77,6 @@ var (
|
||||
tracer trace.Tracer
|
||||
|
||||
_registeredEvents = []events.Unmarshaller{
|
||||
events.PostprocessingFinished{},
|
||||
events.PostprocessingStepFinished{},
|
||||
events.RestartPostprocessing{},
|
||||
events.StartPostprocessingStep{},
|
||||
events.CleanUpload{},
|
||||
events.DeleteRevision{},
|
||||
}
|
||||
)
|
||||
@@ -93,41 +85,19 @@ func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs")
|
||||
}
|
||||
|
||||
// Session is the interface that DecomposedfsSession implements. By combining tus.Upload,
|
||||
// storage.UploadSession and custom functions we can reuse the same struct throughout
|
||||
// the whole upload lifecycle.
|
||||
//
|
||||
// Some functions that are only used by decomposedfs are not yet part of this interface.
|
||||
// They might be added after more refactoring.
|
||||
type Session interface {
|
||||
tusd.Upload
|
||||
storage.UploadSession
|
||||
upload.Session
|
||||
LockID() string
|
||||
}
|
||||
|
||||
type IDCachingTree interface {
|
||||
WarmupIDCache(root string, assimilate, onlyDirty bool) error
|
||||
}
|
||||
|
||||
type SessionStore interface {
|
||||
New(ctx context.Context) *upload.DecomposedFsSession
|
||||
SessionFromInfo(info tusd.FileInfo) *upload.DecomposedFsSession
|
||||
List(ctx context.Context) ([]*upload.DecomposedFsSession, error)
|
||||
Get(ctx context.Context, id string) (*upload.DecomposedFsSession, error)
|
||||
}
|
||||
|
||||
// Decomposedfs provides the base for decomposed filesystem implementations
|
||||
type Decomposedfs struct {
|
||||
lu node.PathLookup
|
||||
tp node.Tree
|
||||
trashbin trashbin.Trashbin
|
||||
o *options.Options
|
||||
p permissions.Permissions
|
||||
um usermapper.Mapper
|
||||
chunkHandler *chunking.ChunkHandler
|
||||
stream events.Stream
|
||||
sessionStore SessionStore
|
||||
lu node.PathLookup
|
||||
tp node.Tree
|
||||
trashbin trashbin.Trashbin
|
||||
o *options.Options
|
||||
p permissions.Permissions
|
||||
um usermapper.Mapper
|
||||
stream events.Stream
|
||||
|
||||
UserCache *ttlcache.Cache
|
||||
userSpaceIndex *spaceidindex.Index
|
||||
@@ -184,12 +154,11 @@ func NewDefault(m map[string]interface{}, bs node.Blobstore, es events.Stream, l
|
||||
), log)
|
||||
|
||||
aspects := aspects.Aspects{
|
||||
Lookup: lu,
|
||||
Tree: tp,
|
||||
Permissions: p,
|
||||
EventStream: es,
|
||||
DisableVersioning: o.DisableVersioning,
|
||||
Trashbin: &DecomposedfsTrashbin{},
|
||||
Lookup: lu,
|
||||
Tree: tp,
|
||||
Permissions: p,
|
||||
EventStream: es,
|
||||
Trashbin: &DecomposedfsTrashbin{},
|
||||
}
|
||||
|
||||
return New(o, aspects, log)
|
||||
@@ -251,7 +220,6 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor
|
||||
o: o,
|
||||
p: aspects.Permissions,
|
||||
um: aspects.UserMapper,
|
||||
chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")),
|
||||
stream: aspects.EventStream,
|
||||
UserCache: ttlcache.NewCache(),
|
||||
userSpaceIndex: userSpaceIndex,
|
||||
@@ -260,7 +228,6 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor
|
||||
spaceTypeIndex: spaceTypeIndex,
|
||||
log: log,
|
||||
}
|
||||
fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens, log)
|
||||
if err = fs.trashbin.Setup(fs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -271,7 +238,9 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor
|
||||
return nil, errors.New("need nats for async file processing")
|
||||
}
|
||||
|
||||
ch, err := events.Consume(fs.stream, "dcfs", _registeredEvents...)
|
||||
// a group of its own: the upload coordinator holds the "dcfs" group, and one
|
||||
// group gets one copy of each event
|
||||
ch, err := events.Consume(fs.stream, "dcfs-revisions", _registeredEvents...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -281,201 +250,32 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor
|
||||
}
|
||||
|
||||
for i := 0; i < o.Events.NumConsumers; i++ {
|
||||
go fs.Postprocessing(ch)
|
||||
go fs.ConsumeRevisionEvents(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Postprocessing starts the postprocessing result collector
|
||||
func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) {
|
||||
// ConsumeRevisionEvents starts the revision event collector. Upload postprocessing
|
||||
// results are consumed by the upload coordinator.
|
||||
func (fs *Decomposedfs) ConsumeRevisionEvents(ch <-chan events.Event) {
|
||||
for event := range ch {
|
||||
// derive a per-event context from the event's W3C trace context so the
|
||||
// postprocessing work is linked to the originating request
|
||||
// work is linked to the originating request
|
||||
ctx := event.GetTraceContext(context.Background())
|
||||
ctx = ctxpkg.ContextSetInitiator(ctx, event.InitiatorID)
|
||||
fs.handlePostprocessingEvent(ctx, event)
|
||||
fs.handleRevisionEvent(ctx, event)
|
||||
}
|
||||
}
|
||||
|
||||
// handlePostprocessingEvent processes a single postprocessing event under a span
|
||||
// scoped to that event's trace context.
|
||||
func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event events.Event) {
|
||||
ctx, span := tracer.Start(ctx, "Postprocessing/"+event.Type)
|
||||
// handleRevisionEvent processes a single revision event under a span scoped to
|
||||
// that event's trace context.
|
||||
func (fs *Decomposedfs) handleRevisionEvent(ctx context.Context, event events.Event) {
|
||||
ctx, span := tracer.Start(ctx, "ConsumeRevisionEvents/"+event.Type)
|
||||
defer span.End()
|
||||
log := fs.log
|
||||
switch ev := event.Event.(type) {
|
||||
case events.PostprocessingFinished:
|
||||
sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger()
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
return // NOTE: since we can't get the upload, we can't delete the blob
|
||||
}
|
||||
|
||||
n, err := session.Node(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("could not read node")
|
||||
return
|
||||
}
|
||||
sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger()
|
||||
if !n.Exists {
|
||||
sublog.Debug().Msg("node no longer exists")
|
||||
session.Cleanup(false, true, true, false)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
failed bool
|
||||
revertNodeMetadata bool
|
||||
keepUpload bool
|
||||
)
|
||||
unmarkPostprocessing := true
|
||||
|
||||
switch ev.Outcome {
|
||||
default:
|
||||
sublog.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting")
|
||||
fallthrough
|
||||
case events.PPOutcomeAbort:
|
||||
failed = true
|
||||
revertNodeMetadata = true
|
||||
keepUpload = true
|
||||
metrics.UploadSessionsAborted.Inc()
|
||||
case events.PPOutcomeContinue:
|
||||
if err := session.Finalize(ctx); err != nil {
|
||||
sublog.Error().Err(err).Msg("could not finalize upload")
|
||||
failed = true
|
||||
revertNodeMetadata = false
|
||||
keepUpload = true
|
||||
// keep postprocessing status so the upload is not deleted during housekeeping
|
||||
unmarkPostprocessing = false
|
||||
} else {
|
||||
metrics.UploadSessionsFinalized.Inc()
|
||||
}
|
||||
case events.PPOutcomeDelete:
|
||||
failed = true
|
||||
revertNodeMetadata = true
|
||||
metrics.UploadSessionsDeleted.Inc()
|
||||
}
|
||||
|
||||
getParent := func() *node.Node {
|
||||
p, err := n.Parent(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("could not read parent")
|
||||
return nil
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
var parentId *provider.ResourceId
|
||||
if failed {
|
||||
// if no other upload session is in progress (processing id != session id) or has finished (processing id == "")
|
||||
latestSession, err := n.ProcessingID(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("reading node for session failed")
|
||||
}
|
||||
if latestSession == session.ID() {
|
||||
// propagate reverted sizeDiff after failed postprocessing
|
||||
if err := fs.tp.Propagate(ctx, n, -session.SizeDiff()); err != nil {
|
||||
sublog.Error().Err(err).Msg("could not propagate tree size change")
|
||||
}
|
||||
}
|
||||
} else if p := getParent(); p != nil {
|
||||
parentId = &provider.ResourceId{
|
||||
StorageId: session.ProviderID(),
|
||||
SpaceId: session.SpaceID(),
|
||||
OpaqueId: p.ID,
|
||||
}
|
||||
// update parent tmtime to propagate etag change after successful postprocessing
|
||||
_ = p.SetTMTime(ctx, &now)
|
||||
if err := fs.tp.Propagate(ctx, p, 0); err != nil {
|
||||
sublog.Error().Err(err).Msg("could not propagate etag change")
|
||||
}
|
||||
}
|
||||
|
||||
session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing)
|
||||
|
||||
var isVersion bool
|
||||
if session.NodeExists() {
|
||||
info, err := session.GetInfo(ctx)
|
||||
if err == nil && info.MetaData["versionID"] != "" {
|
||||
isVersion = true
|
||||
}
|
||||
}
|
||||
|
||||
if err := events.Publish(
|
||||
ctx,
|
||||
fs.stream,
|
||||
events.UploadReady{
|
||||
UploadID: ev.UploadID,
|
||||
Failed: failed,
|
||||
ExecutingUser: ev.ExecutingUser,
|
||||
Filename: ev.Filename,
|
||||
FileRef: &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: session.ProviderID(),
|
||||
SpaceId: session.SpaceID(),
|
||||
OpaqueId: session.SpaceID(),
|
||||
},
|
||||
Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())),
|
||||
},
|
||||
ResourceID: &provider.ResourceId{
|
||||
StorageId: session.ProviderID(),
|
||||
SpaceId: session.SpaceID(),
|
||||
OpaqueId: session.NodeID(),
|
||||
},
|
||||
ParentID: parentId,
|
||||
Timestamp: utils.TimeToTS(now),
|
||||
SpaceOwner: n.SpaceOwnerOrManager(ctx),
|
||||
IsVersion: isVersion,
|
||||
ImpersonatingUser: ev.ImpersonatingUser,
|
||||
},
|
||||
); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to publish UploadReady event")
|
||||
}
|
||||
case events.RestartPostprocessing:
|
||||
sublog := log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger()
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
return
|
||||
}
|
||||
n, err := session.Node(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("could not read node")
|
||||
return
|
||||
}
|
||||
sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger()
|
||||
s, err := session.URL(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("could not create url")
|
||||
return
|
||||
}
|
||||
|
||||
metrics.UploadSessionsRestarted.Inc()
|
||||
|
||||
// restart postprocessing
|
||||
if err := events.Publish(ctx, fs.stream, events.BytesReceived{
|
||||
UploadID: session.ID(),
|
||||
URL: s,
|
||||
SpaceOwner: n.SpaceOwnerOrManager(ctx),
|
||||
ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead?
|
||||
ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID},
|
||||
Filename: session.Filename(),
|
||||
Filesize: uint64(session.Size()),
|
||||
}); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to publish BytesReceived event")
|
||||
}
|
||||
case events.CleanUpload:
|
||||
sublog := log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger()
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
return // NOTE: since we can't get the upload, we can't delete the blob
|
||||
}
|
||||
session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true)
|
||||
case events.DeleteRevision:
|
||||
sublog := log.With().Str("event", "DeleteRevision").Interface("nodeid", ev.ResourceID).Logger()
|
||||
n, err := fs.lu.NodeFromID(ctx, ev.ResourceID)
|
||||
@@ -501,152 +301,6 @@ func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event eve
|
||||
if deleteErr != nil {
|
||||
sublog.Error().Err(deleteErr).Msg("Failed to delete revision")
|
||||
}
|
||||
case events.StartPostprocessingStep:
|
||||
sublog := log.With().Str("event", "StartPostprocessingStep").Str("uploadid", ev.UploadID).Logger()
|
||||
if ev.UploadID == "" {
|
||||
sublog.Error().Msg("UploadID is empty, cannot start postprocessing step")
|
||||
return
|
||||
}
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
return
|
||||
}
|
||||
session.SetStatus(upload.SessionStatusProcessing, "started postprocessing step: "+string(ev.StepToStart))
|
||||
err = session.Persist(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to persist upload session after starting postprocessing step")
|
||||
}
|
||||
case events.PostprocessingStepFinished:
|
||||
sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger()
|
||||
if ev.FinishedStep != events.PPStepAntivirus {
|
||||
// atm we are only interested in antivirus results
|
||||
return
|
||||
}
|
||||
|
||||
res := ev.Result.(events.VirusscanResult)
|
||||
if res.ErrorMsg != "" {
|
||||
// scan failed somehow
|
||||
// Should we handle this here?
|
||||
return
|
||||
}
|
||||
sublog = log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger()
|
||||
|
||||
var n *node.Node
|
||||
switch ev.UploadID {
|
||||
case "":
|
||||
// uploadid is empty -> this was an on-demand scan
|
||||
/* ON DEMAND SCANNING NOT SUPPORTED ATM
|
||||
ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser)
|
||||
ref := &provider.Reference{ResourceId: ev.ResourceID}
|
||||
|
||||
no, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan")
|
||||
continue
|
||||
|
||||
}
|
||||
n = no
|
||||
if ev.Outcome == events.PPOutcomeDelete {
|
||||
// antivir wants us to delete the file. We must obey and need to
|
||||
|
||||
// check if there a previous versions existing
|
||||
revs, err := fs.ListRevisions(ctx, ref)
|
||||
if len(revs) == 0 {
|
||||
if err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file")
|
||||
}
|
||||
|
||||
// no versions -> trash file
|
||||
err := fs.Delete(ctx, ref)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource")
|
||||
continue
|
||||
}
|
||||
|
||||
// now purge it from the recycle bin
|
||||
if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash")
|
||||
}
|
||||
|
||||
// remove cache entry in gateway
|
||||
fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID})
|
||||
continue
|
||||
}
|
||||
|
||||
// we have versions - find the newest
|
||||
versions := make(map[uint64]string) // remember all versions - we need them later
|
||||
var nv uint64
|
||||
for _, v := range revs {
|
||||
versions[v.Mtime] = v.Key
|
||||
if v.Mtime > nv {
|
||||
nv = v.Mtime
|
||||
}
|
||||
}
|
||||
|
||||
// restore newest version
|
||||
if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision")
|
||||
continue
|
||||
}
|
||||
|
||||
// now find infected version
|
||||
revs, err = fs.ListRevisions(ctx, ref)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore")
|
||||
}
|
||||
|
||||
for _, v := range revs {
|
||||
// we looking for a version that was previously not there
|
||||
if _, ok := versions[v.Mtime]; ok {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil {
|
||||
log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision")
|
||||
}
|
||||
}
|
||||
|
||||
// remove cache entry in gateway
|
||||
fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID})
|
||||
continue
|
||||
}
|
||||
*/
|
||||
default:
|
||||
// uploadid is not empty -> this is an async upload
|
||||
session, err := fs.sessionStore.Get(ctx, ev.UploadID)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get upload")
|
||||
return
|
||||
}
|
||||
|
||||
n, err = session.Node(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to get node after scan")
|
||||
return
|
||||
}
|
||||
sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger()
|
||||
|
||||
session.SetScanData(res.Description, res.Scandate)
|
||||
var status string
|
||||
switch ev.Outcome {
|
||||
case events.PPOutcomeAbort, events.PPOutcomeDelete:
|
||||
status = upload.SessionStatusFailed
|
||||
default:
|
||||
status = upload.SessionStatusProcessing
|
||||
}
|
||||
session.SetStatus(status, "finished postprocessing step: "+string(ev.FinishedStep))
|
||||
if err := session.Persist(ctx); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to persist scan results")
|
||||
}
|
||||
}
|
||||
|
||||
if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil {
|
||||
sublog.Error().Err(err).Msg("Failed to set scan results")
|
||||
return
|
||||
}
|
||||
|
||||
metrics.UploadSessionsScanned.Inc()
|
||||
default:
|
||||
log.Error().Interface("event", ev).Msg("Unknown event")
|
||||
}
|
||||
@@ -664,6 +318,12 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Capabilities declares decomposedfs fully writeable, stated explicitly so it is
|
||||
// a decision here and not an inherited default.
|
||||
func (fs *Decomposedfs) Capabilities(_ context.Context) storage.Capabilities {
|
||||
return storage.FullCapabilities()
|
||||
}
|
||||
|
||||
// GetQuota returns the quota available
|
||||
// TODO Document in the cs3 should we return quota or free space?
|
||||
func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, remaining uint64, err error) {
|
||||
@@ -1113,7 +773,8 @@ func (fs *Decomposedfs) ListFolder(ctx context.Context, ref *provider.Reference,
|
||||
for i := 0; i < numWorkers; i++ {
|
||||
g.Go(func() error {
|
||||
for child := range work {
|
||||
np := rp
|
||||
// copy the parent's permissions: the workers must not share them
|
||||
np := proto.Clone(rp).(*provider.ResourcePermissions)
|
||||
// add this childs permissions
|
||||
pset, _ := child.PermissionSet(ctx)
|
||||
node.AddPermissions(np, pset)
|
||||
|
||||
Generated
Vendored
+20
-4
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/google/renameio/v2"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
@@ -30,6 +29,7 @@ type HybridBackend struct {
|
||||
offloadLimit int
|
||||
metaCache cache.FileMetadataCache
|
||||
metadataPathFunc MetadataPathFunc
|
||||
metrics *backendMetrics
|
||||
}
|
||||
|
||||
// NewMessageBackend returns a new HybridBackend instance
|
||||
@@ -38,6 +38,7 @@ func NewHybridBackend(offloadLimit int, metadataPathFunc MetadataPathFunc, o cac
|
||||
offloadLimit: offloadLimit,
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
metadataPathFunc: metadataPathFunc,
|
||||
metrics: newBackendMetrics("hybrid"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +47,7 @@ func (HybridBackend) Name() string { return "hybrid" }
|
||||
|
||||
// IdentifyPath returns the space id, node id and mtime of a file
|
||||
func (b HybridBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) {
|
||||
b.metrics.reads.Inc()
|
||||
spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr)
|
||||
if err != nil {
|
||||
if IsNotExist(err) {
|
||||
@@ -66,10 +68,13 @@ func (b HybridBackend) IdentifyPath(_ context.Context, path string) (string, str
|
||||
func (b HybridBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
if err == nil && len(attribs[key]) > 0 {
|
||||
found := err == nil && len(attribs[key]) > 0
|
||||
b.metrics.cacheLookup(found)
|
||||
if found {
|
||||
return attribs[key], err
|
||||
}
|
||||
|
||||
b.metrics.reads.Inc()
|
||||
if isOffloadingAttribute(key) {
|
||||
// check if key is offloaded
|
||||
offloaded, err := xattr.Get(n.InternalPath(), metadataOffloadedAttr())
|
||||
@@ -122,6 +127,7 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by
|
||||
attribs := map[string][]byte{}
|
||||
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
b.metrics.cacheLookup(err == nil)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
}
|
||||
@@ -138,6 +144,7 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by
|
||||
}
|
||||
|
||||
func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipOffloaded bool) (map[string][]byte, error) {
|
||||
b.metrics.reads.Inc()
|
||||
attribs := map[string][]byte{}
|
||||
attrNames, err := b.list(ctx, n)
|
||||
if err != nil {
|
||||
@@ -217,6 +224,8 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
defer func() { _ = unlock() }()
|
||||
}
|
||||
|
||||
// counts once together with reading the offloaded metadata below
|
||||
b.metrics.reads.Inc()
|
||||
offloadAttr, err := xattr.Get(path, metadataOffloadedAttr())
|
||||
offloaded := err == nil && string(offloadAttr) == "1"
|
||||
|
||||
@@ -254,6 +263,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
}
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
if offloaded {
|
||||
metaPath := b.MetadataPath(n)
|
||||
var msgBytes []byte
|
||||
@@ -340,6 +350,7 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro
|
||||
return err
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
err = renameio.WriteFile(b.MetadataPath(n), d, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -376,6 +387,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e
|
||||
}
|
||||
|
||||
if isOffloadingAttribute(key) {
|
||||
b.metrics.reads.Inc()
|
||||
offloadAttr, err := xattr.Get(path, metadataOffloadedAttr())
|
||||
offloaded := err == nil && string(offloadAttr) == "1"
|
||||
if offloaded {
|
||||
@@ -412,12 +424,14 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e
|
||||
return err
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
err = renameio.WriteFile(b.MetadataPath(n), d, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// remove from xattrs
|
||||
b.metrics.writes.Inc()
|
||||
err := xattr.Remove(path, key)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -425,6 +439,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e
|
||||
}
|
||||
} else {
|
||||
// remove from xattrs
|
||||
b.metrics.writes.Inc()
|
||||
err := xattr.Remove(path, key)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -452,6 +467,7 @@ func (b HybridBackend) Purge(ctx context.Context, n MetadataNode) error {
|
||||
|
||||
attribs, err := b.getAll(ctx, n, false)
|
||||
if err == nil {
|
||||
b.metrics.writes.Inc()
|
||||
for attr := range attribs {
|
||||
if strings.HasPrefix(attr, prefixes.OcPrefix) {
|
||||
_ = xattr.Remove(path, attr)
|
||||
@@ -497,7 +513,7 @@ func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
}
|
||||
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
// The lock file's parent directories don't exist yet (or anymore).
|
||||
@@ -517,7 +533,7 @@ func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
mlock, err = lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
mlock, err = lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Generated
Vendored
+41
@@ -0,0 +1,41 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
)
|
||||
|
||||
// lockFile opens and locks the lock file of a node like lockedfile.OpenFile, recording in the metrics how long
|
||||
// it waits for the lock and, once the returned lock is closed, how long the lock has been held
|
||||
func lockFile(n MetadataNode, path string, flag int, m *backendMetrics) (*heldLock, error) {
|
||||
lm := m.locks(n)
|
||||
|
||||
m.locksWaiting.Inc()
|
||||
start := time.Now()
|
||||
f, err := lockedfile.OpenFile(path, flag, 0600)
|
||||
m.locksWaiting.Dec()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
locked := time.Now()
|
||||
lm.wait.Observe(locked.Sub(start).Seconds())
|
||||
return &heldLock{f: f, held: lm.held, since: locked}, nil
|
||||
}
|
||||
|
||||
// heldLock is a locked lock file
|
||||
type heldLock struct {
|
||||
f *lockedfile.File
|
||||
held prometheus.Observer
|
||||
since time.Time
|
||||
}
|
||||
|
||||
// Close unlocks and closes the lock file
|
||||
func (l *heldLock) Close() error {
|
||||
l.held.Observe(time.Since(l.since).Seconds())
|
||||
return l.f.Close()
|
||||
}
|
||||
Generated
Vendored
+10
-2
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
)
|
||||
@@ -41,12 +40,14 @@ import (
|
||||
// MessagePackBackend persists the attributes in messagepack format inside the file
|
||||
type MessagePackBackend struct {
|
||||
metaCache cache.FileMetadataCache
|
||||
metrics *backendMetrics
|
||||
}
|
||||
|
||||
// NewMessagePackBackend returns a new MessagePackBackend instance
|
||||
func NewMessagePackBackend(o cache.Config) MessagePackBackend {
|
||||
return MessagePackBackend{
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
metrics: newBackendMetrics("messagepack"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +57,7 @@ func (MessagePackBackend) Name() string { return "messagepack" }
|
||||
// IdentifyPath returns the id and mtime of a file
|
||||
func (b MessagePackBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) {
|
||||
metaPath := filepath.Clean(path + ".mpk")
|
||||
b.metrics.reads.Inc()
|
||||
source, err := os.Open(metaPath)
|
||||
// // No cached entry found. Read from storage and store in cache
|
||||
if err != nil {
|
||||
@@ -157,6 +159,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
|
||||
defer func() { _ = unlock() }()
|
||||
}
|
||||
// Read current state
|
||||
b.metrics.reads.Inc()
|
||||
_, subspan := tracer.Start(ctx, "os.ReadFile")
|
||||
var msgBytes []byte
|
||||
msgBytes, err = os.ReadFile(metaPath)
|
||||
@@ -192,6 +195,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode,
|
||||
}
|
||||
|
||||
// overwrite file atomically
|
||||
b.metrics.writes.Inc()
|
||||
_, subspan = tracer.Start(ctx, "renameio.Writefile")
|
||||
err = renameio.WriteFile(metaPath, d, 0600)
|
||||
if err != nil {
|
||||
@@ -210,6 +214,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode)
|
||||
defer span.End()
|
||||
attribs := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
b.metrics.cacheLookup(err == nil)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
}
|
||||
@@ -217,6 +222,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode)
|
||||
metaPath := b.MetadataPath(n)
|
||||
|
||||
// No cached entry found. Read from storage and store in cache
|
||||
b.metrics.reads.Inc()
|
||||
_, subspan := tracer.Start(ctx, "os.Open")
|
||||
source, err := os.Open(metaPath)
|
||||
subspan.End()
|
||||
@@ -271,6 +277,7 @@ func (b MessagePackBackend) Purge(_ context.Context, n MetadataNode) error {
|
||||
return err
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
return os.Remove(b.MetadataPath(n))
|
||||
}
|
||||
|
||||
@@ -289,6 +296,7 @@ func (b MessagePackBackend) Rename(oldNode, newNode MetadataNode) error {
|
||||
return err
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
return os.Rename(b.MetadataPath(oldNode), b.MetadataPath(newNode))
|
||||
}
|
||||
|
||||
@@ -305,7 +313,7 @@ func (b MessagePackBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
}
|
||||
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Generated
Vendored
+92
@@ -0,0 +1,92 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
// the metrics of the backends, by the name of the backend, see backendMetrics. A read or write of the
|
||||
// metadata of a node counts once, however many attributes or files it involves.
|
||||
var (
|
||||
reads = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_metadata_reads_total",
|
||||
Help: "Number of times the metadata of a node has been read from the filesystem",
|
||||
}, []string{"backend"})
|
||||
writes = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_metadata_writes_total",
|
||||
Help: "Number of times the metadata of a node has been written to the filesystem",
|
||||
}, []string{"backend"})
|
||||
cacheLookups = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_metadata_cache_lookups_total",
|
||||
Help: "Number of times the metadata of a node has been looked up in the cache, by whether it has been found (result hit) or not (result miss)",
|
||||
}, []string{"backend", "result"})
|
||||
locksWaiting = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Name: "reva_metadata_lock_waiting",
|
||||
Help: "Number of metadata locks being waited for",
|
||||
}, []string{"backend"})
|
||||
lockWait = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "reva_metadata_lock_wait_seconds",
|
||||
Help: "Time spent waiting for metadata locks, by whether the node is a space root",
|
||||
Buckets: prometheus.ExponentialBuckets(.0001, 4, 10),
|
||||
}, []string{"backend", "space_root"})
|
||||
lockHeld = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "reva_metadata_lock_held_seconds",
|
||||
Help: "Time metadata locks have been held, by whether the node is a space root",
|
||||
Buckets: prometheus.ExponentialBuckets(.0001, 4, 10),
|
||||
}, []string{"backend", "space_root"})
|
||||
)
|
||||
|
||||
// backendMetrics are the metrics of a backend
|
||||
type backendMetrics struct {
|
||||
reads prometheus.Counter
|
||||
writes prometheus.Counter
|
||||
cacheHits prometheus.Counter
|
||||
cacheMisses prometheus.Counter
|
||||
locksWaiting prometheus.Gauge
|
||||
// the locks of space roots are the ones every propagation in a space ends at
|
||||
spaceRootLocks lockMetrics
|
||||
otherLocks lockMetrics
|
||||
}
|
||||
|
||||
type lockMetrics struct {
|
||||
wait prometheus.Observer
|
||||
held prometheus.Observer
|
||||
}
|
||||
|
||||
func newBackendMetrics(backend string) *backendMetrics {
|
||||
return &backendMetrics{
|
||||
reads: reads.WithLabelValues(backend),
|
||||
writes: writes.WithLabelValues(backend),
|
||||
cacheHits: cacheLookups.WithLabelValues(backend, "hit"),
|
||||
cacheMisses: cacheLookups.WithLabelValues(backend, "miss"),
|
||||
locksWaiting: locksWaiting.WithLabelValues(backend),
|
||||
spaceRootLocks: lockMetrics{
|
||||
wait: lockWait.WithLabelValues(backend, "true"),
|
||||
held: lockHeld.WithLabelValues(backend, "true"),
|
||||
},
|
||||
otherLocks: lockMetrics{
|
||||
wait: lockWait.WithLabelValues(backend, "false"),
|
||||
held: lockHeld.WithLabelValues(backend, "false"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// cacheLookup counts a lookup in the cache
|
||||
func (m *backendMetrics) cacheLookup(found bool) {
|
||||
if found {
|
||||
m.cacheHits.Inc()
|
||||
} else {
|
||||
m.cacheMisses.Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// locks returns the lock metrics for the locks of a node
|
||||
func (m *backendMetrics) locks(n MetadataNode) lockMetrics {
|
||||
if id := n.GetID(); id != "" && id == n.GetSpaceID() {
|
||||
return m.spaceRootLocks
|
||||
}
|
||||
return m.otherLocks
|
||||
}
|
||||
Generated
Vendored
+16
-6
@@ -31,18 +31,19 @@ import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
)
|
||||
|
||||
// XattrsBackend stores the file attributes in extended attributes
|
||||
type XattrsBackend struct {
|
||||
metaCache cache.FileMetadataCache
|
||||
metrics *backendMetrics
|
||||
}
|
||||
|
||||
// NewMessageBackend returns a new XattrsBackend instance
|
||||
func NewXattrsBackend(o cache.Config) XattrsBackend {
|
||||
return XattrsBackend{
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
metrics: newBackendMetrics("xattrs"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +52,7 @@ func (XattrsBackend) Name() string { return "xattrs" }
|
||||
|
||||
// IdentifyPath returns the space id, node id and mtime of a file
|
||||
func (b XattrsBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) {
|
||||
b.metrics.reads.Inc()
|
||||
spaceID, _ := xattr.Get(path, prefixes.SpaceIDAttr)
|
||||
id, _ := xattr.Get(path, prefixes.IDAttr)
|
||||
parentID, _ := xattr.Get(path, prefixes.ParentidAttr)
|
||||
@@ -66,10 +68,13 @@ func (b XattrsBackend) IdentifyPath(_ context.Context, path string) (string, str
|
||||
func (b XattrsBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
if err == nil && len(attribs[key]) > 0 {
|
||||
found := err == nil && len(attribs[key]) > 0
|
||||
b.metrics.cacheLookup(found)
|
||||
if found {
|
||||
return attribs[key], err
|
||||
}
|
||||
|
||||
b.metrics.reads.Inc()
|
||||
return xattr.Get(n.InternalPath(), key)
|
||||
}
|
||||
|
||||
@@ -102,7 +107,7 @@ func (b XattrsBackend) list(ctx context.Context, n MetadataNode, acquireLock boo
|
||||
|
||||
// listing xattrs failed, try again, either with lock or without
|
||||
if acquireLock {
|
||||
f, err := lockedfile.OpenFile(filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
f, err := lockFile(n, filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, b.metrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -128,11 +133,13 @@ func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, ac
|
||||
|
||||
if !skipCache {
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs)
|
||||
b.metrics.cacheLookup(err == nil)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
}
|
||||
}
|
||||
|
||||
b.metrics.reads.Inc()
|
||||
attrNames, err := b.list(ctx, n, acquireLock)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -184,7 +191,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockedFile, err := lockedfile.OpenFile(b.LockfilePath(n), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
lockedFile, err := lockFile(n, b.LockfilePath(n), os.O_CREATE|os.O_WRONLY, b.metrics)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -202,6 +209,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
xerrs = 0
|
||||
xerr error
|
||||
)
|
||||
b.metrics.writes.Inc()
|
||||
for key, val := range attribs {
|
||||
if xerr = xattr.Set(path, key, val); xerr != nil {
|
||||
// log
|
||||
@@ -223,7 +231,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs
|
||||
func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) error {
|
||||
path := n.InternalPath()
|
||||
if !n.LockHeld() {
|
||||
lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
lockedFile, err := lockFile(n, path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, b.metrics)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -235,6 +243,7 @@ func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) e
|
||||
}()
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
err := xattr.Remove(path, key)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -259,6 +268,7 @@ func (b XattrsBackend) Purge(ctx context.Context, n MetadataNode) error {
|
||||
return err
|
||||
}
|
||||
|
||||
b.metrics.writes.Inc()
|
||||
for attr := range attribs {
|
||||
if strings.HasPrefix(attr, prefixes.OcPrefix) {
|
||||
err := xattr.Remove(path, attr)
|
||||
@@ -298,7 +308,7 @@ func (b XattrsBackend) Lock(n MetadataNode) (UnlockFunc, error) {
|
||||
}
|
||||
|
||||
metaLockPath := b.LockfilePath(n)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+54
-14
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/conversions"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/mime"
|
||||
@@ -57,6 +58,10 @@ import (
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
// ErrMissingParentID is returned when a node exists on disk but has no parent id,
|
||||
// which means its metadata is gone and the node can no longer be resolved.
|
||||
var ErrMissingParentID = errtypes.InternalError("Missing parent ID on node")
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node")
|
||||
}
|
||||
@@ -524,7 +529,7 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath
|
||||
if _, ok := lu.MetadataBackend().(metadata.MessagePackBackend); ok {
|
||||
appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Str("nodeid", n.ID).Interface("attrs", attrs).Bytes("messagepack", d).Msg("missing parent id")
|
||||
}
|
||||
return nil, errtypes.InternalError("Missing parent ID on node")
|
||||
return nil, ErrMissingParentID
|
||||
}
|
||||
|
||||
if n.Type(ctx) == provider.ResourceType_RESOURCE_TYPE_FILE {
|
||||
@@ -952,8 +957,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi
|
||||
|
||||
// share indicator
|
||||
if _, ok := fieldMaskKeysMap["share-types"]; returnAllFields || ok {
|
||||
granteeTypes := n.getGranteeTypes(ctx)
|
||||
if len(granteeTypes) > 0 {
|
||||
granteeTypes, hasGuestShares := n.getGranteeTypes(ctx)
|
||||
if len(granteeTypes) > 0 || hasGuestShares {
|
||||
// TODO add optional property to CS3 ResourceInfo to transport grants?
|
||||
var s strings.Builder
|
||||
first := true
|
||||
@@ -965,6 +970,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi
|
||||
}
|
||||
s.WriteString(strconv.Itoa(int(t)))
|
||||
}
|
||||
if hasGuestShares {
|
||||
if !first {
|
||||
s.WriteString(",")
|
||||
}
|
||||
s.WriteString(strconv.Itoa(int(conversions.ShareTypeGuest)))
|
||||
}
|
||||
ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "share-types", s.String())
|
||||
}
|
||||
}
|
||||
@@ -1369,17 +1380,25 @@ func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) {
|
||||
return grants, nil
|
||||
}
|
||||
|
||||
func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType {
|
||||
// getGranteeTypes returns the grantee types of the node's grants. The second
|
||||
// return value reports whether the node has guest shares, which are stored as
|
||||
// user grants with a mail ACE prefix.
|
||||
func (n *Node) getGranteeTypes(ctx context.Context) ([]provider.GranteeType, bool) {
|
||||
types := []provider.GranteeType{}
|
||||
hasGuestShares := false
|
||||
if g, err := n.ListGrantees(ctx); err == nil {
|
||||
hasUserShares, hasGroupShares := false, false
|
||||
for i := range g {
|
||||
switch {
|
||||
case !hasUserShares && (strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix) || strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix)):
|
||||
case !hasGuestShares && strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix):
|
||||
hasGuestShares = true
|
||||
case !hasUserShares && strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix):
|
||||
hasUserShares = true
|
||||
case !hasGroupShares && strings.HasPrefix(g[i], prefixes.GrantGroupAcePrefix):
|
||||
hasGroupShares = true
|
||||
case hasUserShares && hasGroupShares:
|
||||
}
|
||||
|
||||
if hasUserShares && hasGroupShares && hasGuestShares {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -1390,7 +1409,7 @@ func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType {
|
||||
types = append(types, provider.GranteeType_GRANTEE_TYPE_GROUP)
|
||||
}
|
||||
}
|
||||
return types
|
||||
return types, hasGuestShares
|
||||
}
|
||||
|
||||
// ancestorGrants returns the active grants on the node and all its ancestors. The grants on the
|
||||
@@ -1659,12 +1678,12 @@ func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, er
|
||||
return n.deleteRevision(ctx, n.ID+RevisionIDDelimiter+versionID)
|
||||
}
|
||||
|
||||
revisionPath, err := n.getLatestRevision(ctx)
|
||||
reverted, err := n.RevertLatestRevision(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if revisionPath == "" {
|
||||
if !reverted {
|
||||
// there is no revision - delete the node
|
||||
unlock, err := n.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
@@ -1680,11 +1699,6 @@ func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, er
|
||||
return "", nil
|
||||
}
|
||||
|
||||
latestID := strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, ""))
|
||||
if err := n.RevertUpload(ctx, latestID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// we just deleted the current revision - remove processing flag if set
|
||||
if uploadid, err := n.ProcessingID(ctx); err == nil {
|
||||
return "", n.UnmarkProcessing(ctx, uploadid)
|
||||
@@ -1735,6 +1749,32 @@ func (n *Node) deleteRevision(ctx context.Context, revisionID string) (string, e
|
||||
return blobID, nil
|
||||
}
|
||||
|
||||
// RevertLatestRevision reverts the upload that created the node's latest
|
||||
// revision: the revision is restored onto the node and removed. Unlike
|
||||
// DeleteRevision it neither purges the node when there is no revision nor
|
||||
// touches the processing flag. It returns false if there is no revision to
|
||||
// restore.
|
||||
func (n *Node) RevertLatestRevision(ctx context.Context) (bool, error) {
|
||||
revisionPath, err := n.getLatestRevision(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if revisionPath == "" {
|
||||
return false, nil
|
||||
}
|
||||
return true, n.RevertUpload(ctx, strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, "")))
|
||||
}
|
||||
|
||||
// LatestRevisionKey returns the version key of the node's latest revision, or "" if it has
|
||||
// none.
|
||||
func (n *Node) LatestRevisionKey(ctx context.Context) (string, error) {
|
||||
revisionPath, err := n.getLatestRevision(ctx)
|
||||
if err != nil || revisionPath == "" {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, "")), nil
|
||||
}
|
||||
|
||||
func (n *Node) getLatestRevision(ctx context.Context) (string, error) {
|
||||
revPrefix := n.lu.VersionPath(n.SpaceID, n.ID, "")
|
||||
revisions, err := filepath.Glob(n.lu.VersionPath(n.SpaceID, n.ID, "*"))
|
||||
|
||||
Generated
Vendored
+2
-2
@@ -45,9 +45,9 @@ type Options struct {
|
||||
// the attribute key prefix of the metadata on disk, process wide, see prefixes.SetOcPrefix
|
||||
MetadataPrefix string `mapstructure:"metadata_prefix"`
|
||||
|
||||
// the propagator to use for this fs. currently only `sync` is fully supported, `async` is available as an experimental feature
|
||||
// the propagator to use for this fs. currently only `sync` is fully supported, `async` is deprecated and will be removed in a future release
|
||||
Propagator string `mapstructure:"propagator"`
|
||||
// Options specific to the async propagator
|
||||
// Options specific to the async propagator, which is deprecated
|
||||
AsyncPropagatorOptions AsyncPropagatorOptions `mapstructure:"async_propagator_options"`
|
||||
|
||||
// decomposedfs fs works on top of a dir of uuid nodes
|
||||
|
||||
Generated
Vendored
+552
@@ -0,0 +1,552 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package decomposedfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
)
|
||||
|
||||
// PrepareCreatesNode reports that PrepareUpload creates a new file's node.
|
||||
func (fs *Decomposedfs) PrepareCreatesNode() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// PrepareUpload finalizes node metadata after bytes are received, before postprocessing,
|
||||
// and marks the node as processing for sessionID. A new file's node is created here, under
|
||||
// the id the ref carries. CommitUpload is called after postprocessing completes.
|
||||
//
|
||||
// The previous version is snapshotted through the tree's CreateRevision, like the legacy
|
||||
// upload path does: posix copies the current content, decomposedfs only the metadata. When
|
||||
// another upload is still processing the node, it is snapshotted even with versioning
|
||||
// disabled, together with that upload's processing mark: if this upload is rolled back, the
|
||||
// other one is restored and can still finish.
|
||||
func (fs *Decomposedfs) PrepareUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) {
|
||||
ctx, span := tracer.Start(ctx, "PrepareUpload")
|
||||
defer span.End()
|
||||
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("PrepareUpload: unexpected NodeFromResource failure")
|
||||
return nil, errtypes.InternalError("PrepareUpload: node lookup failed unexpectedly")
|
||||
}
|
||||
// a new file nobody has created yet: PrepareUpload creates it
|
||||
create := !n.Exists && !info.NodeExisted && info.ParentID != "" && info.Name != ""
|
||||
if !n.Exists && !create {
|
||||
return nil, errtypes.NotFound(ref.String())
|
||||
}
|
||||
if create {
|
||||
if err := fs.checkNewFileParent(ctx, n.SpaceID, info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.ParentID = info.ParentID
|
||||
n.Name = info.Name
|
||||
}
|
||||
|
||||
var (
|
||||
committed bool
|
||||
// only a node this upload created is purged, one it is creating only once
|
||||
// the name is its own: Purge removes the name
|
||||
purge = !info.NodeExisted && !create
|
||||
)
|
||||
// registered before the lock is taken, so it runs once the lock is released.
|
||||
// Purge, not trash: the file never had content, and Purge needs no Delete permission
|
||||
defer func() {
|
||||
if committed || !purge {
|
||||
return
|
||||
}
|
||||
if err := n.Purge(ctx); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not purge new node after failed prepare")
|
||||
}
|
||||
}()
|
||||
|
||||
n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if create {
|
||||
// a node still to be created holds no lock, so any lock id is a mismatch
|
||||
if lockID, _ := ctxpkg.ContextGetLockID(ctx); lockID != "" {
|
||||
return nil, errtypes.Aborted("not locked")
|
||||
}
|
||||
} else if err := n.CheckLock(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// scope to space owner GID for posix deployments; no-op with NullMapper
|
||||
if spaceGID, ok := ctx.Value(CtxKeySpaceGID).(uint32); ok {
|
||||
unscope, err := fs.um.ScopeUserByIds(-1, int(spaceGID))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to scope user")
|
||||
}
|
||||
if unscope != nil {
|
||||
defer func() { _ = unscope() }()
|
||||
}
|
||||
}
|
||||
|
||||
var unlock metadata.UnlockFunc
|
||||
if create {
|
||||
// InitNewNode refuses a name already taken, checks the quota, and on failure
|
||||
// removes what it made and releases its lock. It returns the node lock.
|
||||
unlock, err = fs.tp.InitNewNode(ctx, n, uint64(info.Size))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
purge = true
|
||||
} else {
|
||||
unlock, err = fs.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
defer func() {
|
||||
if err := unlock(); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not close lock")
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
sizeDiff int64
|
||||
versionCreated bool
|
||||
versionNode *node.BaseNode
|
||||
oldAttrs node.Attributes
|
||||
oldMtime time.Time
|
||||
)
|
||||
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
if versionCreated {
|
||||
removeRevision(ctx, fs.lu, versionNode)
|
||||
}
|
||||
if info.NodeExisted && oldAttrs != nil {
|
||||
// mtime goes in the same batch: we still hold the metadata lock and
|
||||
// SetMTime would try to retake it
|
||||
if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &oldAttrs, oldMtime); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not restore node mtime during rollback")
|
||||
}
|
||||
if err := n.SetXattrsWithContext(ctx, oldAttrs); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not restore node xattrs during rollback")
|
||||
}
|
||||
// restoring only rewrites the old keys, so the mark this upload added would stay
|
||||
if _, ok := oldAttrs[prefixes.StatusPrefix]; !ok {
|
||||
if err := n.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not remove processing mark during rollback")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
old *node.Node
|
||||
overwrite bool
|
||||
oldBlobsize int64
|
||||
)
|
||||
if info.NodeExisted {
|
||||
old, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.ID, "", false, nil, false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "PrepareUpload: failed to read existing node")
|
||||
}
|
||||
overwrite = old.BlobID != ""
|
||||
oldBlobsize = old.Blobsize
|
||||
}
|
||||
|
||||
// also for new files: the coordinator's check fails open without GetQuota
|
||||
// permission, and CheckQuota guards disk space too. InitNewNode has checked a
|
||||
// node it created.
|
||||
if !create {
|
||||
if _, err := node.CheckQuota(ctx, n.SpaceRoot, overwrite, uint64(oldBlobsize), uint64(info.Size)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if info.NodeExisted {
|
||||
oldMtime, err = old.GetMTime(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
oldEtag, err := node.CalculateEtag(old.ID, oldMtime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if info.IfMatch != "" && info.IfMatch != oldEtag {
|
||||
return nil, errtypes.Aborted("etag mismatch")
|
||||
}
|
||||
if info.IfNoneMatch != "" {
|
||||
if info.IfNoneMatch == "*" {
|
||||
return nil, errtypes.Aborted("etag mismatch, resource exists")
|
||||
}
|
||||
for _, tag := range strings.Split(info.IfNoneMatch, ",") {
|
||||
if tag == oldEtag {
|
||||
return nil, errtypes.Aborted("etag mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !info.IfUnmodifiedSince.IsZero() && oldMtime.After(info.IfUnmodifiedSince) {
|
||||
return nil, errtypes.Aborted("if-unmodified-since mismatch")
|
||||
}
|
||||
|
||||
// capture full node xattrs for rollback before any write
|
||||
oldAttrs, err = fs.lu.MetadataBackend().All(ctx, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// another upload is still processing the node: keep its state, so a rollback
|
||||
// of this upload can hand the node back to it
|
||||
supersededMark, superseding := oldAttrs[prefixes.StatusPrefix]
|
||||
if !fs.o.DisableVersioning || superseding {
|
||||
timestamp := oldMtime.UTC().Format(time.RFC3339Nano)
|
||||
versionPath, err := fs.tp.CreateRevision(ctx, n, timestamp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
versionNode = node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+timestamp, fs.lu)
|
||||
versionCreated = true
|
||||
if err := os.Chtimes(versionPath, oldMtime, oldMtime); err != nil {
|
||||
return nil, errtypes.InternalError(fmt.Sprintf("failed to set mtime on version node: %s", err))
|
||||
}
|
||||
if superseding {
|
||||
revisionAttrs := map[string][]byte{prefixes.StatusPrefix: supersededMark}
|
||||
if fs.o.DisableVersioning {
|
||||
// not a version, only kept for a rollback: a live commit removes it
|
||||
revisionAttrs[uploadSnapshotAttr] = []byte("1")
|
||||
}
|
||||
if err := fs.lu.MetadataBackend().SetMultiple(ctx, versionNode, revisionAttrs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sizeDiff = info.Size - old.Blobsize
|
||||
} else {
|
||||
// InitNewNode has cached a node it created
|
||||
if c, ok := fs.lu.(node.IDCacher); ok && !create {
|
||||
if err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id")
|
||||
}
|
||||
}
|
||||
sizeDiff = info.Size
|
||||
}
|
||||
|
||||
attrs := node.Attributes{}
|
||||
attrs.SetString(prefixes.IDAttr, n.ID)
|
||||
attrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE))
|
||||
attrs.SetString(prefixes.ParentidAttr, n.ParentID)
|
||||
attrs.SetString(prefixes.NameAttr, n.Name)
|
||||
attrs.SetString(prefixes.BlobIDAttr, sessionID)
|
||||
attrs.SetInt64(prefixes.BlobsizeAttr, info.Size)
|
||||
attrs[prefixes.ChecksumPrefix+"sha1"] = info.Checksums.SHA1
|
||||
attrs[prefixes.ChecksumPrefix+"md5"] = info.Checksums.MD5
|
||||
attrs[prefixes.ChecksumPrefix+"adler32"] = info.Checksums.Adler32
|
||||
// in the same batch, so marking the node costs no write of its own
|
||||
attrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+sessionID)
|
||||
|
||||
mtime := time.Now()
|
||||
if !info.MTime.IsZero() {
|
||||
mtime = info.MTime
|
||||
}
|
||||
if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil {
|
||||
return nil, errors.Wrap(err, "failed to set mtime")
|
||||
}
|
||||
|
||||
if err := n.SetXattrsWithContext(ctx, attrs); err != nil {
|
||||
return nil, errors.Wrap(err, "could not write metadata")
|
||||
}
|
||||
|
||||
if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil {
|
||||
return nil, errors.Wrap(err, "could not propagate size change")
|
||||
}
|
||||
committed = true
|
||||
|
||||
result := &storage.PrepareUploadResult{VersionCreated: versionCreated, SizeDiff: sizeDiff}
|
||||
if !info.NodeExisted {
|
||||
// read off the disk, not through ListGrants: an uploader who cannot see the
|
||||
// space root still needs the manager of a project space reported
|
||||
result.SpaceOwner = n.SpaceOwnerOrManager(ctx)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// checkNewFileParent re-checks, for a file PrepareUpload is about to create, what
|
||||
// TouchFile would have checked: the parent still exists, and the user may still upload
|
||||
// into it. Either may have changed while the bytes were in flight.
|
||||
func (fs *Decomposedfs) checkNewFileParent(ctx context.Context, spaceID string, info storage.UploadInfo) error {
|
||||
parentID := &provider.ResourceId{SpaceId: spaceID, OpaqueId: info.ParentID}
|
||||
parent, err := fs.lu.NodeFromID(ctx, parentID)
|
||||
if err != nil {
|
||||
return errtypes.InternalError(err.Error())
|
||||
}
|
||||
f, _ := storagespace.FormatReference(&provider.Reference{ResourceId: parentID, Path: info.Name})
|
||||
if !parent.Exists {
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
|
||||
rp, err := fs.p.AssemblePermissions(ctx, parent)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.InitiateFileUpload:
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// removeRevision deletes a revision PrepareUpload created, with its metadata, while the
|
||||
// caller holds the node lock (Node.DeleteRevision would try to take it again). Its blob
|
||||
// is kept: the revision shares it with the node, whose old metadata is being restored.
|
||||
func removeRevision(ctx context.Context, lu node.PathLookup, revisionNode *node.BaseNode) {
|
||||
log := appctx.GetLogger(ctx).With().Str("revisionid", revisionNode.ID).Logger()
|
||||
if err := os.Remove(revisionNode.InternalPath()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Err(err).Msg("could not remove revision during rollback")
|
||||
}
|
||||
if err := lu.MetadataBackend().Purge(ctx, revisionNode); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Err(err).Msg("could not purge revision metadata during rollback")
|
||||
}
|
||||
if err := os.Remove(lu.MetadataBackend().LockfilePath(revisionNode)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Err(err).Msg("could not remove revision lockfile during rollback")
|
||||
}
|
||||
}
|
||||
|
||||
// RollbackUpload reverts the node state written by PrepareUpload after a failed or aborted
|
||||
// postprocessing run. It restores the previous revision and reverts the optimistic size
|
||||
// propagation. Unlike upstream it never purges an existing node: without a revision (versioning
|
||||
// disabled) the previous metadata cannot be restored, so it returns an error instead. On posix
|
||||
// this also keeps the file itself, which still holds the previous content until CommitUpload.
|
||||
// When the node metadata has become unreadable the ids recorded in info are used to release
|
||||
// its quota, and to remove the node if the upload created it.
|
||||
func (fs *Decomposedfs) RollbackUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.RollbackInfo) error {
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
if !errors.Is(err, node.ErrMissingParentID) {
|
||||
// the node might only be temporarily unreadable, e.g. because a cache
|
||||
// is unavailable. Do not treat it as orphaned.
|
||||
return err
|
||||
}
|
||||
// The node metadata is unreadable, so the upload can never finish and its
|
||||
// quota would stay consumed forever.
|
||||
return fs.rollbackOrphaned(ctx, ref, sessionID, info, err)
|
||||
}
|
||||
if !n.Exists {
|
||||
return nil // nothing was written yet
|
||||
}
|
||||
n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
curProcessingID, err := n.ProcessingID(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("RollbackUpload: could not read processing ID: %w", err)
|
||||
}
|
||||
if curProcessingID != sessionID {
|
||||
// a newer upload took over the node. The revision it kept of this upload would
|
||||
// hand the node back to an upload that is gone.
|
||||
fs.retireUploadRevision(ctx, n, sessionID, info.NodeExisted)
|
||||
return nil
|
||||
}
|
||||
|
||||
sizeDiff := info.SizeDiff
|
||||
if info.NodeExisted {
|
||||
restored, purge, err := fs.restoreLatestRevision(ctx, n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !restored {
|
||||
return errtypes.PreconditionFailed("RollbackUpload: no revision to restore for node " + n.ID)
|
||||
}
|
||||
if purge {
|
||||
// the file was created by an upload that was rolled back while this one
|
||||
// was processing, so nothing is left to keep
|
||||
if err := n.Purge(ctx); err != nil {
|
||||
return fmt.Errorf("RollbackUpload: could not purge node: %w", err)
|
||||
}
|
||||
sizeDiff += n.Blobsize
|
||||
}
|
||||
} else {
|
||||
// the upload created this node, so undo it. Purge, not trash: the file never
|
||||
// became visible, and Purge needs no Delete permission
|
||||
if err := n.Purge(ctx); err != nil {
|
||||
return fmt.Errorf("RollbackUpload: could not purge node: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if sizeDiff != 0 {
|
||||
if err := fs.tp.Propagate(ctx, n, -sizeDiff); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("RollbackUpload: could not revert propagate")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// restoreLatestRevision restores the node from the revision PrepareUpload created, its
|
||||
// content included on posix, and removes the revision. If the revision kept the processing
|
||||
// mark of an upload this one superseded, the mark is restored too, so that upload can still
|
||||
// finish. It reports false if the node has no revision, and purge if the revision says the
|
||||
// file should not exist anymore (see retireUploadRevision); n then describes the restored
|
||||
// state, with the size the file accounts for.
|
||||
func (fs *Decomposedfs) restoreLatestRevision(ctx context.Context, n *node.Node) (restored, purge bool, err error) {
|
||||
unlock, err := fs.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
defer func() {
|
||||
if err := unlock(); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not close lock")
|
||||
}
|
||||
}()
|
||||
|
||||
key, err := n.LatestRevisionKey(ctx)
|
||||
if err != nil || key == "" {
|
||||
return false, false, err
|
||||
}
|
||||
revisionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+key, fs.lu)
|
||||
revisionAttrs, err := fs.lu.MetadataBackend().All(ctx, revisionNode)
|
||||
if err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
var mtime time.Time
|
||||
if v, ok := revisionAttrs[prefixes.MTimeAttr]; ok {
|
||||
if mtime, err = time.Parse(time.RFC3339Nano, string(v)); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
}
|
||||
if err := fs.tp.RestoreRevision(ctx, revisionNode, n, mtime); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
if mark, ok := revisionAttrs[prefixes.StatusPrefix]; ok {
|
||||
if err := n.SetXattrsWithContext(ctx, node.Attributes{prefixes.StatusPrefix: mark}); err != nil {
|
||||
return false, false, err
|
||||
}
|
||||
}
|
||||
if v, ok := revisionAttrs[prefixes.BlobsizeAttr]; ok {
|
||||
n.Blobsize, _ = strconv.ParseInt(string(v), 10, 64)
|
||||
}
|
||||
_, purge = revisionAttrs[purgeOnRestoreAttr]
|
||||
|
||||
removeRevision(ctx, fs.lu, revisionNode)
|
||||
return true, purge, nil
|
||||
}
|
||||
|
||||
// purgeOnRestoreAttr marks a revision kept for an upload that created the file and was
|
||||
// rolled back while a newer upload was processing: restoring it removes the file.
|
||||
var purgeOnRestoreAttr = prefixes.OcPrefix + "purgeonrestore"
|
||||
|
||||
// uploadSnapshotAttr marks a revision PrepareUpload kept with versioning disabled, only so
|
||||
// that a rollback can hand the node back to the upload it superseded. It is not a version.
|
||||
var uploadSnapshotAttr = prefixes.OcPrefix + "uploadsnapshot"
|
||||
|
||||
// retireUploadRevision handles the revision a newer upload kept of the upload sessionID,
|
||||
// which is being rolled back. If that upload overwrote the file, the revision is dropped:
|
||||
// the one it kept itself describes the file before it. If it created the file, there is
|
||||
// nothing before it, so the revision is marked to remove the file should the newer upload
|
||||
// be rolled back too.
|
||||
func (fs *Decomposedfs) retireUploadRevision(ctx context.Context, n *node.Node, sessionID string, nodeExisted bool) {
|
||||
log := appctx.GetLogger(ctx).With().Str("nodeid", n.ID).Str("sessionid", sessionID).Logger()
|
||||
unlock, err := fs.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not lock node")
|
||||
return
|
||||
}
|
||||
defer func() {
|
||||
if err := unlock(); err != nil {
|
||||
log.Error().Err(err).Msg("could not close lock")
|
||||
}
|
||||
}()
|
||||
|
||||
revision, revisionUnlock, err := fs.uploadRevision(ctx, n, sessionID)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("could not look up the upload's revision")
|
||||
return
|
||||
}
|
||||
if revision == nil {
|
||||
return
|
||||
}
|
||||
defer func() { _ = revisionUnlock() }()
|
||||
|
||||
if nodeExisted {
|
||||
removeRevision(ctx, fs.lu, &revision.BaseNode)
|
||||
return
|
||||
}
|
||||
if err := revision.SetXattrsWithContext(ctx, node.Attributes{purgeOnRestoreAttr: []byte("1")}); err != nil {
|
||||
log.Error().Err(err).Msg("could not mark the upload's revision")
|
||||
}
|
||||
if err := revision.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) {
|
||||
log.Error().Err(err).Msg("could not remove the processing mark from the upload's revision")
|
||||
}
|
||||
}
|
||||
|
||||
// rollbackOrphaned releases the quota of an upload whose node metadata is unreadable, using
|
||||
// the node and parent ids the upload session recorded. It removes the node only if the upload
|
||||
// created it and it is still empty.
|
||||
func (fs *Decomposedfs) rollbackOrphaned(ctx context.Context, ref *provider.Reference, sessionID string, info storage.RollbackInfo, lookupErr error) error {
|
||||
spaceID := ref.GetResourceId().GetSpaceId()
|
||||
if info.NodeID == "" || info.ParentID == "" || spaceID == "" {
|
||||
return fmt.Errorf("RollbackUpload: node lookup failed: %w", lookupErr)
|
||||
}
|
||||
log := appctx.GetLogger(ctx).With().Str("sessionid", sessionID).Str("nodeid", info.NodeID).Logger()
|
||||
log.Info().Err(lookupErr).Msg("node unreadable, rolling back orphaned upload")
|
||||
|
||||
n := node.New(spaceID, info.NodeID, info.ParentID, info.Filename, info.Size, sessionID,
|
||||
provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu)
|
||||
spaceRoot, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, "", false, nil, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("RollbackUpload: space root lookup failed: %w", err)
|
||||
}
|
||||
n.SpaceRoot = spaceRoot
|
||||
|
||||
if info.SizeDiff != 0 {
|
||||
// an error keeps the session, so the rollback can be retried
|
||||
if err := fs.tp.Propagate(ctx, n, -info.SizeDiff); err != nil {
|
||||
return fmt.Errorf("RollbackUpload: could not revert propagate: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Only a node this upload created may be removed. An existing node holds the
|
||||
// previous version, on posix the user's file itself, so it is left alone.
|
||||
if info.NodeExisted {
|
||||
return nil
|
||||
}
|
||||
// The node file stays empty until the upload is committed. If it has content,
|
||||
// something outside of us wrote it, e.g. an editor saving atomically on posix,
|
||||
// so keep it.
|
||||
if fi, err := os.Stat(n.InternalPath()); err == nil && fi.Size() > 0 {
|
||||
log.Info().Str("nodepath", n.InternalPath()).Msg("orphaned node has content, keeping it")
|
||||
return nil
|
||||
}
|
||||
|
||||
// the orphaned node cannot be resolved by any other means, remove it together
|
||||
// with its metadata. A missing node is fine: it may never have been created.
|
||||
if err := fs.lu.PurgeNode(n); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("RollbackUpload: removing orphaned node failed")
|
||||
}
|
||||
if err := fs.lu.MetadataBackend().Purge(ctx, n); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
log.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("RollbackUpload: purging orphaned node metadata failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
+5
@@ -44,6 +44,8 @@ const (
|
||||
var _propagationGracePeriod = 3 * time.Minute
|
||||
|
||||
// AsyncPropagator implements asynchronous treetime & treesize propagation
|
||||
//
|
||||
// Deprecated: the async propagator will be removed in a future release. Use the sync propagator instead.
|
||||
type AsyncPropagator struct {
|
||||
treeSizeAccounting bool
|
||||
treeTimeAccounting bool
|
||||
@@ -59,6 +61,8 @@ type Change struct {
|
||||
}
|
||||
|
||||
// NewAsyncPropagator returns a new AsyncPropagator instance
|
||||
//
|
||||
// Deprecated: the async propagator will be removed in a future release. Use the sync propagator instead.
|
||||
func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.AsyncPropagatorOptions, lookup node.PathLookup, log *zerolog.Logger) AsyncPropagator {
|
||||
p := AsyncPropagator{
|
||||
treeSizeAccounting: treeSizeAccounting,
|
||||
@@ -69,6 +73,7 @@ func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.A
|
||||
}
|
||||
|
||||
log.Info().Msg("async propagator starting up...")
|
||||
log.Warn().Msg("the async propagator is deprecated and will be removed in a future release, please switch to the sync propagator")
|
||||
|
||||
// spawn a goroutine that watches for stale .processing dirs and fixes them
|
||||
go func() {
|
||||
|
||||
Generated
Vendored
+40
@@ -0,0 +1,40 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package propagator
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
// the metrics of the propagation, by the internal root of the storage, see metrics
|
||||
var (
|
||||
queuedChanges = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_propagation_changes_total",
|
||||
Help: "Number of changes to be propagated",
|
||||
}, []string{"root"})
|
||||
directoryUpdates = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_propagation_directory_updates_total",
|
||||
Help: "Number of directory updates written by the propagation",
|
||||
}, []string{"root"})
|
||||
failures = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "reva_propagation_failures_total",
|
||||
Help: "Number of failed attempts to apply changes to a directory",
|
||||
}, []string{"root"})
|
||||
)
|
||||
|
||||
// metrics are the metrics of a propagator
|
||||
type metrics struct {
|
||||
changes prometheus.Counter
|
||||
updates prometheus.Counter
|
||||
failures prometheus.Counter
|
||||
}
|
||||
|
||||
func newMetrics(internalRoot string) metrics {
|
||||
return metrics{
|
||||
changes: queuedChanges.WithLabelValues(internalRoot),
|
||||
updates: directoryUpdates.WithLabelValues(internalRoot),
|
||||
failures: failures.WithLabelValues(internalRoot),
|
||||
}
|
||||
}
|
||||
Generated
Vendored
+5
@@ -35,6 +35,7 @@ type SyncPropagator struct {
|
||||
treeSizeAccounting bool
|
||||
treeTimeAccounting bool
|
||||
lookup node.PathLookup
|
||||
metrics metrics
|
||||
}
|
||||
|
||||
// NewSyncPropagator returns a new AsyncPropagator instance
|
||||
@@ -43,6 +44,7 @@ func NewSyncPropagator(treeSizeAccounting, treeTimeAccounting bool, lookup node.
|
||||
treeSizeAccounting: treeSizeAccounting,
|
||||
treeTimeAccounting: treeTimeAccounting,
|
||||
lookup: lookup,
|
||||
metrics: newMetrics(lookup.InternalRoot()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,6 +64,7 @@ func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff in
|
||||
sublog.Debug().Msg("propagation disabled or nothing to propagate")
|
||||
return nil
|
||||
}
|
||||
p.metrics.changes.Inc()
|
||||
|
||||
// is propagation enabled for the parent node?
|
||||
root := n.SpaceRoot
|
||||
@@ -80,6 +83,7 @@ func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff in
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
p.metrics.failures.Inc()
|
||||
sublog.Error().Err(err).Msg("error propagating")
|
||||
return err
|
||||
}
|
||||
@@ -195,6 +199,7 @@ func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime t
|
||||
log.Error().Err(err).Msg("Failed to update extend attributes of parent node")
|
||||
return n, true, err
|
||||
}
|
||||
p.metrics.updates.Inc()
|
||||
|
||||
return n, false, nil
|
||||
}
|
||||
+36
-11
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
@@ -177,6 +178,9 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool,
|
||||
}
|
||||
}
|
||||
|
||||
if storage.SkipTouchPropagation(ctx) {
|
||||
return nil
|
||||
}
|
||||
return t.Propagate(ctx, n, 0)
|
||||
}
|
||||
|
||||
@@ -485,14 +489,13 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
trashPath := trashNode.InternalPath()
|
||||
err = os.Rename(nodePath, trashPath)
|
||||
if err != nil {
|
||||
// To roll back changes
|
||||
// TODO remove symlink
|
||||
// Roll back changes
|
||||
_ = os.Remove(trashLink)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return
|
||||
}
|
||||
err = t.lookup.MetadataBackend().Rename(n, trashNode)
|
||||
if err != nil {
|
||||
_ = os.Remove(trashLink)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
_ = os.Rename(trashPath, nodePath)
|
||||
return
|
||||
@@ -506,10 +509,9 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) {
|
||||
|
||||
// finally remove the entry from the parent dir
|
||||
if err = os.Remove(path); err != nil {
|
||||
// To roll back changes
|
||||
// TODO revert the rename
|
||||
// TODO remove symlink
|
||||
// Roll back changes
|
||||
_ = t.lookup.MetadataBackend().Rename(trashNode, n)
|
||||
_ = os.Rename(trashPath, nodePath)
|
||||
_ = os.Remove(trashLink)
|
||||
_ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr)
|
||||
return
|
||||
}
|
||||
@@ -670,15 +672,31 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
|
||||
h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return unlock, err
|
||||
// the node file is not ours, so there is nothing of ours to remove
|
||||
releaseNewNodeLock(ctx, n, unlock)
|
||||
if errors.Is(err, fs.ErrExist) {
|
||||
return nil, errtypes.AlreadyExists(n.Name)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
h.Close()
|
||||
|
||||
// from here the node file is ours, but the name may not be: a failure removes the
|
||||
// node file and leaves the name to whichever upload linked it. Removed before
|
||||
// unlocking, so nothing can open the node in between.
|
||||
fail := func(err error) (metadata.UnlockFunc, error) {
|
||||
if rErr := os.Remove(n.InternalPath()); rErr != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(rErr).Str("nodeid", n.ID).Msg("could not remove new node file")
|
||||
}
|
||||
releaseNewNodeLock(ctx, n, unlock)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, subspan = tracer.Start(ctx, "node.CheckQuota")
|
||||
_, err = node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return unlock, err
|
||||
return fail(err)
|
||||
}
|
||||
|
||||
// link child name to parent if it is new
|
||||
@@ -694,15 +712,22 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met
|
||||
log.Info().Err(err).Msg("initNewNode: symlink failed")
|
||||
if errors.Is(err, fs.ErrExist) {
|
||||
log.Info().Err(err).Msg("initNewNode: symlink already exists")
|
||||
return unlock, errtypes.AlreadyExists(n.Name)
|
||||
return fail(errtypes.AlreadyExists(n.Name))
|
||||
}
|
||||
return unlock, errors.Wrap(err, "Decomposedfs: could not symlink child entry")
|
||||
return fail(errors.Wrap(err, "Decomposedfs: could not symlink child entry"))
|
||||
}
|
||||
log.Info().Msg("initNewNode: symlink created")
|
||||
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
// releaseNewNodeLock releases the lock of a node InitNewNode failed to create.
|
||||
func releaseNewNodeLock(ctx context.Context, n *node.Node, unlock metadata.UnlockFunc) {
|
||||
if err := unlock(); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not release new node lock")
|
||||
}
|
||||
}
|
||||
|
||||
func (t *Tree) removeNode(ctx context.Context, n *node.Node) error {
|
||||
path := n.InternalPath()
|
||||
logger := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger()
|
||||
|
||||
+4
-433
@@ -20,441 +20,12 @@ package decomposedfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
"github.com/pkg/errors"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// Upload uploads data to the given resource
|
||||
// TODO Upload (and InitiateUpload) needs a way to receive the expected checksum.
|
||||
// Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated?
|
||||
func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) {
|
||||
_, span := tracer.Start(ctx, "Upload")
|
||||
defer span.End()
|
||||
up, err := fs.GetUpload(ctx, req.Ref.GetPath())
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload")
|
||||
}
|
||||
|
||||
session := up.(*upload.DecomposedFsSession)
|
||||
|
||||
ctx = session.Context(ctx)
|
||||
|
||||
if session.Chunk() != "" { // check chunking v1
|
||||
p, assembledFile, err := fs.chunkHandler.WriteChunk(session.Chunk(), req.Body)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
if p == "" {
|
||||
if err = session.Terminate(ctx); err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error removing auxiliary files")
|
||||
}
|
||||
return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String())
|
||||
}
|
||||
fd, err := os.Open(assembledFile)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error opening assembled file")
|
||||
}
|
||||
defer func() {
|
||||
_ = fd.Close()
|
||||
}()
|
||||
defer func() {
|
||||
_ = os.RemoveAll(assembledFile)
|
||||
}()
|
||||
req.Body = fd
|
||||
|
||||
size, err := session.WriteChunk(ctx, 0, req.Body)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file")
|
||||
}
|
||||
session.SetSize(size)
|
||||
} else {
|
||||
size, err := session.WriteChunk(ctx, 0, req.Body)
|
||||
if err != nil {
|
||||
return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file")
|
||||
}
|
||||
if size != req.Length {
|
||||
return &provider.ResourceInfo{}, errtypes.PartialContent("Decomposedfs: unexpected end of stream")
|
||||
}
|
||||
}
|
||||
|
||||
if err := session.FinishUploadDecomposed(ctx); err != nil {
|
||||
return &provider.ResourceInfo{}, err
|
||||
}
|
||||
|
||||
if uff != nil {
|
||||
uploadRef := &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: session.ProviderID(),
|
||||
SpaceId: session.SpaceID(),
|
||||
OpaqueId: session.SpaceID(),
|
||||
},
|
||||
Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())),
|
||||
}
|
||||
executant := session.Executant()
|
||||
uff(session.SpaceOwner(), &executant, uploadRef)
|
||||
}
|
||||
|
||||
ri := &provider.ResourceInfo{
|
||||
// fill with at least fileid, mtime and etag
|
||||
Id: &provider.ResourceId{
|
||||
StorageId: session.ProviderID(),
|
||||
SpaceId: session.SpaceID(),
|
||||
OpaqueId: session.NodeID(),
|
||||
},
|
||||
}
|
||||
|
||||
// add etag to metadata
|
||||
ri.Etag, _ = node.CalculateEtag(session.NodeID(), session.MTime())
|
||||
|
||||
if !session.MTime().IsZero() {
|
||||
ri.Mtime = utils.TimeToTS(session.MTime())
|
||||
}
|
||||
|
||||
return ri, nil
|
||||
}
|
||||
|
||||
// InitiateUpload returns upload ids corresponding to different protocols it supports
|
||||
// TODO read optional content for small files in this request
|
||||
// TODO InitiateUpload (and Upload) needs a way to receive the expected checksum. Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated?
|
||||
func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) {
|
||||
_, span := tracer.Start(ctx, "InitiateUpload")
|
||||
defer span.End()
|
||||
log := appctx.GetLogger(ctx)
|
||||
|
||||
// remember the path from the reference
|
||||
refpath := ref.GetPath()
|
||||
var chunk *chunking.ChunkBLOBInfo
|
||||
var err error
|
||||
if chunking.IsChunked(refpath) { // check chunking v1
|
||||
chunk, err = chunking.GetChunkBLOBInfo(refpath)
|
||||
if err != nil {
|
||||
return nil, errtypes.BadRequest(err.Error())
|
||||
}
|
||||
ref.Path = chunk.Path
|
||||
}
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
switch err.(type) {
|
||||
case nil:
|
||||
// ok
|
||||
case errtypes.IsNotFound:
|
||||
return nil, errtypes.PreconditionFailed(err.Error())
|
||||
default:
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// permissions are checked in NewUpload below
|
||||
|
||||
relative, err := fs.lu.Path(ctx, n, node.NoCheck)
|
||||
// TODO why do we need the path here?
|
||||
// jfd: it is used later when emitting the UploadReady event ...
|
||||
// AAAND refPath might be . when accessing with an id / relative reference ... which causes NodeName to become . But then dir will also always be .
|
||||
// That is why we still have to read the path here: so that the event we emit contains a relative reference with a path relative to the space root. WTF
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lockID, _ := ctxpkg.ContextGetLockID(ctx)
|
||||
|
||||
session := fs.sessionStore.New(ctx)
|
||||
session.SetMetadata("filename", n.Name)
|
||||
session.SetStorageValue("NodeName", n.Name)
|
||||
if chunk != nil {
|
||||
session.SetStorageValue("Chunk", filepath.Base(refpath))
|
||||
}
|
||||
session.SetMetadata("dir", filepath.Dir(relative))
|
||||
session.SetStorageValue("Dir", filepath.Dir(relative))
|
||||
session.SetMetadata("lockid", lockID)
|
||||
|
||||
session.SetSize(uploadLength)
|
||||
session.SetStorageValue("SpaceRoot", n.SpaceRoot.ID) // TODO SpaceRoot -> SpaceID
|
||||
session.SetStorageValue("SpaceOwnerOrManager", n.SpaceOwnerOrManager(ctx).GetOpaqueId()) // TODO needed for what?
|
||||
|
||||
spaceGID, ok := ctx.Value(CtxKeySpaceGID).(uint32)
|
||||
if ok {
|
||||
session.SetStorageValue("SpaceGid", fmt.Sprintf("%d", spaceGID))
|
||||
}
|
||||
|
||||
iid, _ := ctxpkg.ContextGetInitiator(ctx)
|
||||
session.SetMetadata("initiatorid", iid)
|
||||
|
||||
if metadata != nil {
|
||||
session.SetMetadata("providerID", metadata["providerID"])
|
||||
if mtime, ok := metadata["mtime"]; ok {
|
||||
if mtime != "null" {
|
||||
session.SetMetadata("mtime", metadata["mtime"])
|
||||
}
|
||||
}
|
||||
if expiration, ok := metadata["expires"]; ok {
|
||||
if expiration != "null" {
|
||||
session.SetMetadata("expires", metadata["expires"])
|
||||
}
|
||||
}
|
||||
if _, ok := metadata["sizedeferred"]; ok {
|
||||
session.SetSizeIsDeferred(true)
|
||||
}
|
||||
if checksum, ok := metadata["checksum"]; ok {
|
||||
parts := strings.SplitN(checksum, " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
|
||||
}
|
||||
switch parts[0] {
|
||||
case "sha1", "md5", "adler32":
|
||||
session.SetMetadata("checksum", checksum)
|
||||
default:
|
||||
return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
|
||||
}
|
||||
}
|
||||
|
||||
// only check preconditions if they are not empty // TODO or is this a bad request?
|
||||
if metadata["if-match"] != "" {
|
||||
session.SetMetadata("if-match", metadata["if-match"])
|
||||
}
|
||||
if metadata["if-none-match"] != "" {
|
||||
session.SetMetadata("if-none-match", metadata["if-none-match"])
|
||||
}
|
||||
if metadata["if-unmodified-since"] != "" {
|
||||
session.SetMetadata("if-unmodified-since", metadata["if-unmodified-since"])
|
||||
}
|
||||
}
|
||||
|
||||
if session.MTime().IsZero() {
|
||||
session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now()))
|
||||
}
|
||||
|
||||
log.Debug().Str("uploadid", session.ID()).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename")
|
||||
|
||||
_, err = node.CheckQuota(ctx, n.SpaceRoot, n.Exists, uint64(n.Blobsize), uint64(session.Size()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if session.Filename() == "" {
|
||||
return nil, errors.New("Decomposedfs: missing filename in metadata")
|
||||
}
|
||||
if session.Dir() == "" {
|
||||
return nil, errors.New("Decomposedfs: missing dir in metadata")
|
||||
}
|
||||
|
||||
// the parent owner will become the new owner
|
||||
parent, perr := n.Parent(ctx)
|
||||
if perr != nil {
|
||||
return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID)
|
||||
}
|
||||
|
||||
// check permissions
|
||||
var (
|
||||
checkNode *node.Node
|
||||
path string
|
||||
)
|
||||
if n.Exists {
|
||||
// check permissions of file to be overwritten
|
||||
checkNode = n
|
||||
path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{
|
||||
SpaceId: checkNode.SpaceID,
|
||||
OpaqueId: checkNode.ID,
|
||||
}})
|
||||
} else {
|
||||
// check permissions of parent
|
||||
checkNode = parent
|
||||
path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{
|
||||
SpaceId: checkNode.SpaceID,
|
||||
OpaqueId: checkNode.ID,
|
||||
}, Path: n.Name})
|
||||
}
|
||||
rp, err := fs.p.AssemblePermissions(ctx, checkNode)
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, err
|
||||
case !rp.InitiateFileUpload:
|
||||
return nil, errtypes.PermissionDenied(path)
|
||||
}
|
||||
|
||||
// are we trying to overwriting a folder with a file?
|
||||
if n.Exists && n.IsDir(ctx) {
|
||||
return nil, errtypes.PreconditionFailed("resource is not a file")
|
||||
}
|
||||
|
||||
// check lock
|
||||
if err := n.CheckLock(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
usr := ctxpkg.ContextMustGetUser(ctx)
|
||||
|
||||
// fill future node info
|
||||
if n.Exists {
|
||||
if session.HeaderIfNoneMatch() == "*" {
|
||||
return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", n.ParentID, n.Name, n.ID))
|
||||
}
|
||||
session.SetStorageValue("NodeId", n.ID)
|
||||
session.SetStorageValue("NodeExists", "true")
|
||||
} else {
|
||||
session.SetStorageValue("NodeId", uuid.New().String())
|
||||
}
|
||||
session.SetStorageValue("NodeParentId", n.ParentID)
|
||||
session.SetExecutant(usr)
|
||||
session.SetStorageValue("LogLevel", log.GetLevel().String())
|
||||
|
||||
log.Debug().Interface("session", session).Msg("Decomposedfs: built session info")
|
||||
|
||||
err = fs.um.RunInBaseScope(func() error {
|
||||
// Create binary file in the upload folder with no content
|
||||
// It will be used when determining the current offset of an upload
|
||||
err := session.TouchBin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return session.Persist(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics.UploadSessionsInitiated.Inc()
|
||||
|
||||
if uploadLength == 0 {
|
||||
// Directly finish this upload
|
||||
err = session.FinishUploadDecomposed(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]string{
|
||||
"simple": session.ID(),
|
||||
"tus": session.ID(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// UseIn tells the tus upload middleware which extensions it supports.
|
||||
func (fs *Decomposedfs) UseIn(composer *tusd.StoreComposer) {
|
||||
composer.UseCore(fs)
|
||||
composer.UseTerminater(fs)
|
||||
composer.UseConcater(fs)
|
||||
composer.UseLengthDeferrer(fs)
|
||||
composer.UseContentServer(fs)
|
||||
}
|
||||
|
||||
// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol
|
||||
// - the storage needs to implement NewUpload and GetUpload
|
||||
// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload
|
||||
|
||||
// NewUpload returns a new tus Upload instance
|
||||
func (fs *Decomposedfs) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) {
|
||||
return nil, fmt.Errorf("not implemented, use InitiateUpload on the CS3 API to start a new upload")
|
||||
}
|
||||
|
||||
// AsServableUpload returns a ServableUpload
|
||||
// which implements the tusd.ServableUpload interface and
|
||||
func (fs *Decomposedfs) AsServableUpload(u tusd.Upload) tusd.ServableUpload {
|
||||
return u.(*upload.DecomposedFsSession)
|
||||
}
|
||||
|
||||
// GetUpload returns the Upload for the given upload id
|
||||
func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) {
|
||||
var ul tusd.Upload
|
||||
var err error
|
||||
_ = fs.um.RunInBaseScope(func() error {
|
||||
ul, err = fs.sessionStore.Get(ctx, id)
|
||||
return nil
|
||||
})
|
||||
return ul, err
|
||||
}
|
||||
|
||||
// ResolveUpload stats the resource produced by a finished upload, reading it as the upload's
|
||||
// executant. The session is reconstructed from the given tus FileInfo rather than the session
|
||||
// store, so it still resolves after a synchronous upload has cleaned up its session. It returns
|
||||
// the resource info and the executant context, so the caller can derive identity dependent
|
||||
// values such as the WebDAV permissions. https://github.com/opencloud-eu/opencloud/issues/2409
|
||||
func (fs *Decomposedfs) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) {
|
||||
session := fs.sessionStore.SessionFromInfo(info)
|
||||
ctx = session.Context(ctx)
|
||||
ref := session.Reference()
|
||||
ri, err := fs.GetMD(ctx, &ref, nil, nil)
|
||||
return ri, ctx, err
|
||||
}
|
||||
|
||||
// ListUploadSessions returns the upload sessions for the given filter
|
||||
func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) {
|
||||
var sessions []*upload.DecomposedFsSession
|
||||
if filter.ID != nil && *filter.ID != "" {
|
||||
session, err := fs.sessionStore.Get(ctx, *filter.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sessions = []*upload.DecomposedFsSession{session}
|
||||
} else {
|
||||
var err error
|
||||
sessions, err = fs.sessionStore.List(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
filteredSessions := []storage.UploadSession{}
|
||||
now := time.Now()
|
||||
for _, session := range sessions {
|
||||
if filter.Processing != nil && *filter.Processing != session.IsProcessing() {
|
||||
continue
|
||||
}
|
||||
if filter.Expired != nil {
|
||||
if *filter.Expired {
|
||||
if now.Before(session.Expires()) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if now.After(session.Expires()) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
if filter.HasVirus != nil {
|
||||
sr, _ := session.ScanData()
|
||||
infected := sr != ""
|
||||
if *filter.HasVirus != infected {
|
||||
continue
|
||||
}
|
||||
}
|
||||
filteredSessions = append(filteredSessions, session)
|
||||
}
|
||||
return filteredSessions, nil
|
||||
}
|
||||
|
||||
// AsTerminatableUpload returns a TerminatableUpload
|
||||
// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination
|
||||
// the storage needs to implement AsTerminatableUpload
|
||||
func (fs *Decomposedfs) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
}
|
||||
|
||||
// AsLengthDeclarableUpload returns a LengthDeclarableUpload
|
||||
// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation
|
||||
// the storage needs to implement AsLengthDeclarableUpload
|
||||
func (fs *Decomposedfs) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
}
|
||||
|
||||
// AsConcatableUpload returns a ConcatableUpload
|
||||
// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation
|
||||
// the storage needs to implement AsConcatableUpload
|
||||
func (fs *Decomposedfs) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload {
|
||||
return up.(*upload.DecomposedFsSession)
|
||||
// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable.
|
||||
func (fs *Decomposedfs) IsOrphaned(ctx context.Context, ref *provider.Reference) bool {
|
||||
_, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
return err != nil
|
||||
}
|
||||
Generated
Vendored
-364
@@ -1,364 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/google/renameio/v2"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// DecomposedFsSession extends tus upload lifecycle with postprocessing steps.
|
||||
type DecomposedFsSession struct {
|
||||
store DecomposedFsStore
|
||||
// for now, we keep the json files in the uploads folder
|
||||
info tusd.FileInfo
|
||||
}
|
||||
|
||||
const (
|
||||
SessionStatusUploading = "uploading"
|
||||
SessionStatusProcessing = "processing"
|
||||
SessionStatusFailed = "failed"
|
||||
)
|
||||
|
||||
// Context returns a context with the user, logger and lockid used when initiating the upload session
|
||||
func (session *DecomposedFsSession) Context(ctx context.Context) context.Context { // restore logger from file info
|
||||
sub := session.store.log.With().Int("pid", os.Getpid()).Logger()
|
||||
ctx = appctx.WithLogger(ctx, &sub)
|
||||
ctx = ctxpkg.ContextSetLockID(ctx, session.lockID())
|
||||
ctx = ctxpkg.ContextSetUser(ctx, session.executantUser())
|
||||
return ctxpkg.ContextSetInitiator(ctx, session.InitiatorID())
|
||||
}
|
||||
|
||||
func (session *DecomposedFsSession) lockID() string {
|
||||
return session.info.MetaData["lockid"]
|
||||
}
|
||||
func (session *DecomposedFsSession) executantUser() *userpb.User {
|
||||
var o *typespb.Opaque
|
||||
_ = json.Unmarshal([]byte(session.info.Storage["UserOpaque"]), &o)
|
||||
return &userpb.User{
|
||||
Id: &userpb.UserId{
|
||||
Type: userpb.UserType(userpb.UserType_value[session.info.Storage["UserType"]]),
|
||||
Idp: session.info.Storage["Idp"],
|
||||
OpaqueId: session.info.Storage["UserId"],
|
||||
},
|
||||
Username: session.info.Storage["UserName"],
|
||||
DisplayName: session.info.Storage["UserDisplayName"],
|
||||
Opaque: o,
|
||||
}
|
||||
}
|
||||
|
||||
// Purge deletes the upload session metadata and written binary data
|
||||
func (session *DecomposedFsSession) Purge(ctx context.Context) {
|
||||
_, span := tracer.Start(ctx, "Purge")
|
||||
defer span.End()
|
||||
session.Cleanup(true, true, true, true)
|
||||
}
|
||||
|
||||
// TouchBin creates a file to contain the binary data. It's size will be used to keep track of the tus upload offset.
|
||||
func (session *DecomposedFsSession) TouchBin() error {
|
||||
file, err := os.OpenFile(session.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return file.Close()
|
||||
}
|
||||
|
||||
// Persist writes the upload session metadata to disk
|
||||
// events can update the scan outcome and the finished event might read an empty file because of race conditions
|
||||
// so we need to lock the file while writing and use atomic writes
|
||||
func (session *DecomposedFsSession) Persist(ctx context.Context) error {
|
||||
_, span := tracer.Start(ctx, "Persist")
|
||||
defer span.End()
|
||||
infoPath := session.infoPath()
|
||||
// create folder structure (if needed)
|
||||
if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var d []byte
|
||||
d, err := json.Marshal(session.info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return renameio.WriteFile(infoPath, d, 0600)
|
||||
}
|
||||
|
||||
// ToFileInfo returns tus compatible FileInfo so the tus handler can access the upload offset
|
||||
func (session *DecomposedFsSession) ToFileInfo() tusd.FileInfo {
|
||||
return session.info
|
||||
}
|
||||
|
||||
// ProviderID returns the provider id
|
||||
func (session *DecomposedFsSession) ProviderID() string {
|
||||
return session.info.MetaData["providerID"]
|
||||
}
|
||||
|
||||
// SpaceID returns the space id
|
||||
func (session *DecomposedFsSession) SpaceID() string {
|
||||
return session.info.Storage["SpaceRoot"]
|
||||
}
|
||||
|
||||
// NodeID returns the node id
|
||||
func (session *DecomposedFsSession) NodeID() string {
|
||||
return session.info.Storage["NodeId"]
|
||||
}
|
||||
|
||||
// NodeParentID returns the nodes parent id
|
||||
func (session *DecomposedFsSession) NodeParentID() string {
|
||||
return session.info.Storage["NodeParentId"]
|
||||
}
|
||||
|
||||
// NodeExists returns wether or not the node existed during InitiateUpload.
|
||||
// FIXME If two requests try to write the same file they both will store a new
|
||||
// random node id in the session and try to initialize a new node when
|
||||
// finishing the upload. The second request will fail with an already exists
|
||||
// error when trying to create the symlink for the node in the parent directory.
|
||||
// A node should be created as part of InitiateUpload. When listing a directory
|
||||
// we can decide if we want to skip the entry, or expose uploed progress
|
||||
// information. But that is a bigger change and might involve client work.
|
||||
func (session *DecomposedFsSession) NodeExists() bool {
|
||||
return session.info.Storage["NodeExists"] == "true"
|
||||
}
|
||||
|
||||
// HeaderIfMatch returns the if-match header for the upload session
|
||||
func (session *DecomposedFsSession) HeaderIfMatch() string {
|
||||
return session.info.MetaData["if-match"]
|
||||
}
|
||||
|
||||
// HeaderIfNoneMatch returns the if-none-match header for the upload session
|
||||
func (session *DecomposedFsSession) HeaderIfNoneMatch() string {
|
||||
return session.info.MetaData["if-none-match"]
|
||||
}
|
||||
|
||||
// HeaderIfUnmodifiedSince returns the if-unmodified-since header for the upload session
|
||||
func (session *DecomposedFsSession) HeaderIfUnmodifiedSince() string {
|
||||
return session.info.MetaData["if-unmodified-since"]
|
||||
}
|
||||
|
||||
// Node returns the node for the session
|
||||
func (session *DecomposedFsSession) Node(ctx context.Context) (*node.Node, error) {
|
||||
return node.ReadNode(ctx, session.store.lu, session.SpaceID(), session.info.Storage["NodeId"], "", false, nil, true)
|
||||
}
|
||||
|
||||
// ID returns the upload session id
|
||||
func (session *DecomposedFsSession) ID() string {
|
||||
return session.info.ID
|
||||
}
|
||||
|
||||
// Filename returns the name of the node which is not the same as the name af the file being uploaded for legacy chunked uploads
|
||||
func (session *DecomposedFsSession) Filename() string {
|
||||
return session.info.Storage["NodeName"]
|
||||
}
|
||||
|
||||
// Chunk returns the chunk name when a legacy chunked upload was started
|
||||
func (session *DecomposedFsSession) Chunk() string {
|
||||
return session.info.Storage["Chunk"]
|
||||
}
|
||||
|
||||
// SetMetadata is used to fill the upload metadata that will be exposed to the end user
|
||||
func (session *DecomposedFsSession) SetMetadata(key, value string) {
|
||||
session.info.MetaData[key] = value
|
||||
}
|
||||
|
||||
// SetStorageValue is used to set metadata only relevant for the upload session implementation
|
||||
func (session *DecomposedFsSession) SetStorageValue(key, value string) {
|
||||
session.info.Storage[key] = value
|
||||
}
|
||||
|
||||
// SetSize will set the upload size of the underlying tus info.
|
||||
func (session *DecomposedFsSession) SetSize(size int64) {
|
||||
session.info.Size = size
|
||||
}
|
||||
|
||||
// SetSizeIsDeferred is uset to change the SizeIsDeferred property of the underlying tus info.
|
||||
func (session *DecomposedFsSession) SetSizeIsDeferred(value bool) {
|
||||
session.info.SizeIsDeferred = value
|
||||
}
|
||||
|
||||
// Dir returns the directory to which the upload is made
|
||||
// TODO get rid of Dir(), whoever consumes the reference should be able to deal
|
||||
// with a relative reference.
|
||||
// Dir is only used to:
|
||||
// - fill the Path property when emitting the UploadReady event after
|
||||
// postprocessing finished. I wonder why the UploadReady contains a finished
|
||||
// flag ... maybe multiple distinct events would make more sense.
|
||||
// - build the reference that is passed to the FileUploaded event in the
|
||||
// UploadFinishedFunc callback passed to the Upload call used for simple
|
||||
// datatx put requests
|
||||
//
|
||||
// AFAICT only search and audit services consume the path.
|
||||
// - search needs to index from the root anyway. And it only needs the most
|
||||
// recent path to put it in the index. So it should already be able to deal
|
||||
// with an id based reference.
|
||||
// - audit on the other hand needs to log events with the path at the state of
|
||||
// the event ... so it does need the full path.
|
||||
//
|
||||
// I think we can safely determine the path later, right before emitting the
|
||||
// event. And maybe make it configurable, because only audit needs it, anyway.
|
||||
func (session *DecomposedFsSession) Dir() string {
|
||||
return session.info.Storage["Dir"]
|
||||
}
|
||||
|
||||
// Size returns the upload size
|
||||
func (session *DecomposedFsSession) Size() int64 {
|
||||
return session.info.Size
|
||||
}
|
||||
|
||||
// SizeDiff returns the size diff that was calculated after postprocessing
|
||||
func (session *DecomposedFsSession) SizeDiff() int64 {
|
||||
sizeDiff, _ := strconv.ParseInt(session.info.MetaData["sizeDiff"], 10, 64)
|
||||
return sizeDiff
|
||||
}
|
||||
|
||||
// Reference returns a reference that can be used to access the uploaded resource
|
||||
func (session *DecomposedFsSession) Reference() provider.Reference {
|
||||
return provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: session.info.MetaData["providerID"],
|
||||
SpaceId: session.info.Storage["SpaceRoot"],
|
||||
OpaqueId: session.info.Storage["NodeId"],
|
||||
},
|
||||
// Path is not used
|
||||
}
|
||||
}
|
||||
|
||||
// Executant returns the id of the user that initiated the upload session
|
||||
func (session *DecomposedFsSession) Executant() userpb.UserId {
|
||||
return userpb.UserId{
|
||||
Type: userpb.UserType(userpb.UserType_value[session.info.Storage["UserType"]]),
|
||||
Idp: session.info.Storage["Idp"],
|
||||
OpaqueId: session.info.Storage["UserId"],
|
||||
}
|
||||
}
|
||||
|
||||
// SetExecutant is used to remember the user that initiated the upload session
|
||||
func (session *DecomposedFsSession) SetExecutant(u *userpb.User) {
|
||||
session.info.Storage["Idp"] = u.GetId().GetIdp()
|
||||
session.info.Storage["UserId"] = u.GetId().GetOpaqueId()
|
||||
session.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type)
|
||||
session.info.Storage["UserName"] = u.GetUsername()
|
||||
session.info.Storage["UserDisplayName"] = u.GetDisplayName()
|
||||
|
||||
b, _ := json.Marshal(u.GetOpaque())
|
||||
session.info.Storage["UserOpaque"] = string(b)
|
||||
}
|
||||
|
||||
// Offset returns the current upload offset
|
||||
func (session *DecomposedFsSession) Offset() int64 {
|
||||
return session.info.Offset
|
||||
}
|
||||
|
||||
// SpaceOwner returns the id of the space owner
|
||||
func (session *DecomposedFsSession) SpaceOwner() *userpb.UserId {
|
||||
return &userpb.UserId{
|
||||
// idp and type do not seem to be consumed and the node currently only stores the user id anyway
|
||||
OpaqueId: session.info.Storage["SpaceOwnerOrManager"],
|
||||
}
|
||||
}
|
||||
|
||||
// Expires returns the time the upload session expires
|
||||
func (session *DecomposedFsSession) Expires() time.Time {
|
||||
var t time.Time
|
||||
if value, ok := session.info.MetaData["expires"]; ok {
|
||||
t, _ = utils.MTimeToTime(value)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// MTime returns the mtime to use for the uploaded file
|
||||
func (session *DecomposedFsSession) MTime() time.Time {
|
||||
var t time.Time
|
||||
if value, ok := session.info.MetaData["mtime"]; ok {
|
||||
t, _ = utils.MTimeToTime(value)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// IsProcessing returns true if the upload is in the processing state, meaning that the upload has finished and postprocessing is still running.
|
||||
func (session *DecomposedFsSession) IsProcessing() bool {
|
||||
return session.info.MetaData["status"] == SessionStatusProcessing
|
||||
}
|
||||
|
||||
// binPath returns the path to the file storing the binary data.
|
||||
func (session *DecomposedFsSession) binPath() string {
|
||||
return filepath.Join(session.store.root, "uploads", session.info.ID)
|
||||
}
|
||||
|
||||
// infoPath returns the path to the .info file storing the file's info.
|
||||
func (session *DecomposedFsSession) infoPath() string {
|
||||
return sessionPath(session.store.root, session.info.ID)
|
||||
}
|
||||
|
||||
// InitiatorID returns the id of the initiating client
|
||||
func (session *DecomposedFsSession) InitiatorID() string {
|
||||
return session.info.MetaData["initiatorid"]
|
||||
}
|
||||
|
||||
// SetScanData sets virus scan data to the upload session
|
||||
func (session *DecomposedFsSession) SetScanData(result string, date time.Time) {
|
||||
session.info.MetaData["scanResult"] = result
|
||||
session.info.MetaData["scanDate"] = date.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// ScanData returns the virus scan data
|
||||
func (session *DecomposedFsSession) ScanData() (string, time.Time) {
|
||||
date := session.info.MetaData["scanDate"]
|
||||
if date == "" {
|
||||
return "", time.Time{}
|
||||
}
|
||||
d, _ := time.Parse(time.RFC3339, date)
|
||||
return session.info.MetaData["scanResult"], d
|
||||
}
|
||||
|
||||
// SetStatus sets the status of the upload session
|
||||
func (session *DecomposedFsSession) SetStatus(status, msg string) {
|
||||
session.info.MetaData["status"] = status
|
||||
session.info.MetaData["statusMessage"] = msg
|
||||
}
|
||||
|
||||
// Status returns the status of the upload session
|
||||
func (session *DecomposedFsSession) Status() string {
|
||||
return session.info.MetaData["status"]
|
||||
}
|
||||
|
||||
// StatusMessage returns the status message of the upload session
|
||||
func (session *DecomposedFsSession) StatusMessage() string {
|
||||
return session.info.MetaData["statusMessage"]
|
||||
}
|
||||
|
||||
// sessionPath returns the path to the .info file storing the file's info.
|
||||
func sessionPath(root, id string) string {
|
||||
return filepath.Join(root, "uploads", id+".info")
|
||||
}
|
||||
Generated
Vendored
-419
@@ -1,419 +0,0 @@
|
||||
// Copyright 2018-2022 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
iofs "io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
)
|
||||
|
||||
var _idRegexp = regexp.MustCompile(".*/([^/]+).info")
|
||||
|
||||
// PermissionsChecker defines an interface for checking permissions on a Node
|
||||
type PermissionsChecker interface {
|
||||
AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error)
|
||||
}
|
||||
|
||||
// DecomposedFsStore manages upload sessions
|
||||
type DecomposedFsStore struct {
|
||||
fs storage.FS
|
||||
lu node.PathLookup
|
||||
tp node.Tree
|
||||
um usermapper.Mapper
|
||||
root string
|
||||
pub events.Publisher
|
||||
async bool
|
||||
tknopts options.TokenOptions
|
||||
disableVersioning bool
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
// NewSessionStore returns a new DecomposedFsStore
|
||||
func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async bool, tknopts options.TokenOptions, log *zerolog.Logger) *DecomposedFsStore {
|
||||
return &DecomposedFsStore{
|
||||
fs: fs,
|
||||
lu: aspects.Lookup,
|
||||
tp: aspects.Tree,
|
||||
root: root,
|
||||
pub: aspects.EventStream,
|
||||
async: async,
|
||||
tknopts: tknopts,
|
||||
disableVersioning: aspects.DisableVersioning,
|
||||
um: aspects.UserMapper,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// New returns a new upload session
|
||||
func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession {
|
||||
return &DecomposedFsSession{
|
||||
store: store,
|
||||
info: tusd.FileInfo{
|
||||
ID: uuid.New().String(),
|
||||
Storage: map[string]string{
|
||||
"Type": "DecomposedFsStore",
|
||||
},
|
||||
MetaData: tusd.MetaData{
|
||||
"status": string(SessionStatusUploading),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// SessionFromInfo wraps an existing tus FileInfo into an upload session without reading the
|
||||
// session store. It lets callers use an upload's session, e.g. its reference and
|
||||
// executant context, after the store entry has already been cleaned up, as happens for a
|
||||
// synchronous upload in its finish response callback.
|
||||
func (store DecomposedFsStore) SessionFromInfo(info tusd.FileInfo) *DecomposedFsSession {
|
||||
return &DecomposedFsSession{
|
||||
store: store,
|
||||
info: info,
|
||||
}
|
||||
}
|
||||
|
||||
// List lists all upload sessions
|
||||
func (store DecomposedFsStore) List(ctx context.Context) ([]*DecomposedFsSession, error) {
|
||||
uploads := []*DecomposedFsSession{}
|
||||
infoFiles, err := filepath.Glob(filepath.Join(store.root, "uploads", "*.info"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, info := range infoFiles {
|
||||
id := strings.TrimSuffix(filepath.Base(info), filepath.Ext(info))
|
||||
progress, err := store.Get(ctx, id)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Interface("path", info).Msg("Decomposedfs: could not getUploadSession")
|
||||
continue
|
||||
}
|
||||
|
||||
uploads = append(uploads, progress)
|
||||
}
|
||||
return uploads, nil
|
||||
}
|
||||
|
||||
// Get returns the upload session for the given upload id
|
||||
func (store DecomposedFsStore) Get(ctx context.Context, id string) (*DecomposedFsSession, error) {
|
||||
sessionPath := sessionPath(store.root, id)
|
||||
match := _idRegexp.FindStringSubmatch(sessionPath)
|
||||
if len(match) < 2 {
|
||||
return nil, fmt.Errorf("invalid upload path")
|
||||
}
|
||||
|
||||
session := DecomposedFsSession{
|
||||
store: store,
|
||||
info: tusd.FileInfo{},
|
||||
}
|
||||
data, err := os.ReadFile(sessionPath)
|
||||
if err != nil {
|
||||
// handle stale NFS file handles that can occur when the file is deleted betwenn the ATTR and FOPEN call of os.ReadFile
|
||||
if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE {
|
||||
appctx.GetLogger(ctx).Info().Str("session", id).Err(err).Msg("treating stale file handle as not found")
|
||||
err = tusd.ErrNotFound
|
||||
}
|
||||
if errors.Is(err, iofs.ErrNotExist) {
|
||||
// Interpret os.ErrNotExist as 404 Not Found
|
||||
err = tusd.ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &session.info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stat, err := os.Stat(session.binPath())
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// Interpret os.ErrNotExist as 404 Not Found
|
||||
err = tusd.ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
session.info.Offset = stat.Size()
|
||||
|
||||
return &session, nil
|
||||
}
|
||||
|
||||
// Session is the interface used by the Cleanup call
|
||||
type Session interface {
|
||||
ID() string
|
||||
Node(ctx context.Context) (*node.Node, error)
|
||||
Context(ctx context.Context) context.Context
|
||||
Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool)
|
||||
}
|
||||
|
||||
// CreateNodeForUpload will create the target node for the Upload
|
||||
// TODO move this to the node package as NodeFromUpload?
|
||||
// should we in InitiateUpload create the node first? and then the upload?
|
||||
func (store DecomposedFsStore) CreateNodeForUpload(ctx context.Context, session *DecomposedFsSession, initAttrs node.Attributes) (*node.Node, error) {
|
||||
ctx, span := tracer.Start(session.Context(ctx), "CreateNodeForUpload")
|
||||
defer span.End()
|
||||
n := node.New(
|
||||
session.SpaceID(),
|
||||
session.NodeID(),
|
||||
session.NodeParentID(),
|
||||
session.Filename(),
|
||||
session.Size(),
|
||||
session.ID(),
|
||||
provider.ResourceType_RESOURCE_TYPE_FILE,
|
||||
nil,
|
||||
store.lu,
|
||||
)
|
||||
var err error
|
||||
n.SpaceRoot, err = node.ReadNode(ctx, store.lu, session.SpaceID(), session.SpaceID(), "", false, nil, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check lock
|
||||
if err := n.CheckLock(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var unlock metadata.UnlockFunc
|
||||
if session.NodeExists() { // TODO this is wrong. The node should be created when the upload starts, the revisions should be created independently of the node
|
||||
// we do not need to propagate a change when a node is created, only when the upload is ready.
|
||||
// that still creates problems for desktop clients because if another change causes propagation it will detects an empty file
|
||||
// so the first upload has to point to the first revision with the expected size. The file cannot be downloaded, but it can be overwritten (which will create a new revision and make the node reflect the latest revision)
|
||||
// any finished postprocessing will not affect the node metadata.
|
||||
// *thinking* but then initializing an upload will lock the file until the upload has finished. That sucks.
|
||||
// so we have to check if the node has been created meanwhile (well, only in case the upload does not know the nodeid ... or the NodeExists array that is checked by session.NodeExists())
|
||||
// FIXME look at the disk again to see if the file has been created in between, or just try initializing a new node and do the update existing node as a fallback. <- the latter!
|
||||
|
||||
unlock, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size()))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("failed to update existing node")
|
||||
}
|
||||
} else {
|
||||
if c, ok := store.lu.(node.IDCacher); ok {
|
||||
err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id")
|
||||
}
|
||||
}
|
||||
|
||||
unlock, err = store.tp.InitNewNode(ctx, n, uint64(session.Size()))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Err(err).Msg("failed to init new node")
|
||||
}
|
||||
session.info.MetaData["sizeDiff"] = strconv.FormatInt(session.Size(), 10)
|
||||
}
|
||||
defer func() {
|
||||
if unlock == nil {
|
||||
appctx.GetLogger(ctx).Info().Msg("did not get a unlockfunc, not unlocking")
|
||||
return
|
||||
}
|
||||
|
||||
if err := unlock(); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock")
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// overwrite technical information
|
||||
initAttrs.SetString(prefixes.IDAttr, n.ID)
|
||||
initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE))
|
||||
initAttrs.SetString(prefixes.ParentidAttr, n.ParentID)
|
||||
initAttrs.SetString(prefixes.NameAttr, n.Name)
|
||||
initAttrs.SetString(prefixes.BlobIDAttr, n.BlobID)
|
||||
initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize)
|
||||
initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+session.ID())
|
||||
|
||||
// set mtime on the new node
|
||||
mtime := time.Now()
|
||||
if !session.MTime().IsZero() {
|
||||
// overwrite mtime if requested
|
||||
mtime = session.MTime()
|
||||
}
|
||||
err = store.lu.TimeManager().OverrideMtime(ctx, n, &initAttrs, mtime)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
|
||||
}
|
||||
|
||||
// update node metadata with new blobid etc
|
||||
err = n.SetXattrsWithContext(ctx, initAttrs)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not write metadata")
|
||||
}
|
||||
|
||||
err = store.um.RunInBaseScope(func() error {
|
||||
return session.Persist(ctx)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *DecomposedFsSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) {
|
||||
_, span := tracer.Start(ctx, "updateExistingNode")
|
||||
defer span.End()
|
||||
|
||||
// write lock existing node before reading any metadata
|
||||
unlock, err := store.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, "", false, nil, false)
|
||||
if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
|
||||
oldNodeMtime, err := old.GetMTime(ctx)
|
||||
if err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
oldNodeEtag, err := node.CalculateEtag(old.ID, oldNodeMtime)
|
||||
if err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
|
||||
// When the if-match header was set we need to check if the
|
||||
// etag still matches before finishing the upload.
|
||||
if session.HeaderIfMatch() != "" && session.HeaderIfMatch() != oldNodeEtag {
|
||||
return unlock, errtypes.Aborted("etag mismatch")
|
||||
}
|
||||
|
||||
// When the if-none-match header was set we need to check if any of the
|
||||
// etags matches before finishing the upload.
|
||||
if session.HeaderIfNoneMatch() != "" {
|
||||
if session.HeaderIfNoneMatch() == "*" {
|
||||
return unlock, errtypes.Aborted("etag mismatch, resource exists")
|
||||
}
|
||||
for _, ifNoneMatchTag := range strings.Split(session.HeaderIfNoneMatch(), ",") {
|
||||
if ifNoneMatchTag == oldNodeEtag {
|
||||
return unlock, errtypes.Aborted("etag mismatch")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// When the if-unmodified-since header was set we need to check if the
|
||||
// etag still matches before finishing the upload.
|
||||
if session.HeaderIfUnmodifiedSince() != "" {
|
||||
ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, session.HeaderIfUnmodifiedSince())
|
||||
if err != nil {
|
||||
return unlock, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err))
|
||||
}
|
||||
|
||||
if oldNodeMtime.After(ifUnmodifiedSince) {
|
||||
return unlock, errtypes.Aborted("if-unmodified-since mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
if !store.disableVersioning {
|
||||
span.AddEvent("CreateVersion")
|
||||
timestamp := oldNodeMtime.UTC().Format(time.RFC3339Nano)
|
||||
versionID := n.ID + node.RevisionIDDelimiter + timestamp
|
||||
versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
return unlock, err
|
||||
}
|
||||
|
||||
// a revision with this mtime does already exist.
|
||||
// If the blobs are the same we can just delete the old one
|
||||
versionNode := node.NewBaseNode(n.SpaceID, versionID, session.store.lu)
|
||||
if err := validateChecksums(ctx, old, session, versionNode); err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
|
||||
// delete old blob
|
||||
bID, _, err := session.store.lu.ReadBlobIDAndSizeAttr(ctx, versionNode, nil)
|
||||
if err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
if err := session.store.tp.DeleteBlob(&node.Node{BaseNode: node.BaseNode{SpaceID: n.SpaceID}, BlobID: bID}); err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
|
||||
// clean revision file
|
||||
if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp); err != nil {
|
||||
return unlock, err
|
||||
}
|
||||
}
|
||||
|
||||
session.info.MetaData["versionID"] = versionID
|
||||
// keep mtime from previous version
|
||||
span.AddEvent("os.Chtimes")
|
||||
if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil {
|
||||
return unlock, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
session.info.MetaData["sizeDiff"] = strconv.FormatInt((int64(fsize) - old.Blobsize), 10)
|
||||
|
||||
return unlock, nil
|
||||
}
|
||||
|
||||
func validateChecksums(ctx context.Context, n *node.Node, session *DecomposedFsSession, versionNode metadata.MetadataNode) error {
|
||||
for _, t := range []string{"md5", "sha1", "adler32"} {
|
||||
key := prefixes.ChecksumPrefix + t
|
||||
|
||||
checksum, err := n.Xattr(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
revisionChecksum, err := session.store.lu.MetadataBackend().Get(ctx, versionNode, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if string(checksum) == "" || string(revisionChecksum) == "" {
|
||||
return errors.New("checksum not found")
|
||||
}
|
||||
|
||||
if string(checksum) != string(revisionChecksum) {
|
||||
return errors.New("checksum mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
-605
@@ -1,605 +0,0 @@
|
||||
// Copyright 2018-2022 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package upload
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
"github.com/pkg/errors"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
var (
|
||||
tracer trace.Tracer
|
||||
defaultFilePerm = os.FileMode(0664)
|
||||
)
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload")
|
||||
}
|
||||
|
||||
// WriteChunk writes the stream from the reader to the given offset of the upload
|
||||
func (session *DecomposedFsSession) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) {
|
||||
ctx, span := tracer.Start(session.Context(ctx), "WriteChunk")
|
||||
defer span.End()
|
||||
_, subspan := tracer.Start(ctx, "os.OpenFile")
|
||||
file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
subspan.End()
|
||||
|
||||
log := appctx.GetLogger(ctx)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("WriteChunk: error opening upload file")
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
// sync the written chunk to disk. This ensures that the upload can be resumed,
|
||||
// and helps to prevent issues with filesystem/journal freezes at the end of the upload
|
||||
// when committing a large fsync operation on slow disks.
|
||||
_ = disk.Fdatasync(file)
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
// calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum
|
||||
// TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ...
|
||||
// It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used
|
||||
// but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ...
|
||||
_, subspan = tracer.Start(ctx, "io.Copy")
|
||||
n, err := io.Copy(file, src)
|
||||
subspan.End()
|
||||
|
||||
// If the HTTP PATCH request gets interrupted in the middle (e.g. because
|
||||
// the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF.
|
||||
// However, for the decompsedfs driver it's not important whether the stream has ended
|
||||
// on purpose or accidentally.
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
log.Error().Err(err).Msg("WriteChunk: error copying data to upload file")
|
||||
return n, err
|
||||
}
|
||||
|
||||
// update upload.Session.Offset so subsequent code flow can use it.
|
||||
// No need to persist the session as the offset is determined by stating the blob in the GetUpload / ReadSession codepath.
|
||||
// The session offset is written to disk in FinishUpload
|
||||
session.info.Offset += n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// GetInfo returns the FileInfo
|
||||
func (session *DecomposedFsSession) GetInfo(_ context.Context) (tusd.FileInfo, error) {
|
||||
return session.ToFileInfo(), nil
|
||||
}
|
||||
|
||||
// GetReader returns an io.Reader for the upload
|
||||
func (session *DecomposedFsSession) GetReader(ctx context.Context) (io.ReadCloser, error) {
|
||||
_, span := tracer.Start(session.Context(ctx), "GetReader")
|
||||
defer span.End()
|
||||
return os.Open(session.binPath())
|
||||
}
|
||||
|
||||
// FinishUpload finishes an upload and moves the file to the internal destination
|
||||
// implements tusd.DataStore interface
|
||||
// returns tusd errors
|
||||
func (session *DecomposedFsSession) FinishUpload(ctx context.Context) error {
|
||||
err := session.FinishUploadDecomposed(ctx)
|
||||
|
||||
if err != nil {
|
||||
// this is part of the tusd integration and we might be able to
|
||||
// log the error in another place
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Error().Err(err).Msg("failed to finish upload")
|
||||
}
|
||||
|
||||
// we need to return a tusd error here to make the tusd handler return the correct status code
|
||||
switch err.(type) {
|
||||
case errtypes.AlreadyExists:
|
||||
return tusd.NewError("ERR_ALREADY_EXISTS", err.Error(), http.StatusConflict)
|
||||
case errtypes.Aborted:
|
||||
return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed)
|
||||
case errtypes.InsufficientStorage:
|
||||
return tusd.NewError("ERR_INSUFFICIENT_STORAGE", err.Error(), http.StatusInsufficientStorage)
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// FinishUploadDecomposed finishes an upload and moves the file to the internal destination
|
||||
// retures errtypes errors
|
||||
func (session *DecomposedFsSession) FinishUploadDecomposed(ctx context.Context) error {
|
||||
ctx, span := tracer.Start(session.Context(ctx), "FinishUpload")
|
||||
defer span.End()
|
||||
log := appctx.GetLogger(ctx)
|
||||
|
||||
ctx = ctxpkg.ContextSetInitiator(ctx, session.InitiatorID())
|
||||
|
||||
ctx = context.WithoutCancel(ctx) // Do not cancel the finish process, we unconditionally want to complete the upload.
|
||||
sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// compare if they match the sent checksum
|
||||
// TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads
|
||||
if session.info.MetaData["checksum"] != "" {
|
||||
var err error
|
||||
parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2)
|
||||
if len(parts) != 2 {
|
||||
return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'")
|
||||
}
|
||||
switch parts[0] {
|
||||
case "sha1":
|
||||
err = checkHash(parts[1], sha1h)
|
||||
case "md5":
|
||||
err = checkHash(parts[1], md5h)
|
||||
case "adler32":
|
||||
err = checkHash(parts[1], adler32h)
|
||||
default:
|
||||
err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0])
|
||||
}
|
||||
if err != nil {
|
||||
session.Cleanup(false, true, true, false)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// update checksums
|
||||
attrs := node.Attributes{
|
||||
prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
|
||||
}
|
||||
|
||||
// At this point we scope by the space to create the final file in the final location
|
||||
if session.store.um != nil && session.info.Storage["SpaceGid"] != "" {
|
||||
gid, err := strconv.Atoi(session.info.Storage["SpaceGid"])
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to parse space gid")
|
||||
}
|
||||
|
||||
unscope, err := session.store.um.ScopeUserByIds(-1, gid)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to scope user")
|
||||
}
|
||||
if unscope != nil {
|
||||
defer func() { _ = unscope() }()
|
||||
}
|
||||
}
|
||||
|
||||
n, err := session.store.CreateNodeForUpload(ctx, session, attrs)
|
||||
if err != nil {
|
||||
session.SetStatus(SessionStatusFailed, err.Error())
|
||||
if perr := session.Persist(ctx); perr != nil {
|
||||
log.Error().Err(perr).Msg("failed to persist upload session after setting status to failed")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
session.SetStatus(SessionStatusProcessing, "")
|
||||
if err = session.Persist(ctx); err != nil {
|
||||
log.Error().Err(err).Msg("failed to persist upload session after setting status to processing")
|
||||
}
|
||||
|
||||
// increase the processing counter for every started processing
|
||||
// will be decreased in Cleanup()
|
||||
metrics.UploadProcessing.Inc()
|
||||
metrics.UploadSessionsBytesReceived.Inc()
|
||||
|
||||
if session.store.pub != nil && session.info.Size > 0 {
|
||||
u, _ := ctxpkg.ContextGetUser(ctx)
|
||||
s, err := session.URL(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var iu *userpb.User
|
||||
if utils.ExistsInOpaque(u.Opaque, "impersonating-user") {
|
||||
iu = &userpb.User{}
|
||||
if err := utils.ReadJSONFromOpaque(u.Opaque, "impersonating-user", iu); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := events.Publish(ctx, session.store.pub, events.BytesReceived{
|
||||
UploadID: session.ID(),
|
||||
URL: s,
|
||||
SpaceOwner: n.SpaceOwnerOrManager(session.Context(ctx)),
|
||||
ExecutingUser: u,
|
||||
ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID},
|
||||
Filename: session.Filename(),
|
||||
Filesize: uint64(session.Size()),
|
||||
ImpersonatingUser: iu,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// if the upload is synchronous or the upload is empty, finalize it now
|
||||
// for 0-byte uploads we take a shortcut and finalize isn't called elsewhere
|
||||
if !session.store.async || session.info.Size == 0 {
|
||||
// handle postprocessing synchronously
|
||||
err = session.Finalize(ctx)
|
||||
session.Cleanup(err != nil, err == nil, true, true)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("failed to upload")
|
||||
return err
|
||||
}
|
||||
metrics.UploadSessionsFinalized.Inc()
|
||||
}
|
||||
|
||||
return session.store.tp.Propagate(ctx, n, session.SizeDiff())
|
||||
}
|
||||
|
||||
// Terminate terminates the upload
|
||||
func (session *DecomposedFsSession) Terminate(_ context.Context) error {
|
||||
session.Cleanup(true, true, true, true)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeclareLength updates the upload length information
|
||||
func (session *DecomposedFsSession) DeclareLength(ctx context.Context, length int64) error {
|
||||
session.info.Size = length
|
||||
session.info.SizeIsDeferred = false
|
||||
return session.store.um.RunInBaseScope(func() error {
|
||||
return session.Persist(session.Context(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
// ConcatUploads concatenates multiple uploads
|
||||
func (session *DecomposedFsSession) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) {
|
||||
file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = file.Close()
|
||||
}()
|
||||
|
||||
for _, partialUpload := range uploads {
|
||||
fileUpload := partialUpload.(*DecomposedFsSession)
|
||||
|
||||
src, err := os.Open(fileUpload.binPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = src.Close()
|
||||
}()
|
||||
|
||||
if _, err := io.Copy(file, src); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Finalize finalizes the upload (eg moves the file to the internal destination)
|
||||
func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) {
|
||||
ctx, span := tracer.Start(session.Context(ctx), "Finalize")
|
||||
defer span.End()
|
||||
|
||||
n := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(),
|
||||
provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu)
|
||||
|
||||
var (
|
||||
spaceRoot *node.Node
|
||||
)
|
||||
switch spaceRoot, err = session.store.lu.NodeFromSpaceID(ctx, session.SpaceID()); {
|
||||
case err != nil:
|
||||
return fmt.Errorf("failed to get space root for space id %s: %v", session.SpaceID(), err)
|
||||
case spaceRoot == nil:
|
||||
return fmt.Errorf("space root for space id %s not found", session.SpaceID())
|
||||
case spaceRoot.InternalPath() == "":
|
||||
return fmt.Errorf("space root for space id %s has no valid internal path", session.SpaceID())
|
||||
default:
|
||||
n.SpaceRoot = spaceRoot
|
||||
}
|
||||
|
||||
// lock the node before reading its metadata and writing the blob
|
||||
unlock, err := session.store.lu.MetadataBackend().Lock(n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = unlock()
|
||||
}()
|
||||
|
||||
// Read the node attributes while holding the lock acquired above.
|
||||
attribs, err := n.Xattrs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
status := attribs.String(prefixes.StatusPrefix)
|
||||
isProcessing := strings.HasPrefix(status, node.ProcessingStatus)
|
||||
processingID := strings.TrimPrefix(status, node.ProcessingStatus)
|
||||
|
||||
// another upload on this node is in progress or has finished since we started
|
||||
if !isProcessing || processingID != session.ID() {
|
||||
// The node's current content is already this upload's content (e.g. a
|
||||
// newer upload was reverted in the meantime) - no revision is needed,
|
||||
// upload the blob to the node itself.
|
||||
if attribs.String(prefixes.BlobIDAttr) == session.ID() {
|
||||
appctx.GetLogger(ctx).Debug().Str("nodepath", n.InternalPath()).Msg("node already contains this upload's content, no revision needed")
|
||||
} else {
|
||||
versionID := n.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano)
|
||||
// There should be a revision node (created by the other upload that finished before us), read it and upload our blob there.
|
||||
existingRevisionNode, revisionNodeUnlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false)
|
||||
if err != nil || !existingRevisionNode.Exists {
|
||||
// The revision node has not been created. Likely because the file on disk was modified externally and re-assilimated (watchfs == true)
|
||||
// Let's create the revision node now and upload the blob to it.
|
||||
n, revisionNodeUnlock, err = session.createRevisionNodeForUpload(ctx, n, session.MTime().UTC().Format(time.RFC3339Nano))
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Debug().Err(err).Str("versionID", session.MTime().UTC().Format(time.RFC3339Nano)).Msg("failed to create revision node for upload finalization")
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
n = existingRevisionNode
|
||||
}
|
||||
appctx.GetLogger(ctx).Debug().Str("new nodepath", n.InternalPath()).Msg("uploading to revision node, that was created for us by another upload")
|
||||
defer func() { _ = revisionNodeUnlock() }()
|
||||
}
|
||||
}
|
||||
|
||||
// upload the data to the blobstore
|
||||
_, subspan := tracer.Start(ctx, "WriteBlob")
|
||||
err = session.store.tp.WriteBlob(n, session.binPath())
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to upload file to blobstore")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string) (*node.Node, func() error, error) {
|
||||
versionID := baseNode.ID + node.RevisionIDDelimiter + rev
|
||||
log := appctx.GetLogger(ctx)
|
||||
_, err := session.store.tp.CreateRevision(ctx, baseNode, rev)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("versionID", versionID).Msg("failed to create revision node for upload")
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// FIXME: We already calculated the checksums in FinishUpload, we should maybe pass them via the session instead of recalculating them here
|
||||
sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// update checksums
|
||||
attrs := node.Attributes{
|
||||
prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "md5": md5h.Sum(nil),
|
||||
prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil),
|
||||
}
|
||||
revisionNode, unlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false)
|
||||
if err == nil {
|
||||
mtime := session.MTime()
|
||||
attrs.SetString(prefixes.BlobIDAttr, session.ID())
|
||||
attrs.SetInt64(prefixes.BlobsizeAttr, session.Size())
|
||||
revisionNode.BlobID = session.ID()
|
||||
revisionNode.Blobsize = session.Size()
|
||||
|
||||
err = session.store.lu.TimeManager().OverrideMtime(ctx, revisionNode, &attrs, mtime)
|
||||
if err != nil {
|
||||
_ = unlock()
|
||||
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime")
|
||||
}
|
||||
err = revisionNode.SetXattrsWithContext(ctx, attrs)
|
||||
if err != nil {
|
||||
_ = unlock()
|
||||
return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes")
|
||||
}
|
||||
}
|
||||
return revisionNode, unlock, err
|
||||
}
|
||||
|
||||
func checkHash(expected string, h hash.Hash) error {
|
||||
shash := hex.EncodeToString(h.Sum(nil))
|
||||
if expected != shash {
|
||||
return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, shash))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// cleanup cleans up after the upload is finished
|
||||
func (session *DecomposedFsSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) {
|
||||
ctx := session.Context(context.Background())
|
||||
sublog := session.store.log.With().Str("cleanup sessionid", session.ID()).Bool("revertNodeMetadata", revertNodeMetadata).Bool("cleanBin", cleanBin).
|
||||
Bool("cleanInfo", cleanInfo).Bool("unmarkPostprocessing", unmarkPostprocessing).Logger()
|
||||
|
||||
if revertNodeMetadata {
|
||||
n, err := session.Node(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Msg("reading node for session failed")
|
||||
} else {
|
||||
versionID := strings.TrimPrefix(session.info.MetaData["versionID"], n.ID+node.RevisionIDDelimiter)
|
||||
if session.NodeExists() && versionID != "" {
|
||||
sublog.Debug().Str("nodepath", n.InternalPath()).Str("versionID", versionID).Msg("restoring revision")
|
||||
if err := n.RevertUpload(ctx, versionID); err != nil {
|
||||
sublog.Error().Err(err).Str("versionID", versionID).Msg("reverting node metadata failed")
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// if no other upload session is in progress (processing id != session id) or has finished (processing id == "")
|
||||
latestSession, err := n.ProcessingID(ctx)
|
||||
if err != nil {
|
||||
sublog.Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("uploadid", session.ID()).Msg("reading processingid for session failed")
|
||||
}
|
||||
if latestSession == session.ID() {
|
||||
// actually delete the node
|
||||
if err := n.Purge(ctx); err != nil {
|
||||
sublog.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("purging node failed")
|
||||
}
|
||||
}
|
||||
// FIXME else if the upload has become a revision, delete the revision, or if it is the last one, delete the node
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if cleanBin {
|
||||
if err := os.Remove(session.binPath()); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
appctx.GetLogger(ctx).Error().Str("path", session.binPath()).Err(err).Msg("removing upload failed")
|
||||
}
|
||||
}
|
||||
|
||||
if cleanInfo {
|
||||
if err := os.Remove(session.infoPath()); err != nil {
|
||||
sublog.Error().Err(err).Msg("removing upload info failed")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if unmarkPostprocessing {
|
||||
n, err := session.Node(ctx)
|
||||
if err != nil {
|
||||
sublog.Info().Err(err).Msg("could not read node")
|
||||
return
|
||||
}
|
||||
// FIXME: after cleanup the node might already be deleted ...
|
||||
if n != nil { // node can be nil when there was an error before it was created (eg. checksum-mismatch)
|
||||
if err := n.UnmarkProcessing(ctx, session.ID()); err != nil {
|
||||
sublog.Info().Err(err).Str("path", n.InternalPath()).Msg("unmarking processing failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// URL returns a url to download an upload
|
||||
func (session *DecomposedFsSession) URL(_ context.Context) (string, error) {
|
||||
|
||||
u := joinurl(session.store.tknopts.DownloadEndpoint, "tus/", session.ID())
|
||||
if session.store.tknopts.DataGatewayEndpoint == "" {
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// we need to create a token
|
||||
type transferClaims struct {
|
||||
jwt.RegisteredClaims
|
||||
Target string `json:"target"`
|
||||
}
|
||||
|
||||
ttl := time.Duration(session.store.tknopts.TransferExpires) * time.Second
|
||||
claims := transferClaims{
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)),
|
||||
Audience: jwt.ClaimStrings{"reva"},
|
||||
IssuedAt: jwt.NewNumericDate(time.Now()),
|
||||
},
|
||||
Target: u,
|
||||
}
|
||||
|
||||
t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims)
|
||||
|
||||
tkn, err := t.SignedString([]byte(session.store.tknopts.TransferSharedSecret))
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "error signing token with claims %+v", claims)
|
||||
}
|
||||
|
||||
return joinurl(session.store.tknopts.DataGatewayEndpoint, tkn), nil
|
||||
}
|
||||
|
||||
// ServeContent serves the content of the upload and implements the http.ServeContent interface needed by tusd,
|
||||
// it is used by the tusd handler to serve the content of the upload and supports range requests
|
||||
func (session *DecomposedFsSession) ServeContent(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
|
||||
_, span := tracer.Start(session.Context(ctx), "ServeContent")
|
||||
defer span.End()
|
||||
|
||||
f, err := os.Open(session.binPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = f.Close()
|
||||
}()
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var r io.Reader = f
|
||||
if err := func() error {
|
||||
if req.Header.Get("Range") == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ranges, err := download.ParseRange(req.Header.Get("Range"), info.Size())
|
||||
switch {
|
||||
case len(ranges) == 0:
|
||||
fallthrough
|
||||
case errors.Is(err, download.ErrInvalidRange):
|
||||
// ignore invalid range and return the whole file
|
||||
return nil
|
||||
case err != nil:
|
||||
return err
|
||||
}
|
||||
|
||||
r = io.NewSectionReader(f, ranges[0].Start, ranges[0].Length)
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Header().Set("Content-Range", ranges[0].ContentRange(info.Size()))
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(w, r); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// replace with url.JoinPath after switching to go1.19
|
||||
func joinurl(paths ...string) string {
|
||||
var s strings.Builder
|
||||
l := len(paths)
|
||||
for i, p := range paths {
|
||||
s.WriteString(p)
|
||||
if !strings.HasSuffix(p, "/") && i != l-1 {
|
||||
s.WriteString("/")
|
||||
}
|
||||
}
|
||||
|
||||
return s.String()
|
||||
}
|
||||
+100
-11
@@ -22,11 +22,11 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1"
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
)
|
||||
|
||||
// MoveResult carries the resolved references returned by a successful Move.
|
||||
@@ -36,6 +36,44 @@ type MoveResult struct {
|
||||
NewReference *provider.Reference
|
||||
}
|
||||
|
||||
type UploadChecksums struct {
|
||||
SHA1 []byte
|
||||
MD5 []byte
|
||||
Adler32 []byte
|
||||
}
|
||||
|
||||
type UploadInfo struct {
|
||||
NodeExisted bool // true when the target node existed before the upload started
|
||||
Size int64
|
||||
MTime time.Time
|
||||
Checksums UploadChecksums
|
||||
IfMatch string
|
||||
IfNoneMatch string
|
||||
IfUnmodifiedSince time.Time
|
||||
// Where a new file goes, for a driver whose PrepareUpload creates it. Empty
|
||||
// leaves a missing node NotFound.
|
||||
ParentID string
|
||||
Name string
|
||||
}
|
||||
|
||||
type PrepareUploadResult struct {
|
||||
VersionCreated bool
|
||||
SizeDiff int64
|
||||
SpaceOwner *user.UserId // the space owner or a manager, for a new node; nil keeps the one already known
|
||||
}
|
||||
|
||||
// RollbackInfo carries what a driver needs to undo PrepareUpload. NodeID and
|
||||
// ParentID come from the upload session rather than the node, so a rollback can
|
||||
// still release the quota of a node whose own metadata has become unreadable.
|
||||
type RollbackInfo struct {
|
||||
NodeExisted bool // true when the target node had a prior version; drivers with nothing to undo for new nodes may no-op
|
||||
SizeDiff int64
|
||||
NodeID string
|
||||
ParentID string
|
||||
Filename string
|
||||
Size int64
|
||||
}
|
||||
|
||||
// FS is the interface to implement access to the storage.
|
||||
type FS interface {
|
||||
// Minimal set for a readonly storage driver
|
||||
@@ -69,7 +107,7 @@ type FS interface {
|
||||
// CreateDir creates a resource of type container
|
||||
CreateDir(ctx context.Context, ref *provider.Reference) error
|
||||
// TouchFile sets the mtime of a resource, creating an empty file if it does not exist
|
||||
// FIXME(OCISDEV-900) remove markprocessing bool: coordinator calls MarkProcessing(true) explicitly after TouchFile
|
||||
// FIXME(OCISDEV-900) remove markprocessing bool: PrepareUpload marks the node the coordinator touched
|
||||
// FIXME the mtime should either be a time.Time or a CS3 Timestamp, not a string
|
||||
TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error
|
||||
// Delete deletes a resource.
|
||||
@@ -79,15 +117,24 @@ type FS interface {
|
||||
// space owner for event publication. Drivers without this metadata may return
|
||||
// a nil result on success.
|
||||
Move(ctx context.Context, oldRef, newRef *provider.Reference) (*MoveResult, error)
|
||||
// InitiateUpload returns a list of protocols with urls that can be used to append bytes to a new upload session
|
||||
InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error)
|
||||
// Upload creates or updates a resource of type file with a new revision
|
||||
Upload(ctx context.Context, req UploadRequest, uploadFunc UploadFinishedFunc) (*provider.ResourceInfo, error)
|
||||
// MarkProcessing toggles a processing flag on the resource.
|
||||
MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error
|
||||
// CommitUpload writes the staged bytes from source to the resource at ref.
|
||||
// Caller owns source.Body and must close it after CommitUpload returns.
|
||||
// source carries a Body, a Path or both. Caller owns source.Body and must close it
|
||||
// after CommitUpload returns.
|
||||
CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source UploadSource) error
|
||||
// PrepareUpload is called after all bytes are received and before postprocessing begins.
|
||||
// Implementations may lock the target node, snapshot the previous version, write new metadata,
|
||||
// mark the node as processing for sessionID, and propagate size changes. Drivers that do not
|
||||
// require any of these steps may return immediately.
|
||||
PrepareUpload(ctx context.Context, ref *provider.Reference, sessionID string, info UploadInfo) (*PrepareUploadResult, error)
|
||||
// RollbackUpload reverts node state after a failed or aborted postprocessing run.
|
||||
// It is the inverse of PrepareUpload: restores previous metadata and reverts the optimistic
|
||||
// size propagation. The caller (coordinator) is responsible for unmarking the processing flag
|
||||
// and deleting the upload session files. Drivers that performed no work in PrepareUpload may return nil.
|
||||
// Callers must keep the session on a returned error: the rollback is retryable and info
|
||||
// carries state the driver cannot recover once the session files are gone.
|
||||
RollbackUpload(ctx context.Context, ref *provider.Reference, sessionID string, info RollbackInfo) error
|
||||
|
||||
// Revisions
|
||||
|
||||
@@ -176,15 +223,57 @@ type SpaceDisabledListableFS interface {
|
||||
type UploadSource struct {
|
||||
Body io.ReadCloser
|
||||
Length int64
|
||||
// Path is the staged file, if the caller has one. Drivers may move it into
|
||||
// place instead of copying Body; the file is gone after a successful commit.
|
||||
Path string
|
||||
|
||||
// ScanResult is the antivirus verdict: empty means clean.
|
||||
ScanResult string
|
||||
// ScanDate is zero when the upload was not scanned.
|
||||
ScanDate time.Time
|
||||
}
|
||||
|
||||
// UnscopeFunc is a function that unscopes a user
|
||||
type UnscopeFunc func()
|
||||
|
||||
// Composable is the interface that a struct needs to implement
|
||||
// to be composable, so that it can support the TUS methods
|
||||
type ComposableFS interface {
|
||||
UseIn(composer *tusd.StoreComposer)
|
||||
// CapabilitiesOpaqueKey is the StorageSpace/ProviderInfo Opaque key under which
|
||||
// a provider's Capabilities are carried as JSON up to the OCS capabilities handler.
|
||||
const CapabilitiesOpaqueKey = "storage-capabilities"
|
||||
|
||||
// Capabilities is a storage driver's self-declaration of the write-shaped
|
||||
// actions it supports. A false key means unsupported: no merge, no inheritance.
|
||||
type Capabilities struct {
|
||||
Upload bool
|
||||
CreateContainer bool
|
||||
Delete bool
|
||||
Move bool
|
||||
Versioning bool
|
||||
Trash bool
|
||||
Locking bool
|
||||
Sharing bool
|
||||
ArbitraryMetadata bool
|
||||
}
|
||||
|
||||
// FullCapabilities is the default for drivers that do not implement
|
||||
// CapabilityProvider, so existing drivers keep their current behavior.
|
||||
func FullCapabilities() Capabilities {
|
||||
return Capabilities{
|
||||
Upload: true,
|
||||
CreateContainer: true,
|
||||
Delete: true,
|
||||
Move: true,
|
||||
Versioning: true,
|
||||
Trash: true,
|
||||
Locking: true,
|
||||
Sharing: true,
|
||||
ArbitraryMetadata: true,
|
||||
}
|
||||
}
|
||||
|
||||
// CapabilityProvider is an optional interface a storage.FS may implement to
|
||||
// declare its capabilities.
|
||||
type CapabilityProvider interface {
|
||||
Capabilities(ctx context.Context) Capabilities
|
||||
}
|
||||
|
||||
// Registry is the interface that storage registries implement
|
||||
|
||||
+26
-20
@@ -20,7 +20,6 @@ package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
@@ -28,28 +27,17 @@ import (
|
||||
tusd "github.com/tus/tusd/v2/pkg/handler"
|
||||
)
|
||||
|
||||
// UploadFinishedFunc is a callback function used in storage drivers to indicate that an upload has finished
|
||||
type UploadFinishedFunc func(spaceOwner, executant *userpb.UserId, ref *provider.Reference)
|
||||
type skipTouchPropagationKey struct{}
|
||||
|
||||
// UploadRequest us used in FS.Upload() to carry required upload metadata
|
||||
type UploadRequest struct {
|
||||
Ref *provider.Reference
|
||||
Body io.ReadCloser
|
||||
Length int64
|
||||
// ContextSkipTouchPropagation tells TouchFile not to propagate.
|
||||
func ContextSkipTouchPropagation(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, skipTouchPropagationKey{}, true)
|
||||
}
|
||||
|
||||
// UploadsManager defines the interface for storage drivers that allow for managing uploads
|
||||
//
|
||||
// Deprecated: No longer used. Storage drivers should implement the UploadSessionLister.
|
||||
type UploadsManager interface {
|
||||
ListUploads() ([]tusd.FileInfo, error)
|
||||
PurgeExpiredUploads(chan<- tusd.FileInfo) error
|
||||
}
|
||||
|
||||
// UploadSessionLister defines the interface for FS implementations that allow listing and purging upload sessions
|
||||
type UploadSessionLister interface {
|
||||
// ListUploadSessions returns the upload sessions matching the given filter
|
||||
ListUploadSessions(ctx context.Context, filter UploadSessionFilter) ([]UploadSession, error)
|
||||
// SkipTouchPropagation reports whether TouchFile should skip propagation.
|
||||
func SkipTouchPropagation(ctx context.Context) bool {
|
||||
skip, _ := ctx.Value(skipTouchPropagationKey{}).(bool)
|
||||
return skip
|
||||
}
|
||||
|
||||
// UploadSessionResolver defines the interface for FS implementations that can resolve a finished
|
||||
@@ -66,6 +54,20 @@ type UploadSessionResolver interface {
|
||||
ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error)
|
||||
}
|
||||
|
||||
// OrphanChecker defines the interface for FS implementations that can resolve a resource's metadata.
|
||||
type OrphanChecker interface {
|
||||
// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable.
|
||||
IsOrphaned(ctx context.Context, ref *provider.Reference) bool
|
||||
}
|
||||
|
||||
// NodeCreator defines the interface for FS implementations whose PrepareUpload
|
||||
// creates a new file's node from UploadInfo.ParentID and Name, so an upload need
|
||||
// not TouchFile it first.
|
||||
type NodeCreator interface {
|
||||
// PrepareCreatesNode reports whether PrepareUpload creates a missing node.
|
||||
PrepareCreatesNode() bool
|
||||
}
|
||||
|
||||
// UploadSession is the interface that storage drivers need to return whan listing upload sessions.
|
||||
type UploadSession interface {
|
||||
// ID returns the upload id
|
||||
@@ -107,4 +109,8 @@ type UploadSessionFilter struct {
|
||||
Processing *bool
|
||||
Expired *bool
|
||||
HasVirus *bool
|
||||
// Orphaned filters sessions by whether their target node can still be
|
||||
// resolved. Evaluating it requires reading the node metadata of every
|
||||
// session, so it is only evaluated when set.
|
||||
Orphaned *bool
|
||||
}
|
||||
+39
@@ -201,6 +201,45 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (bool, string, er
|
||||
return true, assembledFileName, nil
|
||||
}
|
||||
|
||||
// Assemble saves an intermediate chunk and, once all chunks have arrived,
|
||||
// assembles them into a single temp file. Returns (reader, size, true, nil)
|
||||
// when the transfer is complete. Returns (nil, 0, false, nil) when more
|
||||
// chunks are still expected (partial transfer)
|
||||
// the returned reader removes the temp file on Close
|
||||
func (c *ChunkHandler) Assemble(chunkName string, body io.ReadCloser) (io.ReadCloser, int64, bool, error) {
|
||||
origPath, assembledPath, err := c.WriteChunk(chunkName, body)
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
if origPath == "" {
|
||||
return nil, 0, false, nil // partial
|
||||
}
|
||||
f, err := os.Open(assembledPath)
|
||||
if err != nil {
|
||||
_ = os.RemoveAll(assembledPath)
|
||||
return nil, 0, false, err
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
_ = os.RemoveAll(assembledPath)
|
||||
return nil, 0, false, err
|
||||
}
|
||||
return &assembledFile{File: f, path: assembledPath}, fi.Size(), true, nil
|
||||
}
|
||||
|
||||
// assembledFile wraps an *os.File and removes the temp file on Close.
|
||||
type assembledFile struct {
|
||||
*os.File
|
||||
path string
|
||||
}
|
||||
|
||||
func (a *assembledFile) Close() error {
|
||||
err := a.File.Close()
|
||||
_ = os.RemoveAll(a.path)
|
||||
return err
|
||||
}
|
||||
|
||||
// WriteChunk saves an intermediate chunk temporarily and assembles all chunks
|
||||
// once the final one is received.
|
||||
func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) {
|
||||
|
||||
Generated
Vendored
-38
@@ -1,38 +0,0 @@
|
||||
// Copyright 2018-2024 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package aspects
|
||||
|
||||
import (
|
||||
"github.com/opencloud-eu/reva/v2/pkg/events"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper"
|
||||
)
|
||||
|
||||
// Aspects holds dependencies for handling aspects of the decomposedfs
|
||||
type Aspects struct {
|
||||
Lookup node.PathLookup
|
||||
Tree node.Tree
|
||||
Trashbin trashbin.Trashbin
|
||||
Permissions permissions.Permissions
|
||||
EventStream events.Stream
|
||||
DisableVersioning bool
|
||||
UserMapper usermapper.Mapper
|
||||
}
|
||||
Generated
Vendored
-1359
File diff suppressed because it is too large.
Load diff
-361
@@ -1,361 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package decomposedfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
)
|
||||
|
||||
// DenyGrant denies access to a resource.
|
||||
func (fs *Decomposedfs) DenyGrant(ctx context.Context, ref *provider.Reference, grantee *provider.Grantee) error {
|
||||
_, span := tracer.Start(ctx, "DenyGrant")
|
||||
defer span.End()
|
||||
log := appctx.GetLogger(ctx)
|
||||
|
||||
log.Debug().Interface("ref", ref).Interface("grantee", grantee).Msg("DenyGrant()")
|
||||
|
||||
grantNode, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !grantNode.Exists {
|
||||
return errtypes.NotFound(filepath.Join(grantNode.ParentID, grantNode.Name))
|
||||
}
|
||||
|
||||
// set all permissions to false
|
||||
grant := &provider.Grant{
|
||||
Grantee: grantee,
|
||||
Permissions: &provider.ResourcePermissions{},
|
||||
}
|
||||
|
||||
// add acting user
|
||||
u := ctxpkg.ContextMustGetUser(ctx)
|
||||
grant.Creator = u.GetId()
|
||||
|
||||
rp, err := fs.p.AssemblePermissions(ctx, grantNode)
|
||||
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.DenyGrant:
|
||||
return errtypes.PermissionDenied(filepath.Join(grantNode.ParentID, grantNode.Name))
|
||||
}
|
||||
|
||||
return fs.storeGrant(ctx, grantNode, grant)
|
||||
}
|
||||
|
||||
// AddGrant adds a grant to a resource
|
||||
func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
|
||||
_, span := tracer.Start(ctx, "AddGrant")
|
||||
defer span.End()
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()")
|
||||
grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = unlockFunc()
|
||||
}()
|
||||
|
||||
if grant != nil {
|
||||
return errtypes.AlreadyExists(filepath.Join(grantNode.ParentID, grantNode.Name))
|
||||
}
|
||||
|
||||
owner := grantNode.Owner()
|
||||
grants, err := grantNode.ListGrants(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If the owner is empty and there are no grantees then we are dealing with a just created project space.
|
||||
// In this case we don't need to check for permissions and just add the grant since this will be the project
|
||||
// manager.
|
||||
// When the owner is empty but grants are set then we do want to check the grants.
|
||||
// However, if we are trying to edit an existing grant we do not have to check for permission if the user owns the grant
|
||||
// TODO: find a better to check this
|
||||
if len(grants) != 0 || (owner != nil && owner.OpaqueId != "" && (owner.OpaqueId != grantNode.SpaceID || owner.Type != 8)) {
|
||||
rp, err := fs.p.AssemblePermissions(ctx, grantNode)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.AddGrant:
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
}
|
||||
// check processing
|
||||
if grantNode.IsProcessing(ctx) {
|
||||
return errtypes.ResourceProcessing(ref.String())
|
||||
}
|
||||
return fs.storeGrant(ctx, grantNode, g)
|
||||
}
|
||||
|
||||
// ListGrants lists the grants on the specified resource
|
||||
func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) {
|
||||
_, span := tracer.Start(ctx, "ListGrants")
|
||||
defer span.End()
|
||||
var grantNode *node.Node
|
||||
if grantNode, err = fs.lu.NodeFromResource(ctx, ref); err != nil {
|
||||
return
|
||||
}
|
||||
if !grantNode.Exists {
|
||||
err = errtypes.NotFound(filepath.Join(grantNode.ParentID, grantNode.Name))
|
||||
return
|
||||
}
|
||||
rp, err := fs.p.AssemblePermissions(ctx, grantNode)
|
||||
switch {
|
||||
case err != nil:
|
||||
return nil, err
|
||||
case !rp.ListGrants && !rp.Stat:
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
return nil, errtypes.NotFound(f)
|
||||
}
|
||||
log := appctx.GetLogger(ctx)
|
||||
var attrs node.Attributes
|
||||
if attrs, err = grantNode.Xattrs(ctx); err != nil {
|
||||
log.Error().Err(err).Msg("error listing attributes")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
aces := []*ace.ACE{}
|
||||
for k, v := range attrs {
|
||||
if strings.HasPrefix(k, prefixes.GrantPrefix) {
|
||||
var err error
|
||||
var e *ace.ACE
|
||||
principal := k[len(prefixes.GrantPrefix):]
|
||||
if e, err = ace.Unmarshal(principal, v); err != nil {
|
||||
log.Error().Err(err).Str("principal", principal).Str("attr", k).Msg("could not unmarshal ace")
|
||||
continue
|
||||
}
|
||||
aces = append(aces, e)
|
||||
}
|
||||
}
|
||||
|
||||
uid := ctxpkg.ContextMustGetUser(ctx).GetId()
|
||||
grants = make([]*provider.Grant, 0, len(aces))
|
||||
for i := range aces {
|
||||
g := aces[i].Grant()
|
||||
|
||||
// you may list your own grants even without listgrants permission
|
||||
if !rp.ListGrants && !utils.UserIDEqual(g.Creator, uid) && !utils.UserIDEqual(g.Grantee.GetUserId(), uid) {
|
||||
continue
|
||||
}
|
||||
|
||||
grants = append(grants, g)
|
||||
}
|
||||
|
||||
return grants, nil
|
||||
}
|
||||
|
||||
// RemoveGrant removes a grant from resource
|
||||
func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) {
|
||||
_, span := tracer.Start(ctx, "RemoveGrant")
|
||||
defer span.End()
|
||||
grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = unlockFunc()
|
||||
}()
|
||||
|
||||
if grant == nil {
|
||||
return errtypes.NotFound("grant not found")
|
||||
}
|
||||
|
||||
// you are allowed to remove grants if you created them yourself or have the proper permission
|
||||
if !utils.UserIDEqual(grant.Creator, ctxpkg.ContextMustGetUser(ctx).GetId()) {
|
||||
rp, err := fs.p.AssemblePermissions(ctx, grantNode)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.RemoveGrant:
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
}
|
||||
// check processing
|
||||
if grantNode.IsProcessing(ctx) {
|
||||
return errtypes.ResourceProcessing(ref.String())
|
||||
}
|
||||
if err := grantNode.DeleteGrant(ctx, g, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if isShareGrant(ctx) {
|
||||
// do not invalidate by user or group indexes
|
||||
// FIXME we should invalidate the by-type index, but that requires reference counting
|
||||
} else {
|
||||
// invalidate space grant
|
||||
switch g.Grantee.Type {
|
||||
case provider.GranteeType_GRANTEE_TYPE_USER:
|
||||
// remove from user index
|
||||
if err := fs.userSpaceIndex.Remove(g.Grantee.GetUserId().GetOpaqueId(), grantNode.SpaceID); err != nil {
|
||||
return err
|
||||
}
|
||||
case provider.GranteeType_GRANTEE_TYPE_GROUP:
|
||||
// remove from group index
|
||||
if err := fs.groupSpaceIndex.Remove(g.Grantee.GetGroupId().GetOpaqueId(), grantNode.SpaceID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fs.tp.Propagate(ctx, grantNode, 0)
|
||||
}
|
||||
|
||||
func isShareGrant(ctx context.Context) bool {
|
||||
_, ok := storageprovider.SpaceTypeFromContext(ctx)
|
||||
return !ok
|
||||
}
|
||||
|
||||
// UpdateGrant updates a grant on a resource
|
||||
// TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92
|
||||
func (fs *Decomposedfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error {
|
||||
_, span := tracer.Start(ctx, "UpdateGrant")
|
||||
defer span.End()
|
||||
log := appctx.GetLogger(ctx)
|
||||
log.Debug().Interface("ref", ref).Interface("grant", g).Msg("UpdateGrant()")
|
||||
|
||||
grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = unlockFunc()
|
||||
}()
|
||||
|
||||
if grant == nil {
|
||||
// grant not found
|
||||
// TODO: fallback to AddGrant?
|
||||
return errtypes.NotFound(g.Grantee.GetUserId().GetOpaqueId())
|
||||
}
|
||||
|
||||
// You may update a grant when you have the UpdateGrant permission or created the grant (regardless what your permissions are now)
|
||||
if !utils.UserIDEqual(grant.Creator, ctxpkg.ContextMustGetUser(ctx).GetId()) {
|
||||
rp, err := fs.p.AssemblePermissions(ctx, grantNode)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.UpdateGrant:
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
}
|
||||
|
||||
// check processing
|
||||
if grantNode.IsProcessing(ctx) {
|
||||
return errtypes.ResourceProcessing(ref.String())
|
||||
}
|
||||
return fs.storeGrant(ctx, grantNode, g)
|
||||
}
|
||||
|
||||
// checks if the given grant exists and returns it. Nil grant means it doesn't exist
|
||||
func (fs *Decomposedfs) loadGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (*node.Node, metadata.UnlockFunc, *provider.Grant, error) {
|
||||
_, span := tracer.Start(ctx, "loadGrant")
|
||||
defer span.End()
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if !n.Exists {
|
||||
return nil, nil, nil, errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
|
||||
}
|
||||
|
||||
// lock the metadata file
|
||||
unlockFunc, err := fs.lu.MetadataBackend().Lock(n.InternalPath())
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
grants, err := n.ListGrants(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
for _, grant := range grants {
|
||||
switch grant.Grantee.GetType() {
|
||||
case provider.GranteeType_GRANTEE_TYPE_USER:
|
||||
if g.Grantee.GetUserId().GetOpaqueId() == grant.Grantee.GetUserId().GetOpaqueId() {
|
||||
return n, unlockFunc, grant, nil
|
||||
}
|
||||
case provider.GranteeType_GRANTEE_TYPE_GROUP:
|
||||
if g.Grantee.GetGroupId().GetOpaqueId() == grant.Grantee.GetGroupId().GetOpaqueId() {
|
||||
return n, unlockFunc, grant, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return n, unlockFunc, nil, nil
|
||||
}
|
||||
|
||||
func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provider.Grant) error {
|
||||
_, span := tracer.Start(ctx, "storeGrant")
|
||||
defer span.End()
|
||||
// if is a grant to a space root, the receiver needs the space type to update the indexes
|
||||
spaceType, ok := storageprovider.SpaceTypeFromContext(ctx)
|
||||
if !ok {
|
||||
// this is not a grant on a space root we are just adding a share
|
||||
spaceType = spaceTypeShare
|
||||
}
|
||||
|
||||
// set the grant
|
||||
e := ace.FromGrant(g)
|
||||
principal, value := e.Marshal()
|
||||
attribs := node.Attributes{
|
||||
prefixes.GrantPrefix + principal: value,
|
||||
}
|
||||
if err := n.SetXattrsWithContext(ctx, attribs, false); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).
|
||||
Str("principal", principal).Msg("Could not set grant for principal")
|
||||
return err
|
||||
}
|
||||
|
||||
// update the indexes only after successfully setting the grant
|
||||
err := fs.updateIndexes(ctx, g.GetGrantee(), spaceType, n.SpaceID, n.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return fs.tp.Propagate(ctx, n, 0)
|
||||
}
|
||||
Generated
Vendored
-412
@@ -1,412 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package lookup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/google/uuid"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
const (
|
||||
_spaceTypePersonal = "personal"
|
||||
)
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/lookup")
|
||||
}
|
||||
|
||||
// Lookup implements transformations from filepath to node and back
|
||||
type Lookup struct {
|
||||
Options *options.Options
|
||||
|
||||
metadataBackend metadata.Backend
|
||||
tm node.TimeManager
|
||||
}
|
||||
|
||||
// New returns a new Lookup instance
|
||||
func New(b metadata.Backend, o *options.Options, tm node.TimeManager) *Lookup {
|
||||
return &Lookup{
|
||||
Options: o,
|
||||
metadataBackend: b,
|
||||
tm: tm,
|
||||
}
|
||||
}
|
||||
|
||||
// MetadataBackend returns the metadata backend
|
||||
func (lu *Lookup) MetadataBackend() metadata.Backend {
|
||||
return lu.metadataBackend
|
||||
}
|
||||
|
||||
func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, attrs node.Attributes) (string, int64, error) {
|
||||
blobID := ""
|
||||
blobSize := int64(0)
|
||||
var err error
|
||||
|
||||
if attrs != nil {
|
||||
blobID = attrs.String(prefixes.BlobIDAttr)
|
||||
if blobID != "" {
|
||||
blobSize, err = attrs.Int64(prefixes.BlobsizeAttr)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
attrs, err := lu.metadataBackend.All(ctx, path)
|
||||
if err != nil {
|
||||
return "", 0, errors.Wrapf(err, "error reading blobid xattr")
|
||||
}
|
||||
nodeAttrs := node.Attributes(attrs)
|
||||
blobID = nodeAttrs.String(prefixes.BlobIDAttr)
|
||||
blobSize, err = nodeAttrs.Int64(prefixes.BlobsizeAttr)
|
||||
if err != nil {
|
||||
return "", 0, errors.Wrapf(err, "error reading blobsize xattr")
|
||||
}
|
||||
}
|
||||
return blobID, blobSize, nil
|
||||
}
|
||||
|
||||
func readChildNodeFromLink(path string) (string, error) {
|
||||
link, err := os.Readlink(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
nodeID := strings.TrimLeft(link, "/.")
|
||||
nodeID = strings.ReplaceAll(nodeID, "/", "")
|
||||
return nodeID, nil
|
||||
}
|
||||
|
||||
func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) {
|
||||
nodeID, err := readChildNodeFromLink(filepath.Join(parent.InternalPath(), name))
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "decomposedfs: Wrap: readlink error")
|
||||
}
|
||||
return nodeID, nil
|
||||
}
|
||||
|
||||
// TypeFromPath returns the type of the node at the given path
|
||||
func (lu *Lookup) TypeFromPath(ctx context.Context, path string) provider.ResourceType {
|
||||
// Try to read from xattrs
|
||||
typeAttr, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.TypeAttr)
|
||||
if err == nil {
|
||||
return provider.ResourceType(int32(typeAttr))
|
||||
}
|
||||
|
||||
t := provider.ResourceType_RESOURCE_TYPE_INVALID
|
||||
// Fall back to checking on disk
|
||||
fi, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return t
|
||||
}
|
||||
|
||||
switch {
|
||||
case fi.IsDir():
|
||||
if _, err = lu.metadataBackend.Get(ctx, path, prefixes.ReferenceAttr); err == nil {
|
||||
t = provider.ResourceType_RESOURCE_TYPE_REFERENCE
|
||||
} else {
|
||||
t = provider.ResourceType_RESOURCE_TYPE_CONTAINER
|
||||
}
|
||||
case fi.Mode().IsRegular():
|
||||
t = provider.ResourceType_RESOURCE_TYPE_FILE
|
||||
case fi.Mode()&os.ModeSymlink != 0:
|
||||
t = provider.ResourceType_RESOURCE_TYPE_SYMLINK
|
||||
// TODO reference using ext attr on a symlink
|
||||
// nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// NodeFromResource takes in a request path or request id and converts it to a Node
|
||||
func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) (*node.Node, error) {
|
||||
ctx, span := tracer.Start(ctx, "NodeFromResource")
|
||||
defer span.End()
|
||||
|
||||
if ref.ResourceId != nil {
|
||||
// check if a storage space reference is used
|
||||
// currently, the decomposed fs uses the root node id as the space id
|
||||
n, err := lu.NodeFromID(ctx, ref.ResourceId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// is this a relative reference?
|
||||
if ref.Path != "" {
|
||||
p := filepath.Clean(ref.Path)
|
||||
if p != "." && p != "/" {
|
||||
// walk the relative path
|
||||
n, err = lu.WalkPath(ctx, n, p, false, func(ctx context.Context, n *node.Node) error { return nil })
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
n.SpaceID = ref.ResourceId.SpaceId
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// reference is invalid
|
||||
return nil, fmt.Errorf("invalid reference %+v. resource_id must be set", ref)
|
||||
}
|
||||
|
||||
// NodeFromID returns the internal path for the id
|
||||
func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *node.Node, err error) {
|
||||
ctx, span := tracer.Start(ctx, "NodeFromID")
|
||||
defer span.End()
|
||||
if id == nil {
|
||||
return nil, fmt.Errorf("invalid resource id %+v", id)
|
||||
}
|
||||
if id.OpaqueId == "" {
|
||||
// The Resource references the root of a space
|
||||
return lu.NodeFromSpaceID(ctx, id.SpaceId)
|
||||
}
|
||||
return node.ReadNode(ctx, lu, id.SpaceId, id.OpaqueId, false, nil, false)
|
||||
}
|
||||
|
||||
// Pathify segments the beginning of a string into depth segments of width length
|
||||
// Pathify("aabbccdd", 3, 1) will return "a/a/b/bccdd"
|
||||
func Pathify(id string, depth, width int) string {
|
||||
b := strings.Builder{}
|
||||
i := 0
|
||||
for ; i < depth; i++ {
|
||||
if len(id) <= i*width+width {
|
||||
break
|
||||
}
|
||||
b.WriteString(id[i*width : i*width+width])
|
||||
b.WriteRune(filepath.Separator)
|
||||
}
|
||||
b.WriteString(id[i*width:])
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// NodeFromSpaceID converts a resource id into a Node
|
||||
func (lu *Lookup) NodeFromSpaceID(ctx context.Context, spaceID string) (n *node.Node, err error) {
|
||||
node, err := node.ReadNode(ctx, lu, spaceID, spaceID, false, nil, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
node.SpaceRoot = node
|
||||
return node, nil
|
||||
}
|
||||
|
||||
// GenerateSpaceID generates a new space id and alias
|
||||
func (lu *Lookup) GenerateSpaceID(spaceType string, owner *user.User) (string, error) {
|
||||
switch spaceType {
|
||||
case _spaceTypePersonal:
|
||||
return owner.Id.OpaqueId, nil
|
||||
default:
|
||||
return uuid.New().String(), nil
|
||||
}
|
||||
}
|
||||
|
||||
// Path returns the path for node
|
||||
func (lu *Lookup) Path(ctx context.Context, n *node.Node, hasPermission node.PermissionFunc) (p string, err error) {
|
||||
root := n.SpaceRoot
|
||||
var child *node.Node
|
||||
for n.ID != root.ID {
|
||||
p = filepath.Join(n.Name, p)
|
||||
child = n
|
||||
if n, err = n.Parent(ctx); err != nil {
|
||||
appctx.GetLogger(ctx).
|
||||
Error().Err(err).
|
||||
Str("path", p).
|
||||
Str("spaceid", child.SpaceID).
|
||||
Str("nodeid", child.ID).
|
||||
Str("parentid", child.ParentID).
|
||||
Msg("Path()")
|
||||
return
|
||||
}
|
||||
|
||||
if !hasPermission(n) {
|
||||
break
|
||||
}
|
||||
}
|
||||
p = filepath.Join("/", p)
|
||||
return
|
||||
}
|
||||
|
||||
// WalkPath calls n.Child(segment) on every path segment in p starting at the node r.
|
||||
// If a function f is given it will be executed for every segment node, but not the root node r.
|
||||
// If followReferences is given the current visited reference node is replaced by the referenced node.
|
||||
func (lu *Lookup) WalkPath(ctx context.Context, r *node.Node, p string, followReferences bool, f func(ctx context.Context, n *node.Node) error) (*node.Node, error) {
|
||||
segments := strings.Split(strings.Trim(p, "/"), "/")
|
||||
var err error
|
||||
for i := range segments {
|
||||
if r, err = r.Child(ctx, segments[i]); err != nil {
|
||||
return r, err
|
||||
}
|
||||
|
||||
if followReferences {
|
||||
if attrBytes, err := r.Xattr(ctx, prefixes.ReferenceAttr); err == nil {
|
||||
realNodeID := attrBytes
|
||||
ref, err := refFromCS3(realNodeID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err = lu.NodeFromID(ctx, ref.ResourceId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if r.IsSpaceRoot(ctx) {
|
||||
r.SpaceRoot = r
|
||||
}
|
||||
|
||||
if !r.Exists && i < len(segments)-1 {
|
||||
return r, errtypes.NotFound(segments[i])
|
||||
}
|
||||
if f != nil {
|
||||
if err = f(ctx, r); err != nil {
|
||||
return r, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// InternalRoot returns the internal storage root directory
|
||||
func (lu *Lookup) InternalRoot() string {
|
||||
return lu.Options.Root
|
||||
}
|
||||
|
||||
// InternalPath returns the internal path for a given ID
|
||||
func (lu *Lookup) InternalPath(spaceID, nodeID string) string {
|
||||
return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2))
|
||||
}
|
||||
|
||||
// // ReferenceFromAttr returns a CS3 reference from xattr of a node.
|
||||
// // Supported formats are: "cs3:storageid/nodeid"
|
||||
// func ReferenceFromAttr(b []byte) (*provider.Reference, error) {
|
||||
// return refFromCS3(b)
|
||||
// }
|
||||
|
||||
// refFromCS3 creates a CS3 reference from a set of bytes. This method should remain private
|
||||
// and only be called after validation because it can potentially panic.
|
||||
func refFromCS3(b []byte) (*provider.Reference, error) {
|
||||
parts := string(b[4:])
|
||||
return &provider.Reference{
|
||||
ResourceId: &provider.ResourceId{
|
||||
StorageId: strings.Split(parts, "/")[0],
|
||||
OpaqueId: strings.Split(parts, "/")[1],
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyMetadata copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a shared lock is acquired.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadata(ctx context.Context, src, target string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) {
|
||||
// Acquire a read log on the source node
|
||||
// write lock existing node before reading treesize or tree time
|
||||
lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(src), os.O_RDONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "xattrs: Unable to lock source to read")
|
||||
}
|
||||
defer func() {
|
||||
rerr := lock.Close()
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
return lu.CopyMetadataWithSourceLock(ctx, src, target, filter, lock, acquireTargetLock)
|
||||
}
|
||||
|
||||
// CopyMetadataWithSourceLock copies all extended attributes from source to target.
|
||||
// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix
|
||||
// For the source file, a matching lockedfile is required.
|
||||
// NOTE: target resource will be write locked!
|
||||
func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourcePath, targetPath string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) {
|
||||
switch {
|
||||
case lockedSource == nil:
|
||||
return errors.New("no lock provided")
|
||||
case lockedSource.Name() != lu.MetadataBackend().LockfilePath(sourcePath):
|
||||
return errors.New("lockpath does not match filepath")
|
||||
}
|
||||
|
||||
attrs, err := lu.metadataBackend.All(ctx, sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newAttrs := make(map[string][]byte, 0)
|
||||
for attrName, val := range attrs {
|
||||
if filter != nil {
|
||||
var ok bool
|
||||
if val, ok = filter(attrName, val); !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
newAttrs[attrName] = val
|
||||
}
|
||||
|
||||
return lu.MetadataBackend().SetMultiple(ctx, targetPath, newAttrs, acquireTargetLock)
|
||||
}
|
||||
|
||||
// TimeManager returns the time manager
|
||||
func (lu *Lookup) TimeManager() node.TimeManager {
|
||||
return lu.tm
|
||||
}
|
||||
|
||||
// DetectBackendOnDisk returns the name of the metadata backend being used on disk
|
||||
func DetectBackendOnDisk(root string) string {
|
||||
matches, _ := filepath.Glob(filepath.Join(root, "spaces", "*", "*"))
|
||||
if len(matches) > 0 {
|
||||
base := matches[len(matches)-1]
|
||||
spaceid := strings.ReplaceAll(
|
||||
strings.TrimPrefix(base, filepath.Join(root, "spaces")),
|
||||
"/", "")
|
||||
spaceRoot := Pathify(spaceid, 4, 2)
|
||||
_, err := os.Stat(filepath.Join(base, "nodes", spaceRoot+".mpk"))
|
||||
if err == nil {
|
||||
return "mpk"
|
||||
}
|
||||
_, err = os.Stat(filepath.Join(base, "nodes", spaceRoot+".ini"))
|
||||
if err == nil {
|
||||
return "ini"
|
||||
}
|
||||
}
|
||||
return "xattrs"
|
||||
}
|
||||
Generated
Vendored
-224
@@ -1,224 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package decomposedfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storagespace"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// SetArbitraryMetadata sets the metadata on a resource
|
||||
func (fs *Decomposedfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) {
|
||||
_, span := tracer.Start(ctx, "SetArbitraryMetadata")
|
||||
defer span.End()
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: error resolving ref")
|
||||
}
|
||||
sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger()
|
||||
|
||||
if !n.Exists {
|
||||
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
|
||||
return err
|
||||
}
|
||||
|
||||
rp, err := fs.p.AssemblePermissions(ctx, n)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.InitiateFileUpload: // TODO add explicit SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
|
||||
// Set space owner in context
|
||||
storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx))
|
||||
|
||||
// check lock
|
||||
if err := n.CheckLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errs := []error{}
|
||||
// TODO should we really continue updating when an error occurs?
|
||||
if md.Metadata != nil {
|
||||
if val, ok := md.Metadata["mtime"]; ok {
|
||||
delete(md.Metadata, "mtime")
|
||||
if err := n.SetMtimeString(ctx, val); err != nil {
|
||||
errs = append(errs, errors.Wrap(err, "could not set mtime"))
|
||||
}
|
||||
}
|
||||
// TODO(jfd) special handling for atime?
|
||||
// TODO(jfd) allow setting birth time (btime)?
|
||||
// TODO(jfd) any other metadata that is interesting? fileid?
|
||||
// TODO unset when file is updated
|
||||
// TODO unset when folder is updated or add timestamp to etag?
|
||||
if val, ok := md.Metadata["etag"]; ok {
|
||||
delete(md.Metadata, "etag")
|
||||
if err := n.SetEtag(ctx, val); err != nil {
|
||||
errs = append(errs, errors.Wrap(err, "could not set etag"))
|
||||
}
|
||||
}
|
||||
if val, ok := md.Metadata[node.FavoriteKey]; ok {
|
||||
delete(md.Metadata, node.FavoriteKey)
|
||||
if u, ok := ctxpkg.ContextGetUser(ctx); ok {
|
||||
if uid := u.GetId(); uid != nil {
|
||||
if err := n.SetFavorite(ctx, uid, val); err != nil {
|
||||
sublog.Error().Err(err).
|
||||
Interface("user", u).
|
||||
Msg("could not set favorite flag")
|
||||
errs = append(errs, errors.Wrap(err, "could not set favorite flag"))
|
||||
}
|
||||
} else {
|
||||
sublog.Error().Interface("user", u).Msg("user has no id")
|
||||
errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id"))
|
||||
}
|
||||
} else {
|
||||
sublog.Error().Interface("user", u).Msg("error getting user from ctx")
|
||||
errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx"))
|
||||
}
|
||||
}
|
||||
}
|
||||
for k, v := range md.Metadata {
|
||||
attrName := prefixes.MetadataPrefix + k
|
||||
if err = n.SetXattrString(ctx, attrName, v); err != nil {
|
||||
errs = append(errs, errors.Wrap(err, "Decomposedfs: could not set metadata attribute "+attrName+" to "+k))
|
||||
}
|
||||
}
|
||||
|
||||
switch len(errs) {
|
||||
case 0:
|
||||
return fs.tp.Propagate(ctx, n, 0)
|
||||
case 1:
|
||||
// TODO Propagate if anything changed
|
||||
return errs[0]
|
||||
default:
|
||||
// TODO Propagate if anything changed
|
||||
// TODO how to return multiple errors?
|
||||
return errors.New("multiple errors occurred, see log for details")
|
||||
}
|
||||
}
|
||||
|
||||
// UnsetArbitraryMetadata unsets the metadata on the given resource
|
||||
func (fs *Decomposedfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) {
|
||||
_, span := tracer.Start(ctx, "UnsetArbitraryMetadata")
|
||||
defer span.End()
|
||||
n, err := fs.lu.NodeFromResource(ctx, ref)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: error resolving ref")
|
||||
}
|
||||
sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger()
|
||||
|
||||
if !n.Exists {
|
||||
err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name))
|
||||
return err
|
||||
}
|
||||
|
||||
rp, err := fs.p.AssemblePermissions(ctx, n)
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case !rp.InitiateFileUpload: // TODO use SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91
|
||||
f, _ := storagespace.FormatReference(ref)
|
||||
if rp.Stat {
|
||||
return errtypes.PermissionDenied(f)
|
||||
}
|
||||
return errtypes.NotFound(f)
|
||||
}
|
||||
|
||||
// Set space owner in context
|
||||
storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx))
|
||||
|
||||
// check lock
|
||||
if err := n.CheckLock(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
errs := []error{}
|
||||
for _, k := range keys {
|
||||
switch k {
|
||||
case node.FavoriteKey:
|
||||
// the favorite flag is specific to the user, so we need to incorporate the userid
|
||||
u, ok := ctxpkg.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
sublog.Error().
|
||||
Interface("user", u).
|
||||
Msg("error getting user from ctx")
|
||||
errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx"))
|
||||
continue
|
||||
}
|
||||
var uid *userpb.UserId
|
||||
if uid = u.GetId(); uid == nil || uid.OpaqueId == "" {
|
||||
sublog.Error().
|
||||
Interface("user", u).
|
||||
Msg("user has no id")
|
||||
errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id"))
|
||||
continue
|
||||
}
|
||||
fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp())
|
||||
if err := n.RemoveXattr(ctx, fa, true); err != nil {
|
||||
if metadata.IsAttrUnset(err) {
|
||||
continue // already gone, ignore
|
||||
}
|
||||
sublog.Error().Err(err).
|
||||
Interface("user", u).
|
||||
Str("key", fa).
|
||||
Msg("could not unset favorite flag")
|
||||
errs = append(errs, errors.Wrap(err, "could not unset favorite flag"))
|
||||
}
|
||||
default:
|
||||
if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k, true); err != nil {
|
||||
if metadata.IsAttrUnset(err) {
|
||||
continue // already gone, ignore
|
||||
}
|
||||
sublog.Error().Err(err).
|
||||
Str("key", k).
|
||||
Msg("could not unset metadata")
|
||||
errs = append(errs, errors.Wrap(err, "could not unset metadata"))
|
||||
}
|
||||
}
|
||||
}
|
||||
switch len(errs) {
|
||||
case 0:
|
||||
return fs.tp.Propagate(ctx, n, 0)
|
||||
case 1:
|
||||
// TODO Propagate if anything changed
|
||||
return errs[0]
|
||||
default:
|
||||
// TODO Propagate if anything changed
|
||||
// TODO how to return multiple errors?
|
||||
return errors.New("multiple errors occurred, see log for details")
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-99
@@ -1,99 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pkg/xattr"
|
||||
)
|
||||
|
||||
// IsNotExist checks if there is a os not exists error buried inside the xattr error,
|
||||
// as we cannot just use os.IsNotExist().
|
||||
func IsNotExist(err error) bool {
|
||||
if _, ok := err.(errtypes.IsNotFound); ok {
|
||||
return true
|
||||
}
|
||||
if os.IsNotExist(errors.Cause(err)) {
|
||||
return true
|
||||
}
|
||||
if xerr, ok := errors.Cause(err).(*xattr.Error); ok {
|
||||
if serr, ok2 := xerr.Err.(syscall.Errno); ok2 {
|
||||
return serr == syscall.ENOENT
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsErrRange checks for a syscall.ERANGE buried inside an xattr error. listxattr
|
||||
// returns ERANGE when the set of attribute names grows between the size-probe call
|
||||
// and the read call (a concurrent setxattr landed in between). This is transient and
|
||||
// self-clearing once the competing writer finishes its batch.
|
||||
func IsErrRange(err error) bool {
|
||||
if xerr, ok := errors.Cause(err).(*xattr.Error); ok {
|
||||
if serr, ok2 := xerr.Err.(syscall.Errno); ok2 {
|
||||
return serr == syscall.ERANGE
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsErrTooBig checks for a syscall.E2BIG buried inside an xattr error. listxattr
|
||||
// returns E2BIG when the attribute name list is larger than the maximum size the
|
||||
// kernel will hand out (the 64KiB VFS limit, e.g. on XFS). Unlike ERANGE this is
|
||||
// not a transient race: the list cannot be retrieved at all, so callers should
|
||||
// fail fast instead of retrying.
|
||||
func IsErrTooBig(err error) bool {
|
||||
if xerr, ok := errors.Cause(err).(*xattr.Error); ok {
|
||||
if serr, ok2 := xerr.Err.(syscall.Errno); ok2 {
|
||||
return serr == syscall.E2BIG
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsAttrUnset checks the xattr.ENOATTR from the xattr package which redifines it as ENODATA on platforms that do not natively support it (eg. linux)
|
||||
// see https://github.com/pkg/xattr/blob/8725d4ccc0fcef59c8d9f0eaf606b3c6f962467a/xattr_linux.go#L19-L22
|
||||
func IsAttrUnset(err error) bool {
|
||||
if xerr, ok := errors.Cause(err).(*xattr.Error); ok {
|
||||
if serr, ok2 := xerr.Err.(syscall.Errno); ok2 {
|
||||
return serr == xattr.ENOATTR
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// The os error is buried inside the fs.PathError error
|
||||
func IsNotDir(err error) bool {
|
||||
if perr, ok := errors.Cause(err).(*fs.PathError); ok {
|
||||
if serr, ok2 := perr.Err.(syscall.Errno); ok2 {
|
||||
return serr == syscall.ENOTDIR
|
||||
}
|
||||
}
|
||||
if xerr, ok := errors.Cause(err).(*xattr.Error); ok {
|
||||
if serr, ok2 := xerr.Err.(syscall.Errno); ok2 {
|
||||
return serr == syscall.ENOTDIR
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
Generated
Vendored
-55
@@ -1,55 +0,0 @@
|
||||
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
"github.com/pkg/xattr"
|
||||
)
|
||||
|
||||
// maxListXattrRetries bounds the number of attempts made by listXattr when it
|
||||
// keeps racing with a concurrent writer.
|
||||
const maxListXattrRetries = 10
|
||||
|
||||
// listXattr lists the extended attribute names of the file at path, retrying on
|
||||
// transient ERANGE errors.
|
||||
//
|
||||
// xattr.List is not atomic: internally it issues a listxattr syscall to learn
|
||||
// the size of the extended-attribute name list, allocates a buffer of that
|
||||
// size and issues a second listxattr to read the names into it. If another
|
||||
// goroutine or process adds an extended attribute to the same inode between the
|
||||
// two syscalls, the name list grows beyond the buffer and the kernel returns
|
||||
// ERANGE ("numerical result out of range").
|
||||
//
|
||||
// This is a transient condition: a fresh size query on the next attempt
|
||||
// reflects the new size, so we simply retry a bounded number of times. See the
|
||||
// posix fs watcher, whose assimilation can read a node's xattrs while the node
|
||||
// is still being created, for a concrete source of such concurrent writes.
|
||||
//
|
||||
// E2BIG is deliberately not retried: it means the name list is larger than the
|
||||
// maximum size the kernel will hand out (the 64KiB VFS limit, e.g. on XFS),
|
||||
// which is not a race and cannot be retried away. It is returned as-is.
|
||||
func listXattr(ctx context.Context, path string) ([]string, error) {
|
||||
var (
|
||||
attrs []string
|
||||
err error
|
||||
)
|
||||
for i := 0; i < maxListXattrRetries; i++ {
|
||||
attrs, err = xattr.List(path)
|
||||
if err == nil {
|
||||
if i > 0 {
|
||||
appctx.GetLogger(ctx).Debug().Str("path", path).Int("attempt", i+1).Msg("xattr.List ERANGE recovered after retry")
|
||||
}
|
||||
return attrs, nil
|
||||
}
|
||||
if !IsErrRange(err) {
|
||||
return attrs, err
|
||||
}
|
||||
appctx.GetLogger(ctx).Debug().Err(err).Str("path", path).Int("attempt", i+1).Msg("xattr.List ERANGE, retrying")
|
||||
}
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("path", path).Msg("xattr.List still failing after ERANGE retries")
|
||||
return attrs, err
|
||||
}
|
||||
Generated
Vendored
-330
@@ -1,330 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/google/renameio/v2"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
)
|
||||
|
||||
// MessagePackBackend persists the attributes in messagepack format inside the file
|
||||
type MessagePackBackend struct {
|
||||
rootPath string
|
||||
metaCache cache.FileMetadataCache
|
||||
}
|
||||
|
||||
type readWriteCloseSeekTruncater interface {
|
||||
io.ReadWriteCloser
|
||||
io.Seeker
|
||||
Truncate(int64) error
|
||||
}
|
||||
|
||||
// NewMessagePackBackend returns a new MessagePackBackend instance
|
||||
func NewMessagePackBackend(rootPath string, o cache.Config) MessagePackBackend {
|
||||
return MessagePackBackend{
|
||||
rootPath: filepath.Clean(rootPath),
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the backend
|
||||
func (MessagePackBackend) Name() string { return "messagepack" }
|
||||
|
||||
// All reads all extended attributes for a node
|
||||
func (b MessagePackBackend) All(ctx context.Context, path string) (map[string][]byte, error) {
|
||||
return b.loadAttributes(ctx, path, nil)
|
||||
}
|
||||
|
||||
// Get an extended attribute value for the given key
|
||||
func (b MessagePackBackend) Get(ctx context.Context, path, key string) ([]byte, error) {
|
||||
attribs, err := b.loadAttributes(ctx, path, nil)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
val, ok := attribs[key]
|
||||
if !ok {
|
||||
return []byte{}, &xattr.Error{Op: "mpk.get", Path: path, Name: key, Err: xattr.ENOATTR}
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// GetInt64 reads a string as int64 from the xattrs
|
||||
func (b MessagePackBackend) GetInt64(ctx context.Context, path, key string) (int64, error) {
|
||||
attribs, err := b.loadAttributes(ctx, path, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
val, ok := attribs[key]
|
||||
if !ok {
|
||||
return 0, &xattr.Error{Op: "mpk.get", Path: path, Name: key, Err: xattr.ENOATTR}
|
||||
}
|
||||
i, err := strconv.ParseInt(string(val), 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return i, nil
|
||||
}
|
||||
|
||||
// List retrieves a list of names of extended attributes associated with the
|
||||
// given path in the file system.
|
||||
func (b MessagePackBackend) List(ctx context.Context, path string) ([]string, error) {
|
||||
attribs, err := b.loadAttributes(ctx, path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := []string{}
|
||||
for k := range attribs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (b MessagePackBackend) Set(ctx context.Context, path, key string, val []byte) error {
|
||||
return b.SetMultiple(ctx, path, map[string][]byte{key: val}, true)
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (b MessagePackBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error {
|
||||
return b.saveAttributes(ctx, path, attribs, nil, acquireLock)
|
||||
}
|
||||
|
||||
// Remove an extended attribute key
|
||||
func (b MessagePackBackend) Remove(ctx context.Context, path, key string, acquireLock bool) error {
|
||||
return b.saveAttributes(ctx, path, nil, []string{key}, acquireLock)
|
||||
}
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader (if possible).
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (b MessagePackBackend) AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) {
|
||||
return b.loadAttributes(ctx, path, source)
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) saveAttributes(ctx context.Context, path string, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error {
|
||||
var (
|
||||
err error
|
||||
f readWriteCloseSeekTruncater
|
||||
)
|
||||
ctx, span := tracer.Start(ctx, "saveAttributes")
|
||||
defer func() {
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
} else {
|
||||
span.SetStatus(codes.Ok, "")
|
||||
}
|
||||
span.End()
|
||||
}()
|
||||
|
||||
lockPath := b.LockfilePath(path)
|
||||
metaPath := b.MetadataPath(path)
|
||||
if acquireLock {
|
||||
_, subspan := tracer.Start(ctx, "lockedfile.OpenFile")
|
||||
f, err = lockedfile.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
}
|
||||
// Read current state
|
||||
_, subspan := tracer.Start(ctx, "os.ReadFile")
|
||||
var msgBytes []byte
|
||||
msgBytes, err = os.ReadFile(metaPath)
|
||||
subspan.End()
|
||||
attribs := map[string][]byte{}
|
||||
switch {
|
||||
case err != nil:
|
||||
if !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
case len(msgBytes) == 0:
|
||||
// ugh. an empty file? bail out
|
||||
return errors.New("encountered empty metadata file")
|
||||
default:
|
||||
// only unmarshal if we read data
|
||||
err = msgpack.Unmarshal(msgBytes, &attribs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// prepare metadata
|
||||
for key, val := range setAttribs {
|
||||
attribs[key] = val
|
||||
}
|
||||
for _, key := range deleteAttribs {
|
||||
delete(attribs, key)
|
||||
}
|
||||
var d []byte
|
||||
d, err = msgpack.Marshal(attribs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// overwrite file atomically
|
||||
_, subspan = tracer.Start(ctx, "renameio.Writefile")
|
||||
err = renameio.WriteFile(metaPath, d, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
subspan.End()
|
||||
|
||||
_, subspan = tracer.Start(ctx, "metaCache.PushToCache")
|
||||
err = b.metaCache.PushToCache(b.cacheKey(path), attribs)
|
||||
subspan.End()
|
||||
return err
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) loadAttributes(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) {
|
||||
ctx, span := tracer.Start(ctx, "loadAttributes")
|
||||
defer span.End()
|
||||
attribs := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
}
|
||||
|
||||
metaPath := b.MetadataPath(path)
|
||||
var msgBytes []byte
|
||||
|
||||
if source == nil {
|
||||
// // No cached entry found. Read from storage and store in cache
|
||||
_, subspan := tracer.Start(ctx, "os.OpenFile")
|
||||
// source, err = lockedfile.Open(metaPath)
|
||||
source, err = os.Open(metaPath)
|
||||
subspan.End()
|
||||
// // No cached entry found. Read from storage and store in cache
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// some of the caller rely on ENOTEXISTS to be returned when the
|
||||
// actual file (not the metafile) does not exist in order to
|
||||
// determine whether a node exists or not -> stat the actual node
|
||||
_, subspan := tracer.Start(ctx, "os.Stat")
|
||||
_, err := os.Stat(path)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return attribs, nil // no attributes set yet
|
||||
}
|
||||
}
|
||||
_, subspan = tracer.Start(ctx, "io.ReadAll")
|
||||
msgBytes, err = io.ReadAll(source)
|
||||
source.(*os.File).Close()
|
||||
subspan.End()
|
||||
} else {
|
||||
_, subspan := tracer.Start(ctx, "io.ReadAll")
|
||||
msgBytes, err = io.ReadAll(source)
|
||||
subspan.End()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(msgBytes) == 0 {
|
||||
return nil, errors.New("encountered empty metadata file")
|
||||
}
|
||||
|
||||
err = msgpack.Unmarshal(msgBytes, &attribs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
_, subspan := tracer.Start(ctx, "metaCache.PushToCache")
|
||||
err = b.metaCache.PushToCache(b.cacheKey(path), attribs)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return attribs, nil
|
||||
}
|
||||
|
||||
// IsMetaFile returns whether the given path represents a meta file
|
||||
func (MessagePackBackend) IsMetaFile(path string) bool {
|
||||
return strings.HasSuffix(path, ".mpk") || strings.HasSuffix(path, ".mlock")
|
||||
}
|
||||
|
||||
// Purge purges the data of a given path
|
||||
func (b MessagePackBackend) Purge(_ context.Context, path string) error {
|
||||
if err := b.metaCache.RemoveMetadata(b.cacheKey(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Remove(b.MetadataPath(path))
|
||||
}
|
||||
|
||||
// Rename moves the data for a given path to a new path
|
||||
func (b MessagePackBackend) Rename(oldPath, newPath string) error {
|
||||
data := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data)
|
||||
if err == nil {
|
||||
err = b.metaCache.PushToCache(b.cacheKey(newPath), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = b.metaCache.RemoveMetadata(b.cacheKey(oldPath))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.Rename(b.MetadataPath(oldPath), b.MetadataPath(newPath))
|
||||
}
|
||||
|
||||
// MetadataPath returns the path of the file holding the metadata for the given path
|
||||
func (MessagePackBackend) MetadataPath(path string) string { return path + ".mpk" }
|
||||
|
||||
// LockfilePath returns the path of the lock file
|
||||
func (MessagePackBackend) LockfilePath(path string) string { return path + ".mlock" }
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (b MessagePackBackend) Lock(path string) (UnlockFunc, error) {
|
||||
metaLockPath := b.LockfilePath(path)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() error {
|
||||
err := mlock.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Remove(metaLockPath)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (b MessagePackBackend) cacheKey(path string) string {
|
||||
// rootPath is guaranteed to have no trailing slash
|
||||
// the cache key shouldn't begin with a slash as some stores drop it which can cause
|
||||
// confusion
|
||||
return strings.TrimPrefix(path, b.rootPath+"/")
|
||||
}
|
||||
Generated
Vendored
-129
@@ -1,129 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"go.opentelemetry.io/otel"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var tracer trace.Tracer
|
||||
|
||||
func init() {
|
||||
tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/metadata")
|
||||
}
|
||||
|
||||
var errUnconfiguredError = errors.New("no metadata backend configured. Bailing out")
|
||||
|
||||
type UnlockFunc func() error
|
||||
|
||||
// Backend defines the interface for file attribute backends
|
||||
type Backend interface {
|
||||
Name() string
|
||||
|
||||
All(ctx context.Context, path string) (map[string][]byte, error)
|
||||
Get(ctx context.Context, path, key string) ([]byte, error)
|
||||
|
||||
GetInt64(ctx context.Context, path, key string) (int64, error)
|
||||
List(ctx context.Context, path string) (attribs []string, err error)
|
||||
Set(ctx context.Context, path, key string, val []byte) error
|
||||
SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error
|
||||
Remove(ctx context.Context, path, key string, acquireLock bool) error
|
||||
|
||||
Lock(path string) (UnlockFunc, error)
|
||||
Purge(ctx context.Context, path string) error
|
||||
Rename(oldPath, newPath string) error
|
||||
IsMetaFile(path string) bool
|
||||
MetadataPath(path string) string
|
||||
LockfilePath(path string) string
|
||||
|
||||
AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error)
|
||||
}
|
||||
|
||||
// NullBackend is the default stub backend, used to enforce the configuration of a proper backend
|
||||
type NullBackend struct{}
|
||||
|
||||
// Name returns the name of the backend
|
||||
func (NullBackend) Name() string { return "null" }
|
||||
|
||||
// All reads all extended attributes for a node
|
||||
func (NullBackend) All(ctx context.Context, path string) (map[string][]byte, error) {
|
||||
return nil, errUnconfiguredError
|
||||
}
|
||||
|
||||
// Get an extended attribute value for the given key
|
||||
func (NullBackend) Get(ctx context.Context, path, key string) ([]byte, error) {
|
||||
return []byte{}, errUnconfiguredError
|
||||
}
|
||||
|
||||
// GetInt64 reads a string as int64 from the xattrs
|
||||
func (NullBackend) GetInt64(ctx context.Context, path, key string) (int64, error) {
|
||||
return 0, errUnconfiguredError
|
||||
}
|
||||
|
||||
// List retrieves a list of names of extended attributes associated with the
|
||||
// given path in the file system.
|
||||
func (NullBackend) List(ctx context.Context, path string) ([]string, error) {
|
||||
return nil, errUnconfiguredError
|
||||
}
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (NullBackend) Set(ctx context.Context, path string, key string, val []byte) error {
|
||||
return errUnconfiguredError
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (NullBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error {
|
||||
return errUnconfiguredError
|
||||
}
|
||||
|
||||
// Remove removes an extended attribute key
|
||||
func (NullBackend) Remove(ctx context.Context, path string, key string, acquireLock bool) error {
|
||||
return errUnconfiguredError
|
||||
}
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (NullBackend) Lock(path string) (UnlockFunc, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// IsMetaFile returns whether the given path represents a meta file
|
||||
func (NullBackend) IsMetaFile(path string) bool { return false }
|
||||
|
||||
// Purge purges the data of a given path from any cache that might hold it
|
||||
func (NullBackend) Purge(_ context.Context, purges string) error { return errUnconfiguredError }
|
||||
|
||||
// Rename moves the data for a given path to a new path
|
||||
func (NullBackend) Rename(oldPath, newPath string) error { return errUnconfiguredError }
|
||||
|
||||
// MetadataPath returns the path of the file holding the metadata for the given path
|
||||
func (NullBackend) MetadataPath(path string) string { return "" }
|
||||
|
||||
// LockfilePath returns the path of the lock file
|
||||
func (NullBackend) LockfilePath(path string) string { return "" }
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (NullBackend) AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) {
|
||||
return nil, errUnconfiguredError
|
||||
}
|
||||
Generated
Vendored
-29
@@ -1,29 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
//go:build !freebsd
|
||||
|
||||
package prefixes
|
||||
|
||||
// The default namespace for ocis. As non root users can only manipulate
|
||||
// the user. namespace, which is what is used to store ownCloud specific
|
||||
// metadata. To prevent name collisions with other apps, we are going to
|
||||
// introduce a sub namespace "user.ocis."
|
||||
const (
|
||||
OcisPrefix string = "user.ocis."
|
||||
)
|
||||
Generated
Vendored
-28
@@ -1,28 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
//go:build freebsd
|
||||
|
||||
package prefixes
|
||||
|
||||
// On FreeBSD the `user` namespace is implied through a separate syscall argument
|
||||
// and will fail with invalid argument when you try to start an xattr name with user. or system.
|
||||
// For that reason we drop the superfluous user. prefix for FreeBSD specifically.
|
||||
const (
|
||||
OcisPrefix string = "ocis."
|
||||
)
|
||||
Generated
Vendored
-102
@@ -1,102 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package prefixes
|
||||
|
||||
import "github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace"
|
||||
|
||||
// Declare a list of xattr keys
|
||||
// TODO the below comment is currently copied from the owncloud driver, revisit
|
||||
// Currently,extended file attributes have four separated
|
||||
// namespaces (user, trusted, security and system) followed by a dot.
|
||||
// A non root user can only manipulate the user. namespace, which is what
|
||||
// we will use to store ownCloud specific metadata. To prevent name
|
||||
// collisions with other apps We are going to introduce a sub namespace
|
||||
// "user.ocis." in the xattrs_prefix*.go files.
|
||||
const (
|
||||
TypeAttr string = OcisPrefix + "type"
|
||||
IDAttr string = OcisPrefix + "id"
|
||||
ParentidAttr string = OcisPrefix + "parentid"
|
||||
OwnerIDAttr string = OcisPrefix + "owner.id"
|
||||
OwnerIDPAttr string = OcisPrefix + "owner.idp"
|
||||
OwnerTypeAttr string = OcisPrefix + "owner.type"
|
||||
// the base name of the node
|
||||
// updated when the file is renamed or moved
|
||||
NameAttr string = OcisPrefix + "name"
|
||||
|
||||
BlobIDAttr string = OcisPrefix + "blobid"
|
||||
BlobsizeAttr string = OcisPrefix + "blobsize"
|
||||
|
||||
// statusPrefix is the prefix for the node status
|
||||
StatusPrefix string = OcisPrefix + "nodestatus"
|
||||
|
||||
// scanPrefix is the prefix for the virus scan status and date
|
||||
ScanStatusPrefix string = OcisPrefix + "scanstatus"
|
||||
ScanDatePrefix string = OcisPrefix + "scandate"
|
||||
|
||||
// grantPrefix is the prefix for sharing related extended attributes
|
||||
GrantPrefix string = OcisPrefix + "grant."
|
||||
GrantUserAcePrefix string = OcisPrefix + "grant." + ace.UserAcePrefix
|
||||
GrantGroupAcePrefix string = OcisPrefix + "grant." + ace.GroupAcePrefix
|
||||
MetadataPrefix string = OcisPrefix + "md."
|
||||
|
||||
// favorite flag, per user
|
||||
FavPrefix string = OcisPrefix + "fav."
|
||||
|
||||
// a temporary etag for a folder that is removed when the mtime propagation happens
|
||||
TmpEtagAttr string = OcisPrefix + "tmp.etag"
|
||||
ReferenceAttr string = OcisPrefix + "cs3.ref" // arbitrary metadata
|
||||
ChecksumPrefix string = OcisPrefix + "cs." // followed by the algorithm, eg. ocis.cs.sha1
|
||||
TrashOriginAttr string = OcisPrefix + "trash.origin" // trash origin
|
||||
|
||||
// we use a single attribute to enable or disable propagation of both: synctime and treesize
|
||||
// The propagation attribute is set to '1' at the top of the (sub)tree. Propagation will stop at
|
||||
// that node.
|
||||
PropagationAttr string = OcisPrefix + "propagation"
|
||||
|
||||
// we need mtime to keep mtime in sync with the metadata
|
||||
MTimeAttr string = OcisPrefix + "mtime"
|
||||
// the tree modification time of the tree below this node,
|
||||
// propagated when synctime_accounting is true and
|
||||
// user.ocis.propagation=1 is set
|
||||
// stored as a readable time.RFC3339Nano
|
||||
TreeMTimeAttr string = OcisPrefix + "tmtime"
|
||||
|
||||
// the deletion/disabled time of a space or node
|
||||
// used to mark space roots as disabled
|
||||
// stored as a readable time.RFC3339Nano
|
||||
DTimeAttr string = OcisPrefix + "dtime"
|
||||
|
||||
// the size of the tree below this node,
|
||||
// propagated when treesize_accounting is true and
|
||||
// user.ocis.propagation=1 is set
|
||||
// stored as uint64, little endian
|
||||
TreesizeAttr string = OcisPrefix + "treesize"
|
||||
|
||||
// the quota for the storage space / tree, regardless who accesses it
|
||||
QuotaAttr string = OcisPrefix + "quota"
|
||||
|
||||
// the name given to a storage space. It should not contain any semantics as its only purpose is to be read.
|
||||
SpaceIDAttr string = OcisPrefix + "space.id"
|
||||
SpaceNameAttr string = OcisPrefix + "space.name"
|
||||
SpaceTypeAttr string = OcisPrefix + "space.type"
|
||||
SpaceDescriptionAttr string = OcisPrefix + "space.description"
|
||||
SpaceReadmeAttr string = OcisPrefix + "space.readme"
|
||||
SpaceImageAttr string = OcisPrefix + "space.image"
|
||||
SpaceAliasAttr string = OcisPrefix + "space.alias"
|
||||
)
|
||||
Generated
Vendored
-302
@@ -1,302 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/pkg/xattr"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
)
|
||||
|
||||
// XattrsBackend stores the file attributes in extended attributes
|
||||
type XattrsBackend struct {
|
||||
rootPath string
|
||||
metaCache cache.FileMetadataCache
|
||||
}
|
||||
|
||||
// NewMessageBackend returns a new XattrsBackend instance
|
||||
func NewXattrsBackend(rootPath string, o cache.Config) XattrsBackend {
|
||||
return XattrsBackend{
|
||||
metaCache: cache.GetFileMetadataCache(o),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the name of the backend
|
||||
func (XattrsBackend) Name() string { return "xattrs" }
|
||||
|
||||
// Get an extended attribute value for the given key
|
||||
// No file locking is involved here as reading a single xattr is
|
||||
// considered to be atomic.
|
||||
func (b XattrsBackend) Get(ctx context.Context, path, key string) ([]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs)
|
||||
if err == nil && len(attribs[key]) > 0 {
|
||||
return attribs[key], err
|
||||
}
|
||||
|
||||
return xattr.Get(path, key)
|
||||
}
|
||||
|
||||
// GetInt64 reads a string as int64 from the xattrs
|
||||
func (b XattrsBackend) GetInt64(ctx context.Context, filePath, key string) (int64, error) {
|
||||
attr, err := b.Get(ctx, filePath, key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v, err := strconv.ParseInt(string(attr), 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// List retrieves a list of names of extended attributes associated with the
|
||||
// given path in the file system.
|
||||
func (b XattrsBackend) List(ctx context.Context, filePath string) (attribs []string, err error) {
|
||||
return b.list(ctx, filePath, true)
|
||||
}
|
||||
|
||||
func (b XattrsBackend) list(ctx context.Context, filePath string, acquireLock bool) (attribs []string, err error) {
|
||||
attrs, err := listXattr(ctx, filePath)
|
||||
if err == nil {
|
||||
return attrs, nil
|
||||
}
|
||||
|
||||
// E2BIG means the attribute name list exceeds the maximum size the kernel
|
||||
// returns (64KiB VFS limit, e.g. on XFS). It is not transient: fail fast
|
||||
// instead of paying for the locked retry below.
|
||||
if IsErrTooBig(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// listing xattrs failed, try again, either with lock or without
|
||||
if acquireLock {
|
||||
f, err := lockedfile.OpenFile(filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cleanupLockfile(ctx, f)
|
||||
|
||||
}
|
||||
return listXattr(ctx, filePath)
|
||||
}
|
||||
|
||||
// All reads all extended attributes for a node, protected by a
|
||||
// shared file lock
|
||||
func (b XattrsBackend) All(ctx context.Context, path string) (map[string][]byte, error) {
|
||||
return b.getAll(ctx, path, false, true)
|
||||
}
|
||||
|
||||
func (b XattrsBackend) getAll(ctx context.Context, path string, skipCache, acquireLock bool) (map[string][]byte, error) {
|
||||
attribs := map[string][]byte{}
|
||||
|
||||
if !skipCache {
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs)
|
||||
if err == nil {
|
||||
return attribs, err
|
||||
}
|
||||
}
|
||||
|
||||
attrNames, err := b.list(ctx, path, acquireLock)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(attrNames) == 0 {
|
||||
return attribs, nil
|
||||
}
|
||||
|
||||
var (
|
||||
xerrs = 0
|
||||
xerr error
|
||||
)
|
||||
// error handling: Count if there are errors while reading all attribs.
|
||||
// if there were any, return an error.
|
||||
attribs = make(map[string][]byte, len(attrNames))
|
||||
for _, name := range attrNames {
|
||||
var val []byte
|
||||
if val, xerr = xattr.Get(path, name); xerr != nil && !IsAttrUnset(xerr) {
|
||||
xerrs++
|
||||
} else {
|
||||
attribs[name] = val
|
||||
}
|
||||
}
|
||||
|
||||
if xerrs > 0 {
|
||||
return nil, errors.Wrap(xerr, "Failed to read all xattrs")
|
||||
}
|
||||
|
||||
err = b.metaCache.PushToCache(b.cacheKey(path), attribs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return attribs, nil
|
||||
}
|
||||
|
||||
// Set sets one attribute for the given path
|
||||
func (b XattrsBackend) Set(ctx context.Context, path string, key string, val []byte) (err error) {
|
||||
return b.SetMultiple(ctx, path, map[string][]byte{key: val}, true)
|
||||
}
|
||||
|
||||
// SetMultiple sets a set of attribute for the given path
|
||||
func (b XattrsBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) (err error) {
|
||||
if acquireLock {
|
||||
err := os.MkdirAll(filepath.Dir(path), 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lockedFile, err := lockedfile.OpenFile(b.LockfilePath(path), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanupLockfile(ctx, lockedFile)
|
||||
}
|
||||
|
||||
// error handling: Count if there are errors while setting the attribs.
|
||||
// if there were any, return an error.
|
||||
var (
|
||||
xerrs = 0
|
||||
xerr error
|
||||
)
|
||||
for key, val := range attribs {
|
||||
if xerr = xattr.Set(path, key, val); xerr != nil {
|
||||
// log
|
||||
xerrs++
|
||||
}
|
||||
}
|
||||
if xerrs > 0 {
|
||||
return errors.Wrap(xerr, "Failed to set all xattrs")
|
||||
}
|
||||
|
||||
attribs, err = b.getAll(ctx, path, true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.metaCache.PushToCache(b.cacheKey(path), attribs)
|
||||
}
|
||||
|
||||
// Remove an extended attribute key
|
||||
func (b XattrsBackend) Remove(ctx context.Context, path string, key string, acquireLock bool) error {
|
||||
if acquireLock {
|
||||
lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer cleanupLockfile(ctx, lockedFile)
|
||||
}
|
||||
|
||||
err := xattr.Remove(path, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attribs, err := b.getAll(ctx, path, true, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.metaCache.PushToCache(b.cacheKey(path), attribs)
|
||||
}
|
||||
|
||||
// IsMetaFile returns whether the given path represents a meta file
|
||||
func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".meta.lock") }
|
||||
|
||||
// Purge purges the data of a given path
|
||||
func (b XattrsBackend) Purge(ctx context.Context, path string) error {
|
||||
_, err := os.Stat(path)
|
||||
if err == nil {
|
||||
attribs, err := b.getAll(ctx, path, true, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for attr := range attribs {
|
||||
if strings.HasPrefix(attr, prefixes.OcisPrefix) {
|
||||
err := xattr.Remove(path, attr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return b.metaCache.RemoveMetadata(b.cacheKey(path))
|
||||
}
|
||||
|
||||
// Rename moves the data for a given path to a new path
|
||||
func (b XattrsBackend) Rename(oldPath, newPath string) error {
|
||||
data := map[string][]byte{}
|
||||
err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data)
|
||||
if err == nil {
|
||||
err = b.metaCache.PushToCache(b.cacheKey(newPath), data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return b.metaCache.RemoveMetadata(b.cacheKey(oldPath))
|
||||
}
|
||||
|
||||
// MetadataPath returns the path of the file holding the metadata for the given path
|
||||
func (XattrsBackend) MetadataPath(path string) string { return path }
|
||||
|
||||
// LockfilePath returns the path of the lock file
|
||||
func (XattrsBackend) LockfilePath(path string) string { return path + ".mlock" }
|
||||
|
||||
// Lock locks the metadata for the given path
|
||||
func (b XattrsBackend) Lock(path string) (UnlockFunc, error) {
|
||||
metaLockPath := b.LockfilePath(path)
|
||||
mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() error {
|
||||
err := mlock.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Remove(metaLockPath)
|
||||
}, nil
|
||||
}
|
||||
|
||||
func cleanupLockfile(ctx context.Context, f *lockedfile.File) {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(f.Name())
|
||||
}
|
||||
|
||||
// AllWithLockedSource reads all extended attributes from the given reader.
|
||||
// The path argument is used for storing the data in the cache
|
||||
func (b XattrsBackend) AllWithLockedSource(ctx context.Context, path string, _ io.Reader) (map[string][]byte, error) {
|
||||
return b.All(ctx, path)
|
||||
}
|
||||
|
||||
func (b XattrsBackend) cacheKey(path string) string {
|
||||
// rootPath is guaranteed to have no trailing slash
|
||||
// the cache key shouldn't begin with a slash as some stores drop it which can cause
|
||||
// confusion
|
||||
return strings.TrimPrefix(path, b.rootPath+"/")
|
||||
}
|
||||
Generated
Vendored
-138
@@ -1,138 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerMigration("0001", Migration0001{})
|
||||
}
|
||||
|
||||
type Migration0001 struct{}
|
||||
|
||||
// Migrate creates the spaces directory structure
|
||||
func (m Migration0001) Migrate(migrator *Migrator) (Result, error) {
|
||||
migrator.log.Info().Msg("Migrating spaces directory structure...")
|
||||
|
||||
// create spaces folder and iterate over existing nodes to populate it
|
||||
nodesPath := filepath.Join(migrator.lu.InternalRoot(), "nodes")
|
||||
fi, err := os.Stat(nodesPath)
|
||||
if err == nil && fi.IsDir() {
|
||||
f, err := os.Open(nodesPath)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
nodes, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
|
||||
for _, n := range nodes {
|
||||
nodePath := filepath.Join(nodesPath, n.Name())
|
||||
|
||||
attr, err := migrator.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr)
|
||||
if err == nil && string(attr) == node.RootID {
|
||||
if err := m.moveNode(migrator, n.Name(), n.Name()); err != nil {
|
||||
migrator.log.Error().Err(err).
|
||||
Str("space", n.Name()).
|
||||
Msg("could not move space")
|
||||
continue
|
||||
}
|
||||
m.linkSpaceNode(migrator, "personal", n.Name())
|
||||
}
|
||||
}
|
||||
// TODO delete nodesPath if empty
|
||||
}
|
||||
return stateSucceeded, nil
|
||||
}
|
||||
|
||||
// Rollback is not implemented
|
||||
func (Migration0001) Rollback(_ *Migrator) (Result, error) {
|
||||
return stateFailed, errors.New("rollback not implemented")
|
||||
}
|
||||
|
||||
func (m Migration0001) moveNode(migrator *Migrator, spaceID, nodeID string) error {
|
||||
dirPath := filepath.Join(migrator.lu.InternalRoot(), "nodes", nodeID)
|
||||
f, err := os.Open(dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, child := range children {
|
||||
old := filepath.Join(migrator.lu.InternalRoot(), "nodes", child.Name())
|
||||
new := filepath.Join(migrator.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2))
|
||||
if err := os.Rename(old, new); err != nil {
|
||||
migrator.log.Error().Err(err).
|
||||
Str("space", spaceID).
|
||||
Str("nodes", child.Name()).
|
||||
Str("oldpath", old).
|
||||
Str("newpath", new).
|
||||
Msg("could not rename node")
|
||||
}
|
||||
if child.IsDir() {
|
||||
if err := m.moveNode(migrator, spaceID, child.Name()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// linkSpace creates a new symbolic link for a space with the given type st, and node id
|
||||
func (m Migration0001) linkSpaceNode(migrator *Migrator, spaceType, spaceID string) {
|
||||
spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, spaceID)
|
||||
expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2)
|
||||
linkTarget, err := os.Readlink(spaceTypesPath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
err = os.Symlink(expectedTarget, spaceTypesPath)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).
|
||||
Str("space_type", spaceType).
|
||||
Str("space", spaceID).
|
||||
Msg("could not create symlink")
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).
|
||||
Str("space_type", spaceType).
|
||||
Str("space", spaceID).
|
||||
Msg("could not read symlink")
|
||||
}
|
||||
if linkTarget != expectedTarget {
|
||||
migrator.log.Warn().
|
||||
Str("space_type", spaceType).
|
||||
Str("space", spaceID).
|
||||
Str("expected", expectedTarget).
|
||||
Str("actual", linkTarget).
|
||||
Msg("expected a different link target")
|
||||
}
|
||||
}
|
||||
}
|
||||
Generated
Vendored
-150
@@ -1,150 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/logger"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerMigration("0002", Migration0002{})
|
||||
}
|
||||
|
||||
type Migration0002 struct{}
|
||||
|
||||
// Migrate migrates spacetypes to indexes
|
||||
func (m Migration0002) Migrate(migrator *Migrator) (Result, error) {
|
||||
migrator.log.Info().Msg("Migrating space types indexes...")
|
||||
|
||||
spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes")
|
||||
fi, err := os.Stat(spaceTypesPath)
|
||||
if err == nil && fi.IsDir() {
|
||||
|
||||
f, err := os.Open(spaceTypesPath)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
spaceTypes, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
|
||||
for _, st := range spaceTypes {
|
||||
err := m.moveSpaceType(migrator, st.Name())
|
||||
if err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", st.Name()).
|
||||
Msg("could not move space")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// delete spacetypespath
|
||||
d, err := os.Open(spaceTypesPath)
|
||||
if err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("spacetypesdir", spaceTypesPath).
|
||||
Msg("could not open spacetypesdir")
|
||||
return stateFailed, nil
|
||||
}
|
||||
defer d.Close()
|
||||
_, err = d.Readdirnames(1) // Or f.Readdir(1)
|
||||
if err == io.EOF {
|
||||
// directory is empty we can delete
|
||||
err := os.Remove(spaceTypesPath)
|
||||
if err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("spacetypesdir", d.Name()).
|
||||
Msg("could not delete")
|
||||
}
|
||||
} else {
|
||||
logger.New().Error().Err(err).
|
||||
Str("spacetypesdir", d.Name()).
|
||||
Msg("could not delete, not empty")
|
||||
}
|
||||
}
|
||||
return stateSucceeded, nil
|
||||
}
|
||||
|
||||
// Rollback is not implemented
|
||||
func (Migration0002) Rollback(_ *Migrator) (Result, error) {
|
||||
return stateFailed, errors.New("rollback not implemented")
|
||||
}
|
||||
|
||||
func (m Migration0002) moveSpaceType(migrator *Migrator, spaceType string) error {
|
||||
dirPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType)
|
||||
f, err := os.Open(dirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
children, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, child := range children {
|
||||
old := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, child.Name())
|
||||
target, err := os.Readlink(old)
|
||||
if err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", spaceType).
|
||||
Str("nodes", child.Name()).
|
||||
Str("oldLink", old).
|
||||
Msg("could not read old symlink")
|
||||
continue
|
||||
}
|
||||
newDir := filepath.Join(migrator.lu.InternalRoot(), "indexes", "by-type", spaceType)
|
||||
if err := os.MkdirAll(newDir, 0700); err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", spaceType).
|
||||
Str("nodes", child.Name()).
|
||||
Str("targetDir", newDir).
|
||||
Msg("could not read old symlink")
|
||||
}
|
||||
newLink := filepath.Join(newDir, child.Name())
|
||||
if err := os.Symlink(filepath.Join("..", target), newLink); err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", spaceType).
|
||||
Str("nodes", child.Name()).
|
||||
Str("oldpath", old).
|
||||
Str("newpath", newLink).
|
||||
Msg("could not rename node")
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(old); err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", spaceType).
|
||||
Str("nodes", child.Name()).
|
||||
Str("oldLink", old).
|
||||
Msg("could not remove old symlink")
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := os.Remove(dirPath); err != nil {
|
||||
logger.New().Error().Err(err).
|
||||
Str("space", spaceType).
|
||||
Str("dir", dirPath).
|
||||
Msg("could not remove spaces folder, folder probably not empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Generated
Vendored
-120
@@ -1,120 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/cache"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerMigration("0003", Migration0003{})
|
||||
}
|
||||
|
||||
type Migration0003 struct{}
|
||||
|
||||
// Migrate migrates the file metadata to the current backend.
|
||||
// Only the xattrs -> messagepack path is supported.
|
||||
func (m Migration0003) Migrate(migrator *Migrator) (Result, error) {
|
||||
bod := lookup.DetectBackendOnDisk(migrator.lu.InternalRoot())
|
||||
if bod == "" {
|
||||
return stateFailed, errors.New("could not detect metadata backend on disk")
|
||||
}
|
||||
|
||||
if bod != "xattrs" || migrator.lu.MetadataBackend().Name() != "messagepack" {
|
||||
return stateSucceededRunAgain, nil
|
||||
}
|
||||
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...")
|
||||
xattrs := metadata.NewXattrsBackend(migrator.lu.InternalRoot(), cache.Config{})
|
||||
mpk := metadata.NewMessagePackBackend(migrator.lu.InternalRoot(), cache.Config{})
|
||||
|
||||
spaces, _ := filepath.Glob(filepath.Join(migrator.lu.InternalRoot(), "spaces", "*", "*"))
|
||||
for _, space := range spaces {
|
||||
err := filepath.WalkDir(filepath.Join(space, "nodes"), func(path string, _ fs.DirEntry, err error) error {
|
||||
// Do not continue on error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".mpk") || strings.HasSuffix(path, ".flock") {
|
||||
// None of our business
|
||||
return nil
|
||||
}
|
||||
|
||||
fi, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !fi.IsDir() && !fi.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
mpkPath := mpk.MetadataPath(path)
|
||||
_, err = os.Stat(mpkPath)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
attribs, err := xattrs.All(context.Background(), path)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Str("path", path).Msg("error converting file")
|
||||
return err
|
||||
}
|
||||
if len(attribs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = mpk.SetMultiple(context.Background(), path, attribs, false)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Str("path", path).Msg("error setting attributes")
|
||||
return err
|
||||
}
|
||||
|
||||
for k := range attribs {
|
||||
err = xattrs.Remove(context.Background(), path, k, false)
|
||||
if err != nil {
|
||||
migrator.log.Debug().Err(err).Str("path", path).Msg("error removing xattr")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend")
|
||||
}
|
||||
}
|
||||
|
||||
migrator.log.Info().Msg("done.")
|
||||
return stateSucceeded, nil
|
||||
}
|
||||
|
||||
// Rollback is not implemented
|
||||
func (Migration0003) Rollback(_ *Migrator) (Result, error) {
|
||||
return stateFailed, errors.New("rollback not implemented")
|
||||
}
|
||||
Generated
Vendored
-203
@@ -1,203 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerMigration("0004", Migration0004{})
|
||||
}
|
||||
|
||||
type Migration0004 struct{}
|
||||
|
||||
// Migrate migrates the directory tree based space indexes to messagepack
|
||||
func (Migration0004) Migrate(migrator *Migrator) (Result, error) {
|
||||
root := migrator.lu.InternalRoot()
|
||||
|
||||
// migrate user indexes
|
||||
users, err := os.ReadDir(filepath.Join(root, "indexes", "by-user-id"))
|
||||
if err != nil {
|
||||
migrator.log.Warn().Err(err).Msg("error listing user indexes")
|
||||
}
|
||||
for _, user := range users {
|
||||
if !user.IsDir() {
|
||||
continue
|
||||
}
|
||||
id := user.Name()
|
||||
indexPath := filepath.Join(root, "indexes", "by-user-id", id+".mpk")
|
||||
dirIndexPath := filepath.Join(root, "indexes", "by-user-id", id)
|
||||
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
|
||||
err := migrateSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
|
||||
// migrate group indexes
|
||||
groups, err := os.ReadDir(filepath.Join(root, "indexes", "by-group-id"))
|
||||
if err != nil {
|
||||
migrator.log.Warn().Err(err).Msg("error listing group indexes")
|
||||
}
|
||||
for _, group := range groups {
|
||||
if !group.IsDir() {
|
||||
continue
|
||||
}
|
||||
id := group.Name()
|
||||
indexPath := filepath.Join(root, "indexes", "by-group-id", id+".mpk")
|
||||
dirIndexPath := filepath.Join(root, "indexes", "by-group-id", id)
|
||||
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
|
||||
err := migrateSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
|
||||
// migrate project indexes
|
||||
for _, spaceType := range []string{"personal", "project", "share"} {
|
||||
indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk")
|
||||
dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType)
|
||||
|
||||
_, err := os.Stat(dirIndexPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...")
|
||||
err = migrateSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
|
||||
migrator.log.Info().Msg("done.")
|
||||
return stateSucceeded, nil
|
||||
}
|
||||
|
||||
func migrateSpaceIndex(indexPath, dirIndexPath string) error {
|
||||
links := map[string][]byte{}
|
||||
m, err := filepath.Glob(dirIndexPath + "/*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, match := range m {
|
||||
link, err := os.Readlink(match)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
links[filepath.Base(match)] = []byte(link)
|
||||
}
|
||||
|
||||
// rewrite index as file
|
||||
d, err := msgpack.Marshal(links)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.WriteFile(indexPath, d, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(dirIndexPath)
|
||||
}
|
||||
|
||||
// Rollback migrates the directory messagepack indexes to symlinks
|
||||
func (Migration0004) Rollback(m *Migrator) (Result, error) {
|
||||
root := m.lu.InternalRoot()
|
||||
|
||||
// migrate user indexes
|
||||
users, err := filepath.Glob(filepath.Join(root, "indexes", "by-user-id", "*.mpk"))
|
||||
if err != nil {
|
||||
m.log.Warn().Err(err).Msg("error listing user indexes")
|
||||
}
|
||||
for _, indexPath := range users {
|
||||
dirIndexPath := strings.TrimSuffix(indexPath, ".mpk")
|
||||
|
||||
m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
|
||||
err := downSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
|
||||
// migrate group indexes
|
||||
groups, err := filepath.Glob(filepath.Join(root, "indexes", "by-group-id", "*.mpk"))
|
||||
if err != nil {
|
||||
m.log.Warn().Err(err).Msg("error listing group indexes")
|
||||
}
|
||||
for _, indexPath := range groups {
|
||||
dirIndexPath := strings.TrimSuffix(indexPath, ".mpk")
|
||||
|
||||
m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
|
||||
err := downSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
|
||||
// migrate project indexes
|
||||
for _, spaceType := range []string{"personal", "project", "share"} {
|
||||
indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk")
|
||||
dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType)
|
||||
|
||||
_, err := os.Stat(indexPath)
|
||||
if err != nil || os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
|
||||
m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...")
|
||||
err = downSpaceIndex(indexPath, dirIndexPath)
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index")
|
||||
}
|
||||
}
|
||||
return stateDown, nil
|
||||
}
|
||||
|
||||
func downSpaceIndex(indexPath, dirIndexPath string) error {
|
||||
d, err := os.ReadFile(indexPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
links := map[string][]byte{}
|
||||
err = msgpack.Unmarshal(d, &links)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.MkdirAll(dirIndexPath, 0700)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for link, target := range links {
|
||||
err = os.Symlink(string(target), filepath.Join(dirIndexPath, link))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return os.Remove(indexPath)
|
||||
}
|
||||
Generated
Vendored
-114
@@ -1,114 +0,0 @@
|
||||
// Copyright 2018-2023 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/shamaton/msgpack/v2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
registerMigration("0005", Migration0005{})
|
||||
}
|
||||
|
||||
type Migration0005 struct{}
|
||||
|
||||
// Migrate fixes the messagepack space index data structure
|
||||
func (Migration0005) Migrate(migrator *Migrator) (Result, error) {
|
||||
root := migrator.lu.InternalRoot()
|
||||
|
||||
indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk"))
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
for _, i := range indexes {
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i)
|
||||
|
||||
// Read old-format index
|
||||
oldData, err := os.ReadFile(i)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
oldIndex := map[string][]byte{}
|
||||
err = msgpack.Unmarshal(oldData, &oldIndex)
|
||||
if err != nil {
|
||||
// likely already migrated -> skip
|
||||
migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i)
|
||||
continue
|
||||
}
|
||||
|
||||
// Write new-format index
|
||||
newIndex := map[string]string{}
|
||||
for k, v := range oldIndex {
|
||||
newIndex[k] = string(v)
|
||||
}
|
||||
newData, err := msgpack.Marshal(newIndex)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
err = os.WriteFile(i, newData, 0600)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
}
|
||||
migrator.log.Info().Msg("done.")
|
||||
return stateSucceeded, nil
|
||||
}
|
||||
|
||||
// Rollback rolls back the migration
|
||||
func (Migration0005) Rollback(migrator *Migrator) (Result, error) {
|
||||
root := migrator.lu.InternalRoot()
|
||||
|
||||
indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk"))
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
for _, i := range indexes {
|
||||
migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i)
|
||||
|
||||
oldData, err := os.ReadFile(i)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
oldIndex := map[string]string{}
|
||||
err = msgpack.Unmarshal(oldData, &oldIndex)
|
||||
if err != nil {
|
||||
migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i)
|
||||
continue
|
||||
}
|
||||
|
||||
// Write new-format index
|
||||
newIndex := map[string][]byte{}
|
||||
for k, v := range oldIndex {
|
||||
newIndex[k] = []byte(v)
|
||||
}
|
||||
newData, err := msgpack.Marshal(newIndex)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
err = os.WriteFile(i, newData, 0600)
|
||||
if err != nil {
|
||||
return stateFailed, err
|
||||
}
|
||||
}
|
||||
migrator.log.Info().Msg("done.")
|
||||
return stateDown, nil
|
||||
}
|
||||
Generated
Vendored
-217
@@ -1,217 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package migrator
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node"
|
||||
"github.com/rogpeppe/go-internal/lockedfile"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
const (
|
||||
statePending = "pending"
|
||||
stateFailed = "failed"
|
||||
stateSucceeded = "succeeded"
|
||||
stateDown = "down"
|
||||
stateSucceededRunAgain = "runagain"
|
||||
)
|
||||
|
||||
type migration interface {
|
||||
Migrate(*Migrator) (Result, error)
|
||||
Rollback(*Migrator) (Result, error)
|
||||
}
|
||||
|
||||
var migrations = map[string]migration{}
|
||||
|
||||
type migrationStates map[string]MigrationState
|
||||
|
||||
func registerMigration(name string, migration migration) {
|
||||
migrations[name] = migration
|
||||
}
|
||||
|
||||
func allMigrations() []string {
|
||||
ms := make([]string, 0, len(migrations))
|
||||
|
||||
for k := range migrations {
|
||||
ms = append(ms, k)
|
||||
}
|
||||
|
||||
sort.Strings(ms)
|
||||
return ms
|
||||
}
|
||||
|
||||
// MigrationState holds the state of a migration
|
||||
type MigrationState struct {
|
||||
State string
|
||||
Message string
|
||||
}
|
||||
|
||||
// Result represents the result of a migration run
|
||||
type Result string
|
||||
|
||||
// Migrator runs migrations on an existing decomposedfs
|
||||
type Migrator struct {
|
||||
lu node.PathLookup
|
||||
states migrationStates
|
||||
log *zerolog.Logger
|
||||
}
|
||||
|
||||
// New returns a new Migrator instance
|
||||
func New(lu node.PathLookup, log *zerolog.Logger) Migrator {
|
||||
return Migrator{
|
||||
lu: lu,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// Migrations returns the list of migrations and their states
|
||||
func (m *Migrator) Migrations() (map[string]MigrationState, error) {
|
||||
err := m.readStates()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
states := map[string]MigrationState{}
|
||||
for _, migration := range allMigrations() {
|
||||
if s, ok := m.states[migration]; ok {
|
||||
states[migration] = s
|
||||
} else {
|
||||
states[migration] = MigrationState{
|
||||
State: statePending,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return states, nil
|
||||
}
|
||||
|
||||
// RunMigration runs or rolls back a migration
|
||||
func (m *Migrator) RunMigration(id string, rollback bool) error {
|
||||
if _, ok := migrations[id]; !ok {
|
||||
return fmt.Errorf("invalid migration '%s'", id)
|
||||
}
|
||||
|
||||
lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
|
||||
err = m.readStates()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var res Result
|
||||
if !rollback {
|
||||
m.log.Info().Msg("Running migration " + id + "...")
|
||||
res, err = migrations[id].Migrate(m)
|
||||
} else {
|
||||
m.log.Info().Msg("Rolling back migration " + id + "...")
|
||||
res, err = migrations[id].Rollback(m)
|
||||
}
|
||||
|
||||
// write back state
|
||||
s := m.states[id]
|
||||
s.State = string(res)
|
||||
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Msg("migration " + id + " failed")
|
||||
s.Message = err.Error()
|
||||
}
|
||||
|
||||
m.states[id] = s
|
||||
err = m.writeStates()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.log.Info().Msg("done")
|
||||
return nil
|
||||
}
|
||||
|
||||
// RunMigrations runs all migrations in sequence. Note this sequence must not be changed or it might
|
||||
// damage existing decomposed fs.
|
||||
func (m *Migrator) RunMigrations() error {
|
||||
lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer lock.Close()
|
||||
|
||||
err = m.readStates()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, migration := range allMigrations() {
|
||||
s := m.states[migration]
|
||||
if s.State == stateSucceeded || s.State == stateDown {
|
||||
continue
|
||||
}
|
||||
|
||||
res, err := migrations[migration].Migrate(m)
|
||||
s.State = string(res)
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Msg("migration " + migration + " failed")
|
||||
s.Message = err.Error()
|
||||
}
|
||||
|
||||
m.states[migration] = s
|
||||
err = m.writeStates()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Migrator) readStates() error {
|
||||
m.states = migrationStates{}
|
||||
|
||||
d, err := os.ReadFile(filepath.Join(m.lu.InternalRoot(), ".migrations"))
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(d) > 0 {
|
||||
err = json.Unmarshal(d, &m.states)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Migrator) writeStates() error {
|
||||
d, err := json.Marshal(m.states)
|
||||
if err != nil {
|
||||
m.log.Error().Err(err).Msg("could not marshal migration states")
|
||||
return nil
|
||||
}
|
||||
return os.WriteFile(filepath.Join(m.lu.InternalRoot(), ".migrations"), d, 0600)
|
||||
}
|
||||
Generated
Vendored
-45
@@ -1,45 +0,0 @@
|
||||
package mtimesyncedcache
|
||||
|
||||
import "sync"
|
||||
|
||||
type Map[K comparable, V any] struct {
|
||||
m sync.Map
|
||||
}
|
||||
|
||||
func (m *Map[K, V]) Delete(key K) { m.m.Delete(key) }
|
||||
|
||||
func (m *Map[K, V]) Load(key K) (value V, ok bool) {
|
||||
v, ok := m.m.Load(key)
|
||||
if !ok {
|
||||
return value, ok
|
||||
}
|
||||
return v.(V), ok
|
||||
}
|
||||
|
||||
func (m *Map[K, V]) LoadAndDelete(key K) (value V, loaded bool) {
|
||||
v, loaded := m.m.LoadAndDelete(key)
|
||||
if !loaded {
|
||||
return value, loaded
|
||||
}
|
||||
return v.(V), loaded
|
||||
}
|
||||
|
||||
func (m *Map[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) {
|
||||
a, loaded := m.m.LoadOrStore(key, value)
|
||||
return a.(V), loaded
|
||||
}
|
||||
|
||||
func (m *Map[K, V]) Range(f func(key K, value V) bool) {
|
||||
m.m.Range(func(key, value any) bool { return f(key.(K), value.(V)) })
|
||||
}
|
||||
|
||||
func (m *Map[K, V]) Store(key K, value V) { m.m.Store(key, value) }
|
||||
|
||||
func (m *Map[K, V]) Count() int {
|
||||
l := 0
|
||||
m.Range(func(_ K, _ V) bool {
|
||||
l++
|
||||
return true
|
||||
})
|
||||
return l
|
||||
}
|
||||
Generated
Vendored
-59
@@ -1,59 +0,0 @@
|
||||
package mtimesyncedcache
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Cache[K comparable, T any] struct {
|
||||
entries Map[K, *entry[T]]
|
||||
}
|
||||
|
||||
type entry[T any] struct {
|
||||
mtime time.Time
|
||||
value T
|
||||
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func New[K comparable, T any]() Cache[K, T] {
|
||||
return Cache[K, T]{
|
||||
entries: Map[K, *entry[T]]{},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[K, T]) Store(key K, mtime time.Time, value T) error {
|
||||
c.entries.Store(key, &entry[T]{
|
||||
mtime: mtime,
|
||||
value: value,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Cache[K, T]) Load(key K) (T, bool) {
|
||||
entry, ok := c.entries.Load(key)
|
||||
if !ok {
|
||||
var t T
|
||||
return t, false
|
||||
}
|
||||
return entry.value, true
|
||||
}
|
||||
|
||||
func (c *Cache[K, T]) LoadOrStore(key K, mtime time.Time, f func() (T, error)) (T, error) {
|
||||
e, _ := c.entries.LoadOrStore(key, &entry[T]{})
|
||||
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
if mtime.After(e.mtime) {
|
||||
e.mtime = mtime
|
||||
v, err := f()
|
||||
if err != nil {
|
||||
var t T
|
||||
return t, err
|
||||
}
|
||||
e.value = v
|
||||
c.entries.Store(key, e)
|
||||
}
|
||||
|
||||
return e.value, nil
|
||||
}
|
||||
Generated
Vendored
-360
@@ -1,360 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// SetLock sets a lock on the node
|
||||
func (n *Node) SetLock(ctx context.Context, lock *provider.Lock) error {
|
||||
ctx, span := tracer.Start(ctx, "SetLock")
|
||||
defer span.End()
|
||||
lockFilePath := n.LockFilePath()
|
||||
|
||||
// ensure parent path exists
|
||||
if err := os.MkdirAll(filepath.Dir(lockFilePath), 0700); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock")
|
||||
}
|
||||
|
||||
// get file lock, so that nobody can create the lock in the meantime
|
||||
fileLock, err := filelocks.AcquireWriteLock(n.InternalPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
rerr := filelocks.ReleaseLock(fileLock)
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
// check if already locked
|
||||
l, err := n.ReadLock(ctx, true) // we already have a write file lock, so ReadLock() would fail to acquire a read file lock -> skip it
|
||||
switch err.(type) {
|
||||
case errtypes.NotFound:
|
||||
// file not locked, continue
|
||||
case nil:
|
||||
if l != nil {
|
||||
return errtypes.PreconditionFailed("already locked")
|
||||
}
|
||||
default:
|
||||
return errors.Wrap(err, "Decomposedfs: could check if file already is locked")
|
||||
}
|
||||
|
||||
// O_EXCL to make open fail when the file already exists
|
||||
f, err := os.OpenFile(lockFilePath, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0600)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not create lock file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
if err := json.NewEncoder(f).Encode(lock); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not write lock file")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadLock reads the lock id for a node
|
||||
func (n Node) ReadLock(ctx context.Context, skipFileLock bool) (*provider.Lock, error) {
|
||||
ctx, span := tracer.Start(ctx, "ReadLock")
|
||||
defer span.End()
|
||||
|
||||
// ensure parent path exists
|
||||
_, subspan := tracer.Start(ctx, "os.MkdirAll")
|
||||
err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700)
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: error creating parent folder for lock")
|
||||
}
|
||||
|
||||
// the caller of ReadLock already may hold a file lock
|
||||
if !skipFileLock {
|
||||
_, subspan := tracer.Start(ctx, "filelocks.AcquireReadLock")
|
||||
fileLock, err := filelocks.AcquireReadLock(n.InternalPath())
|
||||
subspan.End()
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, subspan := tracer.Start(ctx, "filelocks.ReleaseLock")
|
||||
rerr := filelocks.ReleaseLock(fileLock)
|
||||
subspan.End()
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
_, subspan = tracer.Start(ctx, "os.Open")
|
||||
f, err := os.Open(n.LockFilePath())
|
||||
subspan.End()
|
||||
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, errtypes.NotFound("no lock found")
|
||||
}
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not open lock file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
lock := &provider.Lock{}
|
||||
if err := json.NewDecoder(f).Decode(lock); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not decode lock file, ignoring")
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not read lock file")
|
||||
}
|
||||
|
||||
// lock already expired
|
||||
if lock.Expiration != nil && time.Now().After(time.Unix(int64(lock.Expiration.Seconds), int64(lock.Expiration.Nanos))) {
|
||||
|
||||
_, subspan = tracer.Start(ctx, "os.Remove")
|
||||
err = os.Remove(f.Name())
|
||||
subspan.End()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Decomposedfs: could not remove expired lock file")
|
||||
}
|
||||
// we successfully deleted the expired lock
|
||||
return nil, errtypes.NotFound("no lock found")
|
||||
}
|
||||
|
||||
return lock, nil
|
||||
}
|
||||
|
||||
// RefreshLock refreshes the node's lock
|
||||
func (n *Node) RefreshLock(ctx context.Context, lock *provider.Lock, existingLockID string) error {
|
||||
ctx, span := tracer.Start(ctx, "RefreshLock")
|
||||
defer span.End()
|
||||
|
||||
// ensure parent path exists
|
||||
if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock")
|
||||
}
|
||||
fileLock, err := filelocks.AcquireWriteLock(n.InternalPath())
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
rerr := filelocks.ReleaseLock(fileLock)
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
f, err := os.OpenFile(n.LockFilePath(), os.O_RDWR, os.ModeExclusive)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return errtypes.PreconditionFailed("lock does not exist")
|
||||
case err != nil:
|
||||
return errors.Wrap(err, "Decomposedfs: could not open lock file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
readLock := &provider.Lock{}
|
||||
if err := json.NewDecoder(f).Decode(readLock); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not read lock")
|
||||
}
|
||||
|
||||
// check refresh lockID match
|
||||
if existingLockID == "" && readLock.LockId != lock.LockId {
|
||||
return errtypes.Aborted("mismatching lock ID")
|
||||
}
|
||||
|
||||
// check if UnlockAndRelock sends the correct lockID
|
||||
if existingLockID != "" && readLock.LockId != existingLockID {
|
||||
return errtypes.Aborted("mismatching existing lock ID")
|
||||
}
|
||||
|
||||
if ok, err := isLockModificationAllowed(ctx, readLock, lock); !ok {
|
||||
return err
|
||||
}
|
||||
|
||||
// Rewind to the beginning of the file before writing a refreshed lock
|
||||
_, err = f.Seek(0, 0)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not seek to the beginning of the lock file")
|
||||
}
|
||||
if err := json.NewEncoder(f).Encode(lock); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not write lock file")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Unlock unlocks the node
|
||||
func (n *Node) Unlock(ctx context.Context, lock *provider.Lock) error {
|
||||
ctx, span := tracer.Start(ctx, "Unlock")
|
||||
defer span.End()
|
||||
|
||||
// ensure parent path exists
|
||||
if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock")
|
||||
}
|
||||
fileLock, err := filelocks.AcquireWriteLock(n.InternalPath())
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
rerr := filelocks.ReleaseLock(fileLock)
|
||||
|
||||
// if err is non nil we do not overwrite that
|
||||
if err == nil {
|
||||
err = rerr
|
||||
}
|
||||
}()
|
||||
|
||||
f, err := os.OpenFile(n.LockFilePath(), os.O_RDONLY, os.ModeExclusive)
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return errtypes.Aborted("lock does not exist")
|
||||
case err != nil:
|
||||
return errors.Wrap(err, "Decomposedfs: could not open lock file")
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
oldLock := &provider.Lock{}
|
||||
if err := json.NewDecoder(f).Decode(oldLock); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not read lock")
|
||||
}
|
||||
|
||||
// check lock
|
||||
if lock == nil || (oldLock.LockId != lock.LockId) {
|
||||
return errtypes.Locked(oldLock.LockId)
|
||||
}
|
||||
|
||||
if ok, err := isLockModificationAllowed(ctx, oldLock, lock); !ok {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = os.Remove(f.Name()); err != nil {
|
||||
return errors.Wrap(err, "Decomposedfs: could not remove lock file")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// CheckLock compares the context lock with the node lock
|
||||
func (n *Node) CheckLock(ctx context.Context) error {
|
||||
ctx, span := tracer.Start(ctx, "CheckLock")
|
||||
defer span.End()
|
||||
contextLock, _ := ctxpkg.ContextGetLockID(ctx)
|
||||
diskLock, _ := n.ReadLock(ctx, false)
|
||||
if diskLock != nil {
|
||||
switch contextLock {
|
||||
case "":
|
||||
return errtypes.Locked(diskLock.LockId) // no lockid in request
|
||||
case diskLock.LockId:
|
||||
return nil // ok
|
||||
default:
|
||||
return errtypes.Aborted("mismatching lock")
|
||||
}
|
||||
}
|
||||
if contextLock != "" {
|
||||
return errtypes.Aborted("not locked") // no lock on disk. why is there a lockid in the context
|
||||
}
|
||||
return nil // ok
|
||||
}
|
||||
|
||||
func readLocksIntoOpaque(ctx context.Context, n *Node, ri *provider.ResourceInfo) error {
|
||||
lock, err := n.ReadLock(ctx, false)
|
||||
if err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not read lock")
|
||||
return err
|
||||
}
|
||||
|
||||
// reencode to ensure valid json
|
||||
var b []byte
|
||||
if b, err = json.Marshal(lock); err != nil {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not marshal locks")
|
||||
}
|
||||
if ri.Opaque == nil {
|
||||
ri.Opaque = &types.Opaque{
|
||||
Map: map[string]*types.OpaqueEntry{},
|
||||
}
|
||||
}
|
||||
ri.Opaque.Map["lock"] = &types.OpaqueEntry{
|
||||
Decoder: "json",
|
||||
Value: b,
|
||||
}
|
||||
ri.Lock = lock
|
||||
return err
|
||||
}
|
||||
|
||||
func (n *Node) hasLocks(ctx context.Context) bool {
|
||||
_, err := os.Stat(n.LockFilePath()) // FIXME better error checking
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isLockModificationAllowed(ctx context.Context, oldLock *provider.Lock, newLock *provider.Lock) (bool, error) {
|
||||
if oldLock.Type == provider.LockType_LOCK_TYPE_SHARED {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
appNameEquals := oldLock.AppName == newLock.AppName
|
||||
if !appNameEquals {
|
||||
return false, errtypes.PermissionDenied("app names of the locks are mismatching")
|
||||
}
|
||||
|
||||
var lockUserEquals, contextUserEquals bool
|
||||
if oldLock.User == nil && newLock.GetUser() == nil {
|
||||
// no user lock set
|
||||
lockUserEquals = true
|
||||
contextUserEquals = true
|
||||
} else {
|
||||
lockUserEquals = utils.UserIDEqual(oldLock.User, newLock.GetUser())
|
||||
if !lockUserEquals {
|
||||
return false, errtypes.PermissionDenied("users of the locks are mismatching")
|
||||
}
|
||||
|
||||
u := ctxpkg.ContextMustGetUser(ctx)
|
||||
contextUserEquals = utils.UserIDEqual(oldLock.User, u.Id)
|
||||
if !contextUserEquals {
|
||||
return false, errtypes.PermissionDenied("lock holder and current user are mismatching")
|
||||
}
|
||||
}
|
||||
|
||||
return appNameEquals && lockUserEquals && contextUserEquals, nil
|
||||
|
||||
}
|
||||
Generated
Vendored
-1561
File diff suppressed because it is too large.
Load diff
Generated
Vendored
-38
@@ -1,38 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package node
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// GetAvailableSize stats the filesystem and return the available bytes
|
||||
func GetAvailableSize(path string) (uint64, error) {
|
||||
stat := syscall.Statfs_t{}
|
||||
err := syscall.Statfs(path, &stat)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// convert stat.Bavail to uint64 because it returns an int64 on freebsd
|
||||
return uint64(stat.Bavail) * uint64(stat.Bsize), nil //nolint:unconvert
|
||||
}
|
||||
Generated
Vendored
-38
@@ -1,38 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
package node
|
||||
|
||||
import "golang.org/x/sys/windows"
|
||||
|
||||
// GetAvailableSize stats the filesystem and return the available bytes
|
||||
func GetAvailableSize(path string) (uint64, error) {
|
||||
var free, total, avail uint64
|
||||
pathPtr, err := windows.UTF16PtrFromString(path)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
err = windows.GetDiskFreeSpaceEx(pathPtr, &avail, &total, &free)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return avail, nil
|
||||
}
|
||||
Generated
Vendored
-232
@@ -1,232 +0,0 @@
|
||||
// Copyright 2018-2021 CERN
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// In applying this license, CERN does not waive the privileges and immunities
|
||||
// granted to it by virtue of its status as an Intergovernmental Organization
|
||||
// or submit itself to any jurisdiction.
|
||||
|
||||
package node
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
|
||||
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/appctx"
|
||||
ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
|
||||
"github.com/opencloud-eu/reva/v2/pkg/utils"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// PermissionFunc should return true when the user has permission to access the node
|
||||
type PermissionFunc func(*Node) bool
|
||||
|
||||
var (
|
||||
// NoCheck doesn't check permissions, returns true always
|
||||
NoCheck PermissionFunc = func(_ *Node) bool {
|
||||
return true
|
||||
}
|
||||
)
|
||||
|
||||
// NoPermissions represents an empty set of permissions
|
||||
func NoPermissions() *provider.ResourcePermissions {
|
||||
return &provider.ResourcePermissions{}
|
||||
}
|
||||
|
||||
// ShareFolderPermissions defines permissions for the shared jail
|
||||
func ShareFolderPermissions() *provider.ResourcePermissions {
|
||||
return &provider.ResourcePermissions{
|
||||
// read permissions
|
||||
ListContainer: true,
|
||||
Stat: true,
|
||||
InitiateFileDownload: true,
|
||||
GetPath: true,
|
||||
GetQuota: true,
|
||||
ListFileVersions: true,
|
||||
}
|
||||
}
|
||||
|
||||
// OwnerPermissions defines permissions for nodes owned by the user
|
||||
func OwnerPermissions() *provider.ResourcePermissions {
|
||||
return &provider.ResourcePermissions{
|
||||
// all permissions
|
||||
AddGrant: true,
|
||||
CreateContainer: true,
|
||||
Delete: true,
|
||||
GetPath: true,
|
||||
GetQuota: true,
|
||||
InitiateFileDownload: true,
|
||||
InitiateFileUpload: true,
|
||||
ListContainer: true,
|
||||
ListFileVersions: true,
|
||||
ListGrants: true,
|
||||
ListRecycle: true,
|
||||
Move: true,
|
||||
PurgeRecycle: true,
|
||||
RemoveGrant: true,
|
||||
RestoreFileVersion: true,
|
||||
RestoreRecycleItem: true,
|
||||
Stat: true,
|
||||
UpdateGrant: true,
|
||||
DenyGrant: true,
|
||||
}
|
||||
}
|
||||
|
||||
// ServiceAccountPermissions defines the permissions for nodes when requested by a service account
|
||||
func ServiceAccountPermissions() *provider.ResourcePermissions {
|
||||
// TODO: Different permissions for different service accounts
|
||||
return &provider.ResourcePermissions{
|
||||
Stat: true,
|
||||
ListContainer: true,
|
||||
GetPath: true, // for search index
|
||||
InitiateFileUpload: true, // for personal data export
|
||||
InitiateFileDownload: true, // for full-text-search
|
||||
RemoveGrant: true, // for share expiry
|
||||
ListRecycle: true, // for purge-trash-bin command
|
||||
PurgeRecycle: true, // for purge-trash-bin command
|
||||
RestoreRecycleItem: true, // for cli restore command
|
||||
Delete: true, // for cli restore command with replace option
|
||||
CreateContainer: true, // for space provisioning
|
||||
AddGrant: true, // for initial project space member assignment
|
||||
}
|
||||
}
|
||||
|
||||
// Permissions implements permission checks
|
||||
type Permissions struct {
|
||||
lu PathLookup
|
||||
}
|
||||
|
||||
// NewPermissions returns a new Permissions instance
|
||||
func NewPermissions(lu PathLookup) *Permissions {
|
||||
return &Permissions{
|
||||
lu: lu,
|
||||
}
|
||||
}
|
||||
|
||||
// AssemblePermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes
|
||||
func (p *Permissions) AssemblePermissions(ctx context.Context, n *Node) (ap *provider.ResourcePermissions, err error) {
|
||||
return p.assemblePermissions(ctx, n, true)
|
||||
}
|
||||
|
||||
// AssembleTrashPermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes
|
||||
func (p *Permissions) AssembleTrashPermissions(ctx context.Context, n *Node) (ap *provider.ResourcePermissions, err error) {
|
||||
return p.assemblePermissions(ctx, n, false)
|
||||
}
|
||||
|
||||
// assemblePermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes
|
||||
func (p *Permissions) assemblePermissions(ctx context.Context, n *Node, failOnTrashedSubtree bool) (ap *provider.ResourcePermissions, err error) {
|
||||
u, ok := ctxpkg.ContextGetUser(ctx)
|
||||
if !ok {
|
||||
return NoPermissions(), nil
|
||||
}
|
||||
|
||||
if u.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE {
|
||||
return ServiceAccountPermissions(), nil
|
||||
}
|
||||
|
||||
// are we reading a revision?
|
||||
if strings.Contains(n.ID, RevisionIDDelimiter) {
|
||||
// verify revision key format
|
||||
kp := strings.SplitN(n.ID, RevisionIDDelimiter, 2)
|
||||
if len(kp) != 2 {
|
||||
return NoPermissions(), errtypes.NotFound(n.ID)
|
||||
}
|
||||
// use the actual node for the permission assembly
|
||||
n.ID = kp[0]
|
||||
}
|
||||
|
||||
// determine root
|
||||
rn := n.SpaceRoot
|
||||
cn := n
|
||||
ap = &provider.ResourcePermissions{}
|
||||
|
||||
// for an efficient group lookup convert the list of groups to a map
|
||||
// groups are just strings ... groupnames ... or group ids ??? AAARGH !!!
|
||||
groupsMap := make(map[string]bool, len(u.Groups))
|
||||
for i := range u.Groups {
|
||||
groupsMap[u.Groups[i]] = true
|
||||
}
|
||||
|
||||
// for all segments, starting at the leaf
|
||||
for cn.ID != rn.ID {
|
||||
if np, accessDenied, err := cn.ReadUserPermissions(ctx, u); err == nil {
|
||||
// check if we have a denial on this node
|
||||
if accessDenied {
|
||||
return np, nil
|
||||
}
|
||||
AddPermissions(ap, np)
|
||||
} else {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", cn.SpaceID).Str("nodeid", cn.ID).Msg("error reading permissions")
|
||||
// continue with next segment
|
||||
}
|
||||
|
||||
if cn, err = cn.Parent(ctx); err != nil {
|
||||
// We get an error but get a parent, but can not read it from disk (eg. it has been deleted already)
|
||||
if cn != nil {
|
||||
return ap, errors.Wrap(err, "Decomposedfs: error getting parent for node "+cn.ID)
|
||||
}
|
||||
// We do not have a parent, so we assume the next valid parent is the spaceRoot (which must always exist)
|
||||
cn = n.SpaceRoot
|
||||
}
|
||||
if failOnTrashedSubtree && !cn.Exists {
|
||||
return NoPermissions(), errtypes.NotFound(n.ID)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// for the root node
|
||||
if np, accessDenied, err := cn.ReadUserPermissions(ctx, u); err == nil {
|
||||
// check if we have a denial on this node
|
||||
if accessDenied {
|
||||
return np, nil
|
||||
}
|
||||
AddPermissions(ap, np)
|
||||
} else {
|
||||
appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", cn.SpaceID).Str("nodeid", cn.ID).Msg("error reading root node permissions")
|
||||
}
|
||||
|
||||
// check if the current user is the owner
|
||||
if utils.UserIDEqual(u.Id, n.Owner()) {
|
||||
return OwnerPermissions(), nil
|
||||
}
|
||||
|
||||
appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("user", u).Msg("returning agregated permissions")
|
||||
return ap, nil
|
||||
}
|
||||
|
||||
// AddPermissions merges a set of permissions into another
|
||||
// TODO we should use a bitfield for this ...
|
||||
func AddPermissions(l *provider.ResourcePermissions, r *provider.ResourcePermissions) {
|
||||
l.AddGrant = l.AddGrant || r.AddGrant
|
||||
l.CreateContainer = l.CreateContainer || r.CreateContainer
|
||||
l.Delete = l.Delete || r.Delete
|
||||
l.GetPath = l.GetPath || r.GetPath
|
||||
l.GetQuota = l.GetQuota || r.GetQuota
|
||||
l.InitiateFileDownload = l.InitiateFileDownload || r.InitiateFileDownload
|
||||
l.InitiateFileUpload = l.InitiateFileUpload || r.InitiateFileUpload
|
||||
l.ListContainer = l.ListContainer || r.ListContainer
|
||||
l.ListFileVersions = l.ListFileVersions || r.ListFileVersions
|
||||
l.ListGrants = l.ListGrants || r.ListGrants
|
||||
l.ListRecycle = l.ListRecycle || r.ListRecycle
|
||||
l.Move = l.Move || r.Move
|
||||
l.PurgeRecycle = l.PurgeRecycle || r.PurgeRecycle
|
||||
l.RemoveGrant = l.RemoveGrant || r.RemoveGrant
|
||||
l.RestoreFileVersion = l.RestoreFileVersion || r.RestoreFileVersion
|
||||
l.RestoreRecycleItem = l.RestoreRecycleItem || r.RestoreRecycleItem
|
||||
l.Stat = l.Stat || r.Stat
|
||||
l.UpdateGrant = l.UpdateGrant || r.UpdateGrant
|
||||
l.DenyGrant = l.DenyGrant || r.DenyGrant
|
||||
}
|
||||
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user