diff --git a/go.mod b/go.mod index 90d16363fb..bdceca636e 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/go-micro/plugins/v4/store/nats-js-kv v0.0.0-20240726082623-6831adfdcdc4 github.com/go-micro/plugins/v4/wrapper/monitoring/prometheus v1.2.0 github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry v1.2.0 - github.com/go-playground/validator/v10 v10.30.3 + github.com/go-playground/validator/v10 v10.30.5 github.com/go-resty/resty/v2 v2.17.2 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/gofrs/flock v0.13.1 @@ -68,7 +68,7 @@ require ( github.com/open-policy-agent/opa v1.21.1 github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 - github.com/opencloud-eu/reva/v2 v2.52.1 + github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc github.com/opensearch-project/opensearch-go/v4 v4.7.3 github.com/orcaman/concurrent-map v1.0.0 github.com/pkg/errors v0.9.1 @@ -218,7 +218,7 @@ require ( github.com/go-micro/plugins/v4/store/nats-js v1.2.1 // indirect github.com/go-micro/plugins/v4/store/redis v1.2.1 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/locales v0.14.2 // indirect github.com/go-playground/universal-translator v0.18.2 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-sql-driver/mysql v1.10.1 // indirect @@ -259,7 +259,7 @@ require ( github.com/klauspost/crc32 v1.3.0 // indirect github.com/kovidgoyal/go-parallel v1.1.1 // indirect github.com/kovidgoyal/go-shm v1.0.0 // indirect - github.com/leodido/go-urn v1.4.0 // indirect + github.com/leodido/go-urn v1.5.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/dsig v1.4.0 // indirect github.com/lestrrat-go/dsig-secp256k1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index c66e3cb7fe..1c4c8ba53d 100644 --- a/go.sum +++ b/go.sum @@ -373,12 +373,12 @@ github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/locales v0.14.2 h1:d8UmcrM6Nip0hfGZKLGpAvZH37XB4TS0xzK9B56YNCY= +github.com/go-playground/locales v0.14.2/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.2 h1:LCsMLC9RzmbUMNUPVYD15dmcjwYAJhmX8mPZRW4rAVU= github.com/go-playground/universal-translator v0.18.2/go.mod h1:67VZIMp5lQpDWlnStOct22q1bkdJGJqHghbOtmkawxk= -github.com/go-playground/validator/v10 v10.30.3 h1:4MU6YkEwx7GbcPJOZxrtbu+QfF3pJLJuaYTeAH0DYy8= -github.com/go-playground/validator/v10 v10.30.3/go.mod h1:4Axh7oCNGcoGkqLoE4YWt6n20mcEIsPRlB7vPk3lpyc= +github.com/go-playground/validator/v10 v10.30.5 h1:YyCXvVShZbs2Sm3Mb53eNOlhRXctSOzW5QJAouCTZL4= +github.com/go-playground/validator/v10 v10.30.5/go.mod h1:wEqiaov48pXX1kjhc3Da8y0M0Dtg/BK7gurFBLgwFrQ= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk= @@ -610,8 +610,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/leodido/go-urn v1.5.0 h1:pLqT2kq1zpHW/1D18QMjMpdtX7cekxqtJJjg5ANyWw0= +github.com/leodido/go-urn v1.5.0/go.mod h1:9BORnCDhdPBJNDEX+w1bJisa8yOKYi116VeO96s4ifE= github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69 h1:ZLo0bXX17iUZpb/Sp7N+1uK4Jb7dl0fm5x5ETfzYSvY= github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69/go.mod h1:ksG5iXViKefoupjy+0qQjAVoaDnylnQ1ejWl9g14wh8= github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= @@ -761,8 +761,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI= github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 h1:3ptbiFhSK4fzY/rC3idcYUveiwepMoOov1sBpc7Rg3k= github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU= -github.com/opencloud-eu/reva/v2 v2.52.1 h1:c4tjmqtbJeqDjztTNq4Jvl3dWWSMHhXqV1JixKLKbT4= -github.com/opencloud-eu/reva/v2 v2.52.1/go.mod h1:rAwuI3PX8n3RxBXHfpMtAKqfwxTIKhZol1Bi3Qj39p8= +github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc h1:ltmetYT21ZUBhP3cZD/l3cQ+TSHctayzSJpIyG2XdLI= +github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc/go.mod h1:649GUlGdmY6ukU+7G7fHw+EQlZoLSLhztahgqI9TAe4= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI= github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= diff --git a/vendor/github.com/go-playground/locales/README.md b/vendor/github.com/go-playground/locales/README.md index 7b6be2c647..ee6fa0e921 100644 --- a/vendor/github.com/go-playground/locales/README.md +++ b/vendor/github.com/go-playground/locales/README.md @@ -139,11 +139,13 @@ func main() { fmt.Println(l.CardinalPluralRule(1.0, 1)) fmt.Println(l.CardinalPluralRule(3, 0)) - // Ordinal Plural Rules - fmt.Println(l.OrdinalPluralRule(21, 0)) // 21st - fmt.Println(l.OrdinalPluralRule(22, 0)) // 22nd - fmt.Println(l.OrdinalPluralRule(33, 0)) // 33rd - fmt.Println(l.OrdinalPluralRule(34, 0)) // 34th + // Ordinal Plural Rules. The method returns the locales/plurals.Rule + // (e.g. One, Two, Few, Other) for the number; converting that rule into + // the actual suffix ("st", "nd", "rd", "th") is left to the caller. + fmt.Println(l.OrdinalPluralRule(21, 0)) // One (-> "21st") + fmt.Println(l.OrdinalPluralRule(22, 0)) // Two (-> "22nd") + fmt.Println(l.OrdinalPluralRule(33, 0)) // Few (-> "33rd") + fmt.Println(l.OrdinalPluralRule(34, 0)) // Other (-> "34th") // Range Plural Rules fmt.Println(l.RangePluralRule(1, 0, 1, 0)) // 1-1 diff --git a/vendor/github.com/go-playground/locales/en/en.go b/vendor/github.com/go-playground/locales/en/en.go index baf343d4ff..6aa88f7ba9 100644 --- a/vendor/github.com/go-playground/locales/en/en.go +++ b/vendor/github.com/go-playground/locales/en/en.go @@ -498,10 +498,9 @@ func (en *en) FmtTimeShort(t time.Time) string { b := make([]byte, 0, 32) - h := t.Hour() - - if h > 12 { - h -= 12 + h := t.Hour() % 12 + if h == 0 { + h = 12 } b = strconv.AppendInt(b, int64(h), 10) @@ -528,10 +527,9 @@ func (en *en) FmtTimeMedium(t time.Time) string { b := make([]byte, 0, 32) - h := t.Hour() - - if h > 12 { - h -= 12 + h := t.Hour() % 12 + if h == 0 { + h = 12 } b = strconv.AppendInt(b, int64(h), 10) @@ -565,10 +563,9 @@ func (en *en) FmtTimeLong(t time.Time) string { b := make([]byte, 0, 32) - h := t.Hour() - - if h > 12 { - h -= 12 + h := t.Hour() % 12 + if h == 0 { + h = 12 } b = strconv.AppendInt(b, int64(h), 10) @@ -607,10 +604,9 @@ func (en *en) FmtTimeFull(t time.Time) string { b := make([]byte, 0, 32) - h := t.Hour() - - if h > 12 { - h -= 12 + h := t.Hour() % 12 + if h == 0 { + h = 12 } b = strconv.AppendInt(b, int64(h), 10) diff --git a/vendor/github.com/go-playground/locales/rules.go b/vendor/github.com/go-playground/locales/rules.go index 9202900149..6481269e36 100644 --- a/vendor/github.com/go-playground/locales/rules.go +++ b/vendor/github.com/go-playground/locales/rules.go @@ -98,7 +98,7 @@ type Translator interface { // returns the locales narrow weekday given the 'weekday' provided WeekdayNarrow(weekday time.Weekday) string - // WeekdaysNarrowreturns the locales narrow weekdays + // WeekdaysNarrow returns the locales narrow weekdays WeekdaysNarrow() []string // returns the locales short weekday given the 'weekday' provided @@ -113,7 +113,7 @@ type Translator interface { // returns the locales wide weekdays WeekdaysWide() []string - // The following Functions are common Formatting functionsfor the Translator's Locale + // The following Functions are common Formatting functions for the Translator's Locale // returns 'num' with digits/precision of 'v' for locale and handles both Whole and Real numbers based on 'v' FmtNumber(num float64, v uint64) string @@ -228,14 +228,14 @@ func W(n float64, v uint64) (w int64) { s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64) - // with either be '0' or '0.xxxx', so if 1 then w will be zero + // will either be '0' or '0.xxxx', so if 1 then w will be zero // otherwise need to parse if len(s) != 1 { s = s[2:] - end := len(s) + 1 + end := 0 - for i := end; i >= 0; i-- { + for i := len(s) - 1; i >= 0; i-- { if s[i] != '0' { end = i + 1 break @@ -253,7 +253,7 @@ func F(n float64, v uint64) (f int64) { s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64) - // with either be '0' or '0.xxxx', so if 1 then f will be zero + // will either be '0' or '0.xxxx', so if 1 then f will be zero // otherwise need to parse if len(s) != 1 { @@ -270,14 +270,14 @@ func T(n float64, v uint64) (t int64) { s := strconv.FormatFloat(n-float64(int64(n)), 'f', int(v), 64) - // with either be '0' or '0.xxxx', so if 1 then t will be zero + // will either be '0' or '0.xxxx', so if 1 then t will be zero // otherwise need to parse if len(s) != 1 { s = s[2:] - end := len(s) + 1 + end := 0 - for i := end; i >= 0; i-- { + for i := len(s) - 1; i >= 0; i-- { if s[i] != '0' { end = i + 1 break @@ -286,8 +286,11 @@ func T(n float64, v uint64) (t int64) { // ignoring error, because it can't fail as we generated // the string internally from a real number - t, _ = strconv.ParseInt(s[:end], 10, 64) + if end > 0 { + t, _ = strconv.ParseInt(s[:end], 10, 64) + } } return } + diff --git a/vendor/github.com/go-playground/validator/v10/.gitignore b/vendor/github.com/go-playground/validator/v10/.gitignore index 347072e38a..087a2ea52e 100644 --- a/vendor/github.com/go-playground/validator/v10/.gitignore +++ b/vendor/github.com/go-playground/validator/v10/.gitignore @@ -7,6 +7,8 @@ _obj _test bin +.gocache/ +.gomodcache/ # Architecture specific extensions/prefixes *.[568vq] diff --git a/vendor/github.com/go-playground/validator/v10/.golangci.yaml b/vendor/github.com/go-playground/validator/v10/.golangci.yaml index e69d817be1..65efd23828 100644 --- a/vendor/github.com/go-playground/validator/v10/.golangci.yaml +++ b/vendor/github.com/go-playground/validator/v10/.golangci.yaml @@ -14,6 +14,7 @@ linters: - errorlint - exhaustive - exhaustruct + - exhaustruct_v5 - forbidigo - forcetypeassert - funlen diff --git a/vendor/github.com/go-playground/validator/v10/Makefile b/vendor/github.com/go-playground/validator/v10/Makefile index e7caab7f12..bb494a227e 100644 --- a/vendor/github.com/go-playground/validator/v10/Makefile +++ b/vendor/github.com/go-playground/validator/v10/Makefile @@ -3,7 +3,7 @@ GOCMD=go linters-install: @golangci-lint --version >/dev/null 2>&1 || { \ echo "installing linting tools..."; \ - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.0.2; \ + curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v2.13.1; \ } lint: linters-install diff --git a/vendor/github.com/go-playground/validator/v10/README.md b/vendor/github.com/go-playground/validator/v10/README.md index 8462d21149..8129082d23 100644 --- a/vendor/github.com/go-playground/validator/v10/README.md +++ b/vendor/github.com/go-playground/validator/v10/README.md @@ -85,7 +85,7 @@ validate := validator.New(validator.WithRequiredStructEnabled()) | eqcsfield | Field Equals Another Field (relative)| | eqfield | Field Equals Another Field | | fieldcontains | Check the indicated characters are present in the Field | -| fieldexcludes | Check the indicated characters are not present in the field | +| fieldexcludes | Current field does not contain another field's value | | gtcsfield | Field Greater Than Another Relative Field | | gtecsfield | Field Greater Than or Equal To Another Relative Field | | gtefield | Field Greater Than or Equal To Another Field | @@ -132,6 +132,7 @@ validate := validator.New(validator.WithRequiredStructEnabled()) | origin | Web origin (URL with HTTP(S) scheme and host, but no path/query/fragment) | | url_encoded | URL Encoded | | urn_rfc2141 | Urn RFC 2141 String | +| urn_rfc8141 | Urn RFC 8141 String | ### Strings: diff --git a/vendor/github.com/go-playground/validator/v10/baked_in.go b/vendor/github.com/go-playground/validator/v10/baked_in.go index 679729a5fd..eb1c66c45b 100644 --- a/vendor/github.com/go-playground/validator/v10/baked_in.go +++ b/vendor/github.com/go-playground/validator/v10/baked_in.go @@ -143,6 +143,7 @@ var ( "https_url": isHttpsURL, "uri": isURI, "origin": isOrigin, + "urn_rfc8141": isUrnRFC8141, // RFC 8141 "urn_rfc2141": isUrnRFC2141, // RFC 2141 "file": isFile, "filepath": isFilePath, @@ -1695,6 +1696,26 @@ func isHttpsURL(fl FieldLevel) bool { panic(fmt.Sprintf("Bad field type %s", field.Type())) } +// isUrnRFC8141 is the validation function for validating if the current field's value is a valid URN as per RFC 8141. +func isUrnRFC8141(fl FieldLevel) bool { + field := fl.Field() + + switch field.Kind() { + case reflect.String: + + str := field.String() + if str == "" { + return false + } + + _, match := urn.Parse([]byte(str), urn.WithParsingMode(urn.RFC8141Only)) + + return match + } + + panic(fmt.Sprintf("Bad field type %T", field.Interface())) +} + // isUrnRFC2141 is the validation function for validating if the current field's value is a valid URN as per RFC 2141. func isUrnRFC2141(fl FieldLevel) bool { field := fl.Field() @@ -2896,7 +2917,7 @@ func isHostnameRFC1123(fl FieldLevel) bool { func isFQDN(fl FieldLevel) bool { val := fl.Field().String() - if val == "" { + if val == "" || len(strings.TrimSuffix(val, ".")) > 253 { return false } @@ -3024,9 +3045,18 @@ func isHostnamePort(fl FieldLevel) bool { // IsPort validates if the current field's value represents a valid port func isPort(fl FieldLevel) bool { - val := fl.Field().Uint() + field := fl.Field() - return val >= 1 && val <= 65535 + switch field.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + val := field.Int() + return val >= 1 && val <= 65535 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + val := field.Uint() + return val >= 1 && val <= 65535 + default: + panic(fmt.Sprintf("Bad field type %s", field.Type())) + } } // isLowercase is the validation function for validating if the current field's value is a lowercase string. @@ -3510,4 +3540,3 @@ var ( errMethodReturnNoValues = errors.New(`method return o values (void)`) errMethodReturnInvalidType = errors.New(`method should return invalid type`) ) - diff --git a/vendor/github.com/go-playground/validator/v10/doc.go b/vendor/github.com/go-playground/validator/v10/doc.go index 7b1fd078b6..2db8794a73 100644 --- a/vendor/github.com/go-playground/validator/v10/doc.go +++ b/vendor/github.com/go-playground/validator/v10/doc.go @@ -803,13 +803,14 @@ other types. Usage: containsfield=InnerStructField.Field -# Field Excludes Another Field +# Field Excludes Another Field's Value -This does the same as excludes except for struct fields. It should only be used -with string types. See the behavior of reflect.Value.String() for behavior on -other types. +This validates that the current field's string value does not contain the +string value of the field named by the parameter. It should only be used with +string types. If the referenced field cannot be resolved, validation succeeds. +See the behavior of reflect.Value.String() for behavior on other types. - Usage: excludesfield=InnerStructField.Field + Usage: fieldexcludes=InnerStructField.Field # Unique @@ -1026,6 +1027,13 @@ This will accept any uri the golang request uri accepts Usage: uri +# Urn RFC 8141 String + +This validataes that a string value contains a valid URN +according to the RFC 8141 spec. + + Usage: urn_rfc8141 + # Urn RFC 2141 String This validates that a string value contains a valid URN @@ -1437,7 +1445,7 @@ can be used to validate fields typically passed to sockets and connections. # Port -This validates that the value falls within the valid port number range of 1 to 65,535. +This validates that an integer value (signed or unsigned) falls within the valid port range of 1-65,535. Usage: port diff --git a/vendor/github.com/go-playground/validator/v10/translations/en/en.go b/vendor/github.com/go-playground/validator/v10/translations/en/en.go index 516d54cf8d..e670e2ebfe 100644 --- a/vendor/github.com/go-playground/validator/v10/translations/en/en.go +++ b/vendor/github.com/go-playground/validator/v10/translations/en/en.go @@ -310,7 +310,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er return } - if err = ut.Add("max-items", "{0} must contain at maximum {1}", false); err != nil { + if err = ut.Add("max-items", "{0} must contain at most {1}", false); err != nil { return } if err = ut.AddCardinal("max-items-item", "{0} item", locales.PluralRuleOne, false); err != nil { @@ -548,7 +548,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er { tag: "lte", customRegisFunc: func(ut ut.Translator) (err error) { - if err = ut.Add("lte-string", "{0} must be at maximum {1} in length", false); err != nil { + if err = ut.Add("lte-string", "{0} must be at most {1} in length", false); err != nil { return } @@ -564,7 +564,7 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er return } - if err = ut.Add("lte-items", "{0} must contain at maximum {1}", false); err != nil { + if err = ut.Add("lte-items", "{0} must contain at most {1}", false); err != nil { return } @@ -1187,6 +1187,62 @@ func RegisterDefaultTranslations(v *validator.Validate, trans ut.Translator) (er return t }, }, + { + tag: "startswith", + translation: "{0} must start with '{1}'", + override: false, + customTransFunc: func(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), fe.Field(), fe.Param()) + if err != nil { + log.Printf("warning: error translating FieldError: %#v", fe) + return fe.(error).Error() + } + + return t + }, + }, + { + tag: "endswith", + translation: "{0} must end with '{1}'", + override: false, + customTransFunc: func(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), fe.Field(), fe.Param()) + if err != nil { + log.Printf("warning: error translating FieldError: %#v", fe) + return fe.(error).Error() + } + + return t + }, + }, + { + tag: "startsnotwith", + translation: "{0} must not start with '{1}'", + override: false, + customTransFunc: func(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), fe.Field(), fe.Param()) + if err != nil { + log.Printf("warning: error translating FieldError: %#v", fe) + return fe.(error).Error() + } + + return t + }, + }, + { + tag: "endsnotwith", + translation: "{0} must not end with '{1}'", + override: false, + customTransFunc: func(ut ut.Translator, fe validator.FieldError) string { + t, err := ut.T(fe.Tag(), fe.Field(), fe.Param()) + if err != nil { + log.Printf("warning: error translating FieldError: %#v", fe) + return fe.(error).Error() + } + + return t + }, + }, { tag: "excludes", translation: "{0} cannot contain the text '{1}'", diff --git a/vendor/github.com/leodido/go-urn/machine.go b/vendor/github.com/leodido/go-urn/machine.go index aec1ba69cb..82d058673c 100644 --- a/vendor/github.com/leodido/go-urn/machine.go +++ b/vendor/github.com/leodido/go-urn/machine.go @@ -24,6 +24,7 @@ var ( err8141QComponentStart = "expecting only one q-component (starting with the ?= sequence) [col %d]" err8141MalformedRComp = "expecting a non-empty r-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]" err8141MalformedQComp = "expecting a non-empty q-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]" + errInvalidSyntax = "invalid URN syntax [col %d]" ) var _toStateActions []byte = []byte{ 0, 0, 0, 0, 0, 0, 0, 0, @@ -4358,7 +4359,7 @@ func (m *machine) Parse(input []byte) (*URN, error) { if m.parsingMode != RFC8141Only { // Throw an error when: - // - we are entering here matching the the prefix in the namespace identifier part + // - we are entering here matching the prefix in the namespace identifier part // - looking ahead (3 chars) we find a colon if pos := m.p + 3; pos < m.pe && m.data[pos] == 58 && output.prefix != "" { m.err = fmt.Errorf(errNoUrnWithinID, pos) @@ -5026,6 +5027,9 @@ func (m *machine) Parse(input []byte) (*URN, error) { } if m.cs < firstFinal || m.cs == enFail { + if m.err == nil { + m.err = fmt.Errorf(errInvalidSyntax, m.p) + } return nil, m.err } diff --git a/vendor/github.com/leodido/go-urn/machine.go.rl b/vendor/github.com/leodido/go-urn/machine.go.rl index 0a17421998..481e12fd5c 100644 --- a/vendor/github.com/leodido/go-urn/machine.go.rl +++ b/vendor/github.com/leodido/go-urn/machine.go.rl @@ -24,6 +24,7 @@ var ( err8141QComponentStart = "expecting only one q-component (starting with the ?= sequence) [col %d]" err8141MalformedRComp = "expecting a non-empty r-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]" err8141MalformedQComp = "expecting a non-empty q-component containing alnum, hex, or others ([~&()+,-.:=@;$_!*'] or [/?] but not at its beginning) [col %d]" + errInvalidSyntax = "invalid URN syntax [col %d]" ) %%{ @@ -48,7 +49,7 @@ action set_pre { action throw_pre_urn_err { if m.parsingMode != RFC8141Only { // Throw an error when: - // - we are entering here matching the the prefix in the namespace identifier part + // - we are entering here matching the prefix in the namespace identifier part // - looking ahead (3 chars) we find a colon if pos := m.p + 3; pos < m.pe && m.data[pos] == 58 && output.prefix != "" { m.err = fmt.Errorf(errNoUrnWithinID, pos) @@ -366,6 +367,9 @@ func (m *machine) Parse(input []byte) (*URN, error) { %% write exec; if m.cs < first_final || m.cs == en_fail { + if m.err == nil { + m.err = fmt.Errorf(errInvalidSyntax, m.p) + } return nil, m.err } @@ -383,4 +387,4 @@ func (m *machine) WithParsingMode(x ParsingMode) { m.startParsingAt = en_scim_only } m.parsingModeSet = true -} \ No newline at end of file +} diff --git a/vendor/github.com/leodido/go-urn/urn.go b/vendor/github.com/leodido/go-urn/urn.go index 894d6258dc..3c2110dc29 100644 --- a/vendor/github.com/leodido/go-urn/urn.go +++ b/vendor/github.com/leodido/go-urn/urn.go @@ -8,7 +8,7 @@ import ( const errInvalidURN = "invalid URN: %s" -// URN represents an Uniform Resource Name. +// URN represents a Uniform Resource Name. // // The general form represented is: // @@ -32,7 +32,7 @@ type URN struct { // Normalize turns the receiving URN into its norm version. // -// Which means: lowercase prefix, lowercase namespace identifier, and immutate namespace specific string chars (except tokens which are lowercased). +// Which means: lowercase prefix, lowercase namespace identifier, and leave namespace-specific string chars unchanged (except tokens which are lowercased). func (u *URN) Normalize() *URN { return &URN{ prefix: "urn", @@ -82,10 +82,10 @@ func (u *URN) String() string { return res } -// Parse is responsible to create an URN instance from a byte array matching the correct URN syntax (RFC 2141). +// Parse is responsible for creating a URN instance from a byte array matching the correct URN syntax (RFC 2141). func Parse(u []byte, options ...Option) (*URN, bool) { urn, err := NewMachine(options...).Parse(u) - if err != nil { + if err != nil || urn == nil { return nil, false } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider/storageprovider.go b/vendor/github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider/storageprovider.go index dbd9ca7196..f141af610e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider/storageprovider.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider/storageprovider.go @@ -47,6 +47,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" "github.com/opencloud-eu/reva/v2/pkg/storagespace" + "github.com/opencloud-eu/reva/v2/pkg/upload" "github.com/opencloud-eu/reva/v2/pkg/utils" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -71,6 +72,7 @@ type config struct { MountID string `mapstructure:"mount_id"` UploadExpiration int64 `mapstructure:"upload_expiration" docs:"0;Duration for how long uploads will be valid."` Events eventconfig `mapstructure:"events" docs:"0;Event stream configuration"` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` } type eventconfig struct { @@ -106,6 +108,7 @@ func (c *config) init() { type Service struct { conf *config Storage storage.FS + Coordinator upload.Coordinator dataServerURL *url.URL availableXS []*provider.ResourceChecksumPriority } @@ -175,7 +178,14 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. c.init() - fs, err := getFS(c, log) + // One stream for both the driver and the coordinator: a second one would open a + // second nats connection for the same events. + evstream, err := estreamFromConfig(c.Events) + if err != nil { + return nil, err + } + + fs, err := getFS(c, evstream, log) if err != nil { return nil, err } @@ -202,9 +212,16 @@ func New(m map[string]interface{}, ss *grpc.Server, log *zerolog.Logger) (rgrpc. return nil, err } + // storageprovider only initiates uploads; the data path assembles chunks, so no chunking here. + coord, err := upload.NewCoordinatorFromConfig(c.UploadDirectory, c.Drivers[c.Driver], fs, evstream, log, false) + if err != nil { + return nil, fmt.Errorf("storageprovider: %w", err) + } + service := &Service{ conf: c, Storage: fs, + Coordinator: coord, dataServerURL: u, availableXS: xsTypes, } @@ -414,7 +431,7 @@ func (s *Service) InitiateFileUpload(ctx context.Context, req *provider.Initiate metadata["expires"] = strconv.Itoa(int(expirationTimestamp.Seconds)) } - uploadIDs, err := s.Storage.InitiateUpload(ctx, req.Ref, uploadLength, metadata) + uploadIDs, err := s.Coordinator.InitiateUpload(ctx, req.Ref, uploadLength, metadata) if err != nil { var st *rpc.Status switch err.(type) { @@ -582,6 +599,10 @@ func (s *Service) ListStorageSpaces(ctx context.Context, req *provider.ListStora }, nil } + caps := storage.FullCapabilities() + if cp, ok := s.Storage.(storage.CapabilityProvider); ok { + caps = cp.Capabilities(ctx) + } for _, sp := range spaces { if sp.GetId() == nil || sp.GetId().GetOpaqueId() == "" { log.Error().Str("service", "storageprovider").Str("driver", s.conf.Driver).Interface("space", sp).Msg("space is missing space id and root id") @@ -589,6 +610,7 @@ func (s *Service) ListStorageSpaces(ctx context.Context, req *provider.ListStora } s.addMissingStorageProviderID(sp.GetRoot(), sp.GetId()) + sp.Opaque = utils.AppendJSONToOpaque(sp.Opaque, storage.CapabilitiesOpaqueKey, caps) } return &provider.ListStorageSpacesResponse{ @@ -1330,12 +1352,7 @@ func (s *Service) addMissingStorageProviderID(resourceID *provider.ResourceId, s } } -func getFS(c *config, log *zerolog.Logger) (storage.FS, error) { - evstream, err := estreamFromConfig(c.Events) - if err != nil { - return nil, err - } - +func getFS(c *config, evstream events.Stream, log *zerolog.Logger) (storage.FS, error) { if f, ok := registry.NewFuncs[c.Driver]; ok { driverConf := c.Drivers[c.Driver] driverConf["mount_id"] = c.MountID // pass the mount id to the driver diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go index 7b7bec0727..45decb911a 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/basic/basic.go @@ -52,5 +52,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re // fall back to hostname if not configured realm = r.Host } - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Basic realm="%s"`, realm)) + w.Header().Add("WWW-Authenticate", auth.Challenge("Basic", realm)) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go index 62168019f0..3ed45596cf 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/interceptors/auth/credential/strategy/bearer/bearer.go @@ -63,5 +63,5 @@ func (s *strategy) AddWWWAuthenticate(w http.ResponseWriter, r *http.Request, re // fall back to hostname if not configured realm = r.Host } - w.Header().Add("WWW-Authenticate", fmt.Sprintf(`Bearer realm="%s"`, realm)) + w.Header().Add("WWW-Authenticate", auth.Challenge("Bearer", realm)) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/dataprovider/dataprovider.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/dataprovider/dataprovider.go index ad7b5be561..79ca3b4750 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/dataprovider/dataprovider.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/dataprovider/dataprovider.go @@ -33,6 +33,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/rhttp/router" "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" + "github.com/opencloud-eu/reva/v2/pkg/upload" ) func init() { @@ -51,6 +52,7 @@ type config struct { NatsEnableTLS bool `mapstructure:"nats_enable_tls"` NatsUsername string `mapstructure:"nats_username"` NatsPassword string `mapstructure:"nats_password"` + UploadDirectory string `mapstructure:"upload_directory" docs:";Local directory for staging upload sessions. Overrides the driver's root. Required for drivers that have no local filesystem root."` } func (c *config) init() { @@ -104,7 +106,21 @@ func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) return nil, err } - dataTXs, err := getDataTXs(conf, fs, evstream, log) + // the data path assembles chunks, so enable chunking + coord, err := upload.NewCoordinatorFromConfig(conf.UploadDirectory, conf.Drivers[conf.Driver], fs, evstream, log, true) + if err != nil { + return nil, fmt.Errorf("dataprovider: %w", err) + } + + // only the data path consumes postprocessing results: one consumer group gets + // one copy of each event, so a second subscriber would take half of them + if ac := upload.AsyncConfFromDriverConf(conf.Drivers[conf.Driver]); ac.Enabled { + if err := coord.StartPostprocessing(evstream, ac); err != nil { + return nil, fmt.Errorf("dataprovider: could not start postprocessing: %w", err) + } + } + + dataTXs, err := getDataTXs(conf, coord, fs, evstream, log) if err != nil { return nil, err } @@ -126,7 +142,7 @@ func getFS(c *config, stream events.Stream, log *zerolog.Logger) (storage.FS, er return nil, fmt.Errorf("driver not found: %s", c.Driver) } -func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { +func getDataTXs(c *config, coord upload.Coordinator, fs storage.FS, publisher events.Publisher, log *zerolog.Logger) (map[string]http.Handler, error) { if c.DataTXs == nil { c.DataTXs = make(map[string]map[string]interface{}) } @@ -146,7 +162,7 @@ func getDataTXs(c *config, fs storage.FS, publisher events.Publisher, log *zerol for t := range c.DataTXs { if f, ok := datatxregistry.NewFuncs[t]; ok { if tx, err := f(c.DataTXs[t], publisher, log); err == nil { - if handler, err := tx.Handler(fs); err == nil { + if handler, err := tx.Handler(coord, fs); err == nil { txs[t] = handler } } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go index 30dd16ab9c..2b1c0bc91b 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/dav.go @@ -20,7 +20,6 @@ package ocdav import ( "context" - "fmt" "net/http" "path" "path/filepath" @@ -35,6 +34,7 @@ import ( "github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/errors" "github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/auth" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/reva/v2/pkg/rhttp/router" @@ -438,7 +438,7 @@ func handleBasicAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta } func authChallenge(scheme, realm string) string { - return fmt.Sprintf("%s realm=\"%s\", charset=\"UTF-8\"", scheme, realm) + return auth.Challenge(scheme, realm, `charset="UTF-8"`) } func handleSignatureAuth(ctx context.Context, selector pool.Selectable[gatewayv1beta1.GatewayAPIClient], token, sig, expiration string) (*gatewayv1beta1.AuthenticateResponse, error) { diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go index da869a905b..f707a2b1ff 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/propfind/propfind.go @@ -1538,6 +1538,8 @@ func mdToPropResponse(ctx context.Context, pf *XML, md *provider.ResourceInfo, p types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeUser)) + "") case "2": // provider.GranteeType_GRANTEE_TYPE_GROUP types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeGroup)) + "") + case "4": // conversions.ShareTypeGuest, guest shares are stored as user grants + types.WriteString("" + strconv.Itoa(int(conversions.ShareTypeGuest)) + "") default: sublog.Debug().Interface("shareType", shareType).Msg("unknown share type, ignoring") } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go index fc567f627e..329dc4b52b 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/tus.go @@ -337,7 +337,8 @@ func (s *svc) handleTusPost(ctx context.Context, w http.ResponseWriter, r *http. sReq.Ref.Path = uReq.Ref.GetPath() sReq.Ref.ResourceId = nil } else { - if resid, err := storagespace.ParseID(httpRes.Header.Get(net.HeaderOCFileID)); err == nil { + // new files have no node id yet; keep the path-based ref instead + if resid, err := storagespace.ParseID(httpRes.Header.Get(net.HeaderOCFileID)); err == nil && resid.GetOpaqueId() != "" { sReq.Ref = &provider.Reference{ ResourceId: &resid, } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/capabilities.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/capabilities.go index 350fed869c..e746485cb7 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/capabilities.go +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/capabilities.go @@ -31,6 +31,7 @@ type Handler struct { c ocs.CapabilitiesData defaultUploadProtocol string userAgentChunkingMap map[string]string + gatewayAddr string } // Init initializes this and any contained handlers @@ -38,6 +39,7 @@ func (h *Handler) Init(c *config.Config) { h.c = c.Capabilities h.defaultUploadProtocol = c.DefaultUploadProtocol h.userAgentChunkingMap = c.UserAgentChunkingMap + h.gatewayAddr = c.GatewaySvc // capabilities if h.c.Capabilities == nil { @@ -120,8 +122,9 @@ func (h *Handler) Init(c *config.Config) { if h.c.Capabilities.Dav == nil { h.c.Capabilities.Dav = &ocs.CapabilitiesDav{} } - if h.c.Capabilities.Dav.Trashbin == "" { - h.c.Capabilities.Dav.Trashbin = "1.0" + // Trashbin is deprecated for clients, but we keep announcing it until they read the providers section. + if h.c.Capabilities.Dav.Trashbin == "" { //nolint:staticcheck // SA1019: still announced for older clients + h.c.Capabilities.Dav.Trashbin = "1.0" //nolint:staticcheck // SA1019: still announced for older clients } if h.c.Capabilities.Dav.Reports == nil { h.c.Capabilities.Dav.Reports = []string{} @@ -241,5 +244,10 @@ func (h *Handler) Init(c *config.Config) { // Handler renders the capabilities func (h *Handler) GetCapabilities(w http.ResponseWriter, r *http.Request) { c := h.getCapabilitiesForUserAgent(r.UserAgent()) + if providers := h.resolveProviders(r.Context()); len(providers) > 0 && c.Capabilities != nil { + caps := *c.Capabilities + caps.Providers = providers + c.Capabilities = &caps + } response.WriteOCSSuccess(w, r, c) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/providers.go b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/providers.go new file mode 100644 index 0000000000..f0ce769095 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocs/handlers/cloud/capabilities/providers.go @@ -0,0 +1,53 @@ +package capabilities + +import ( + "context" + + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs" + "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" + "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/utils" +) + +// resolveProviders builds the per-provider capability map from the spaces visible +// to the current user, keyed by provider ID. Returns nil on no user context or an +// unreachable gateway, so the response omits the section rather than failing. +func (h *Handler) resolveProviders(ctx context.Context) map[string]*ocs.ProviderCapabilities { + log := appctx.GetLogger(ctx) + + gc, err := pool.GetGatewayServiceClient(h.gatewayAddr) + if err != nil { + log.Error().Err(err).Msg("capabilities: error getting gateway client") + return nil + } + + res, err := gc.ListStorageSpaces(ctx, &provider.ListStorageSpacesRequest{}) + if err != nil { + log.Error().Err(err).Msg("capabilities: error listing storage spaces") + return nil + } + if res.GetStatus().GetCode() != rpc.Code_CODE_OK { + return nil + } + + providers := map[string]*ocs.ProviderCapabilities{} + for _, sp := range res.GetStorageSpaces() { + id := sp.GetRoot().GetStorageId() + if id == "" { + continue + } + if _, done := providers[id]; done { + continue + } + var caps storage.Capabilities + if err := utils.ReadJSONFromOpaque(sp.GetOpaque(), storage.CapabilitiesOpaqueKey, &caps); err != nil { + continue + } + providers[id] = ocs.NewProviderCapabilities(caps) + } + return providers +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go new file mode 100644 index 0000000000..b4091811f0 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/auth/challenge.go @@ -0,0 +1,14 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import "strings" + +var quotedStringEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) + +// Challenge builds a WWW-Authenticate challenge for the given scheme with the +// realm escaped as a quoted-string. Additional params are appended without escaping. +func Challenge(scheme, realm string, params ...string) string { + return strings.Join(append([]string{scheme + ` realm="` + quotedStringEscaper.Replace(realm) + `"`}, params...), ", ") +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go index 699dd89961..bf013d438e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/conversions/main.go @@ -61,6 +61,9 @@ const ( // ShareTypeSpaceMembershipGroup represents an action regarding group type space members ShareTypeSpaceMembershipGroup ShareType = 8 + // ShareTypeGuest represents a guest share + ShareTypeGuest ShareType = 4 + // ShareWithUserTypeUser represents a normal user ShareWithUserTypeUser ShareWithUserType = 0 diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/events/postprocessing.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/events/postprocessing.go index e443f82947..3769cb7a8b 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/events/postprocessing.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/events/postprocessing.go @@ -103,6 +103,7 @@ type PostprocessingStepFinished struct { UploadID string ExecutingUser *user.User Filename string + ResourceID *provider.ResourceId FinishedStep Postprocessingstep // name of the step Result interface{} // result information see VirusscanResult for example @@ -145,6 +146,7 @@ type VirusscanResult struct { type PostprocessingFinished struct { UploadID string Filename string + ResourceID *provider.ResourceId SpaceOwner *user.UserId ExecutingUser *user.User Result map[Postprocessingstep]interface{} // it is a map[step]Event diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/mime/mime.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/mime/mime.go index f88ba5cc18..1165b8c32b 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/mime/mime.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/mime/mime.go @@ -33,6 +33,7 @@ var mimeTypes = map[string]string{ "3g2": "video/3gpp2", "3gp": "video/3gpp", "3gpp": "video/3gpp", + "3mf": "model/3mf", "7z": "application/x-7z-compressed", "aab": "application/x-authorware-bin", "aac": "audio/x-aac", @@ -82,6 +83,7 @@ var mimeTypes = map[string]string{ "azs": "application/vnd.airzip.filesecure.azs", "azv": "image/vnd.airzip.accelerator.azv", "azw": "application/vnd.amazon.ebook", + "azw3": "application/vnd.amazon.mobi8-ebook", "bat": "application/x-msdownload", "bcpio": "application/x-bcpio", "bdf": "application/x-font-bdf", @@ -91,6 +93,7 @@ var mimeTypes = map[string]string{ "bh2": "application/vnd.fujitsu.oasysprs", "bin": "application/octet-stream", "blb": "application/x-blorb", + "blend": "application/x-blender", "blorb": "application/x-blorb", "bmi": "application/vnd.bmi", "bmp": "image/x-ms-bmp", @@ -276,6 +279,7 @@ var mimeTypes = map[string]string{ "f4v": "video/x-f4v", "f77": "text/x-fortran", "f90": "text/x-fortran", + "fb2": "application/x-fictionbook+xml", "fbs": "image/vnd.fastbidsheet", "fcdt": "application/vnd.adobe.formscentral.fcdt", "fcs": "application/vnd.isac.fcs", @@ -407,6 +411,7 @@ var mimeTypes = map[string]string{ "iota": "application/vnd.astraea-software.iota", "ipfix": "application/ipfix", "ipk": "application/vnd.shana.informed.package", + "ipynb": "application/x-ipynb+json", "irm": "application/vnd.ibm.rights-management", "irp": "application/vnd.irepository.package+xml", "iso": "application/x-iso9660-image", @@ -456,6 +461,7 @@ var mimeTypes = map[string]string{ "kpr": "application/vnd.kde.kpresenter", "kpt": "application/vnd.kde.kpresenter", "kpxx": "application/vnd.ds-keypoint", + "kra": "application/x-krita", "ksp": "application/vnd.kde.kspread", "ktr": "application/vnd.kahootz", "ktx": "image/ktx", @@ -495,6 +501,7 @@ var mimeTypes = map[string]string{ "m3u": "audio/x-mpegurl", "m3u8": "application/vnd.apple.mpegurl", "m4a": "audio/x-m4a", + "m4b": "audio/x-m4b", "m4p": "application/mp4", "m4u": "video/vnd.mpegurl", "m4v": "video/x-m4v", diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/ocm.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/ocm.go index 4936a46008..61a5035758 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/ocm.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/ocm.go @@ -40,6 +40,7 @@ import ( "github.com/rs/zerolog" "github.com/studio-b12/gowebdav" + ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/mime" @@ -218,7 +219,7 @@ func (d *driver) CreateDir(ctx context.Context, ref *provider.Reference) error { } func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { - client, _, rel, err := d.webdavClient(ctx, nil, ref) + client, _, rel, err := d.serviceWebdavClient(ctx, ref) if err != nil { return err } @@ -226,7 +227,7 @@ func (d *driver) Delete(ctx context.Context, ref *provider.Reference) error { } func (d *driver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error { - client, _, rel, err := d.webdavClient(ctx, nil, ref) + client, _, rel, err := d.serviceWebdavClient(ctx, ref) if err != nil { return err } @@ -327,7 +328,19 @@ func extractChecksum(props gowebdav.Props) *provider.ResourceChecksum { } func (d *driver) GetMD(ctx context.Context, ref *provider.Reference, _ []string, _ []string) (*provider.ResourceInfo, error) { - client, share, rel, err := d.webdavClient(ctx, nil, ref) + var ( + client *gowebdav.Client + share *ocmpb.ReceivedShare + rel string + err error + ) + if token, ok := ctxpkg.ContextGetToken(ctx); ok && token != "" { + client, share, rel, err = d.webdavClient(ctx, nil, ref) + } else { + // the upload coordinator stats an upload it finishes with the context of the + // upload's executant, whose request, and with it the token, is gone by then + client, share, rel, err = d.serviceWebdavClient(ctx, ref) + } if err != nil { return nil, err } @@ -494,6 +507,10 @@ func (d *driver) GetLock(ctx context.Context, ref *provider.Reference) (*provide if err != nil { return nil, err } + // gowebdav formats a missing lockdiscovery property as "" + if token == "" || token == "" { + return nil, errtypes.NotFound("no lock found") + } return &provider.Lock{LockId: token, Type: provider.LockType_LOCK_TYPE_EXCL}, nil } @@ -617,3 +634,11 @@ func (d *driver) AddLabel(ctx context.Context, ref *provider.Reference, userID * func (d *driver) RemoveLabel(ctx context.Context, ref *provider.Reference, userID *userpb.UserId, label string) error { return errtypes.NotSupported("RemoveLabel not implemented") } + +func (d *driver) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil +} + +func (d *driver) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { + return nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/upload.go index 1331c9e6e3..5753a66a5d 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/ocm/storage/received/upload.go @@ -20,399 +20,57 @@ package ocm import ( "context" - "crypto/md5" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "hash" - "hash/adler32" - "io" "net/http" - "os" - "path/filepath" - "strings" - "github.com/google/uuid" - tusd "github.com/tus/tusd/v2/pkg/handler" + "github.com/studio-b12/gowebdav" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + ocmpb "github.com/cs3org/go-cs3apis/cs3/sharing/ocm/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/utils" ) -var defaultFilePerm = os.FileMode(0664) - -func (d *driver) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { - return []storage.UploadSession{}, nil -} -func (d *driver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - shareID, rel := shareInfoFromReference(ref) - p := getPathFromShareIDAndRelPath(shareID, rel) - - info := tusd.FileInfo{ - MetaData: tusd.MetaData{ - "filename": filepath.Base(p), - "dir": filepath.Dir(p), - }, - Size: uploadLength, - } - - upload, err := d.NewUpload(ctx, info) - if err != nil { - return nil, err - } - - info, _ = upload.GetInfo(ctx) - - return map[string]string{ - "simple": info.ID, - "tus": info.ID, - }, nil -} - +// MarkProcessing is a no-op: the file lives on the remote instance, so there is no +// local node to flag while postprocessing runs. func (d *driver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { - return errtypes.NotSupported("op not supported") + return nil } -func (d *driver) CommitUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.UploadSource) error { - return errtypes.NotSupported("op not supported") -} - -func (d *driver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - shareID, _ := shareInfoFromReference(req.Ref) - u, err := d.GetUpload(ctx, shareID.OpaqueId) +// CommitUpload streams the staged bytes to the remote instance. +func (d *driver) CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source storage.UploadSource) error { + client, _, rel, err := d.serviceWebdavClient(ctx, ref) if err != nil { - return &provider.ResourceInfo{}, err + return err } - info, err := u.GetInfo(ctx) - if err != nil { - return &provider.ResourceInfo{}, err - } - - defer cleanup(&upload{Info: info}) - - client, _, rel, err := d.webdavClient(ctx, nil, &provider.Reference{ - Path: filepath.Join(info.MetaData["dir"], info.MetaData["filename"]), - }) - if err != nil { - return &provider.ResourceInfo{}, err - } client.SetInterceptor(func(method string, rq *http.Request) { // Set the content length on the request struct directly instead of the header. // The content-length header gets reset by the golang http library before - // sendind out the request, resulting in chunked encoding to be used which + // sending out the request, resulting in chunked encoding to be used which // breaks the quota checks in ocdav. if method == "PUT" { - rq.ContentLength = req.Length + rq.ContentLength = source.Length } }) - locktoken, _ := ctxpkg.ContextGetLockID(ctx) - return &provider.ResourceInfo{}, client.WriteStream(rel, req.Body, 0, locktoken) + // the coordinator owns source.Body and closes it after this returns + lockToken, _ := ctxpkg.ContextGetLockID(ctx) + return client.WriteStream(rel, source.Body, 0, lockToken) } -// UseIn tells the tus upload middleware which extensions it supports. -func (d *driver) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(d) - composer.UseTerminater(d) - composer.UseConcater(d) - composer.UseLengthDeferrer(d) -} - -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (d *driver) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (d *driver) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (d *driver) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload) -} - -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload - -// NewUpload returns a new tus Upload instance -func (d *driver) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) { - return NewUpload(ctx, d, d.c.StorageRoot, info) -} - -// GetUpload returns the Upload for the given upload id -func (d *driver) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - return GetUpload(ctx, d, d.c.StorageRoot, id) -} -func NewUpload(ctx context.Context, d *driver, storageRoot string, info tusd.FileInfo) (tusd.Upload, error) { - if info.MetaData["filename"] == "" { - return nil, errors.New("decomposedfs: missing filename in metadata") +// serviceWebdavClient builds a webdav client without needing a request auth token: +// it authenticates as the service account and looks up the share on behalf of the +// upload's executant. +func (d *driver) serviceWebdavClient(ctx context.Context, ref *provider.Reference) (*gowebdav.Client, *ocmpb.ReceivedShare, string, error) { + serviceUserCtx, err := utils.GetServiceUserContextWithContext(context.Background(), d.gateway, d.c.ServiceAccountID, d.c.ServiceAccountSecret) + if err != nil { + return nil, nil, "", err } - if info.MetaData["dir"] == "" { - return nil, errors.New("decomposedfs: missing dir in metadata") - } - - uploadRoot := filepath.Join(storageRoot, "uploads") - info.ID = uuid.New().String() - - user, ok := ctxpkg.ContextGetUser(ctx) + executant, ok := ctxpkg.ContextGetUser(ctx) if !ok { - return nil, errors.New("no user in context") + return nil, nil, "", errtypes.BadRequest("ocm: no executant in context") } - info.MetaData["user"] = user.GetId().GetOpaqueId() - info.MetaData["idp"] = user.GetId().GetIdp() - - info.Storage = map[string]string{ - "Type": "OCM", - "Path": uploadRoot, - } - - u := &upload{ - Info: info, - Ctx: ctx, - d: d, - } - - err := os.MkdirAll(uploadRoot, 0755) - if err != nil { - return nil, err - } - - file, err := os.OpenFile(u.BinPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return nil, err - } - defer file.Close() - - err = u.Persist() - if err != nil { - return nil, err - } - return u, nil -} - -func GetUpload(ctx context.Context, d *driver, storageRoot string, id string) (tusd.Upload, error) { - info := tusd.FileInfo{} - data, err := os.ReadFile(filepath.Join(storageRoot, "uploads", id+".info")) - if err != nil { - return nil, err - } - err = json.Unmarshal(data, &info) - if err != nil { - return nil, err - } - upload := &upload{ - Info: info, - Ctx: ctx, - d: d, - } - return upload, nil -} - -type upload struct { - Info tusd.FileInfo - Ctx context.Context - - d *driver -} - -func (u *upload) InfoPath() string { - return filepath.Join(u.Info.Storage["Path"], u.Info.ID+".info") -} - -func (u *upload) BinPath() string { - return filepath.Join(u.Info.Storage["Path"], u.Info.ID) -} - -func (u *upload) Persist() error { - data, err := json.Marshal(u.Info) - if err != nil { - return err - } - return os.WriteFile(u.InfoPath(), data, defaultFilePerm) -} - -func (u *upload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return 0, err - } - defer file.Close() - - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... - n, err := io.Copy(file, src) - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the ocm driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil && err != io.ErrUnexpectedEOF { - return n, err - } - - u.Info.Offset += n - return n, u.Persist() -} - -func (u *upload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { - return u.Info, nil -} - -func (u *upload) GetReader(ctx context.Context) (io.ReadCloser, error) { - return os.Open(u.BinPath()) -} - -func (u *upload) FinishUpload(ctx context.Context) error { - log := appctx.GetLogger(u.Ctx) - - // calculate the checksum of the written bytes - // they will all be written to the metadata later, so we cannot omit any of them - // TODO only calculate the checksum in sync that was requested to match, the rest could be async ... but the tests currently expect all to be present - // TODO the hashes all implement BinaryMarshaler so we could try to persist the state for resumable upload. we would neet do keep track of the copied bytes ... - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - { - f, err := os.Open(u.BinPath()) - if err != nil { - // we can continue if no oc checksum header is set - log.Info().Err(err).Str("binPath", u.BinPath()).Msg("error opening binPath") - } - defer f.Close() - - r1 := io.TeeReader(f, sha1h) - r2 := io.TeeReader(r1, md5h) - - _, err = io.Copy(adler32h, r2) - if err != nil { - log.Info().Err(err).Msg("error copying checksums") - } - } - - defer cleanup(u) - // compare if they match the sent checksum - // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - if u.Info.MetaData["checksum"] != "" { - var err error - parts := strings.SplitN(u.Info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = u.checkHash(parts[1], sha1h) - case "md5": - err = u.checkHash(parts[1], md5h) - case "adler32": - err = u.checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if err != nil { - return err - } - } - - // send to the remote storage via webdav - // shareID, rel := shareInfoFromReference(u.Info.MetaData["ref"]) - // p := getPathFromShareIDAndRelPath(shareID, rel) - - serviceUserCtx, err := utils.GetServiceUserContextWithContext(context.Background(), u.d.gateway, u.d.c.ServiceAccountID, u.d.c.ServiceAccountSecret) - if err != nil { - return err - } - client, _, rel, err := u.d.webdavClient(serviceUserCtx, &userpb.UserId{ - OpaqueId: u.Info.MetaData["user"], - Idp: u.Info.MetaData["idp"], - }, &provider.Reference{ - Path: filepath.Join(u.Info.MetaData["dir"], u.Info.MetaData["filename"]), - }) - if err != nil { - return err - } - - client.SetInterceptor(func(method string, rq *http.Request) { - // Set the content length on the request struct directly instead of the header. - // The content-length header gets reset by the golang http library before - // sendind out the request, resulting in chunked encoding to be used which - // breaks the quota checks in ocdav. - if method == "PUT" { - rq.ContentLength = u.Info.Size - } - }) - - f, err := os.Open(u.BinPath()) - if err != nil { - return err - } - defer f.Close() - return client.WriteStream(rel, f, 0, "") -} - -func (u *upload) Terminate(ctx context.Context) error { - cleanup(u) - return nil -} - -func (u *upload) ConcatUploads(_ context.Context, uploads []tusd.Upload) error { - file, err := os.OpenFile(u.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*upload) - - src, err := os.Open(fileUpload.BinPath()) - if err != nil { - return err - } - defer src.Close() - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - return nil -} - -func (u *upload) DeclareLength(ctx context.Context, length int64) error { - u.Info.Size = length - u.Info.SizeIsDeferred = false - return nil -} - -func (u *upload) checkHash(expected string, h hash.Hash) error { - if expected != hex.EncodeToString(h.Sum(nil)) { - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", u.Info.MetaData["checksum"], h.Sum(nil))) - } - return nil -} - -func cleanup(u *upload) { - if u == nil { - return - } - _ = os.Remove(u.BinPath()) - _ = os.Remove(u.InfoPath()) + return d.webdavClient(serviceUserCtx, executant.GetId(), ref) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs/capabilities.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs/capabilities.go index 871b182b95..ea4b73c207 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs/capabilities.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/owncloud/ocs/capabilities.go @@ -20,6 +20,10 @@ package ocs import ( "encoding/xml" + "maps" + "slices" + + "github.com/opencloud-eu/reva/v2/pkg/storage" ) // ocsBool implements the xml/json Marshaler interface. The OCS API inconsistency require us to parse boolean values @@ -62,6 +66,63 @@ type Capabilities struct { Theme *CapabilitiesTheme `json:"theme,omitempty" xml:"theme,omitempty" mapstructure:"theme"` Notifications *CapabilitiesNotifications `json:"notifications,omitempty" xml:"notifications,omitempty"` Groupware *CapabilitiesGroupware `json:"groupware,omitempty" xml:"groupware,omitempty" mapstructure:"groupware"` + // Providers is the per-provider capability section, keyed by provider ID. It + // supersedes the deprecated global storage keys under Files/Dav; there is no + // fallback between them. + Providers ProviderCapabilitiesMap `json:"providers,omitempty" xml:"providers,omitempty" mapstructure:"providers"` +} + +// ProviderCapabilitiesMap is the per-provider section keyed by provider ID. +type ProviderCapabilitiesMap map[string]*ProviderCapabilities + +// MarshalXML renders the map as ......, +// ordered by id. encoding/xml cannot marshal a Go map, so XML output is produced +// explicitly; JSON uses the default map encoding. +func (m ProviderCapabilitiesMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if len(m) == 0 { + return nil + } + if err := e.EncodeToken(start); err != nil { + return err + } + for _, id := range slices.Sorted(maps.Keys(m)) { + el := xml.StartElement{ + Name: xml.Name{Local: "provider"}, + Attr: []xml.Attr{{Name: xml.Name{Local: "id"}, Value: id}}, + } + if err := e.EncodeElement(m[id], el); err != nil { + return err + } + } + return e.EncodeToken(start.End()) +} + +// ProviderCapabilities is the wire form of a provider's declared capabilities. +type ProviderCapabilities struct { + Upload ocsBool `json:"upload" xml:"upload"` + CreateContainer ocsBool `json:"create_container" xml:"create_container"` + Delete ocsBool `json:"delete" xml:"delete"` + Move ocsBool `json:"move" xml:"move"` + Versioning ocsBool `json:"versioning" xml:"versioning"` + Trash ocsBool `json:"trash" xml:"trash"` + Locking ocsBool `json:"locking" xml:"locking"` + Sharing ocsBool `json:"sharing" xml:"sharing"` + ArbitraryMetadata ocsBool `json:"arbitrary_metadata" xml:"arbitrary_metadata"` +} + +// NewProviderCapabilities maps a driver's declaration onto the OCS response type. +func NewProviderCapabilities(c storage.Capabilities) *ProviderCapabilities { + return &ProviderCapabilities{ + Upload: ocsBool(c.Upload), + CreateContainer: ocsBool(c.CreateContainer), + Delete: ocsBool(c.Delete), + Move: ocsBool(c.Move), + Versioning: ocsBool(c.Versioning), + Trash: ocsBool(c.Trash), + Locking: ocsBool(c.Locking), + Sharing: ocsBool(c.Sharing), + ArbitraryMetadata: ocsBool(c.ArbitraryMetadata), + } } // CapabilitiesSearch holds the search capabilities @@ -192,10 +253,13 @@ type CapabilitiesAppProvider struct { // CapabilitiesFiles TODO this is storage specific, not global. What effect do these options have on the clients? type CapabilitiesFiles struct { - PrivateLinks ocsBool `json:"privateLinks" xml:"privateLinks" mapstructure:"private_links"` - BigFileChunking ocsBool `json:"bigfilechunking" xml:"bigfilechunking"` - Undelete ocsBool `json:"undelete" xml:"undelete"` - Versioning ocsBool `json:"versioning" xml:"versioning"` + PrivateLinks ocsBool `json:"privateLinks" xml:"privateLinks" mapstructure:"private_links"` + BigFileChunking ocsBool `json:"bigfilechunking" xml:"bigfilechunking"` + // Deprecated: use capabilities.providers[].trash. + Undelete ocsBool `json:"undelete" xml:"undelete"` + // Deprecated: use capabilities.providers[].versioning. + Versioning ocsBool `json:"versioning" xml:"versioning"` + // Deprecated: use capabilities.providers[].arbitrary_metadata. Favorites ocsBool `json:"favorites" xml:"favorites"` FullTextSearch ocsBool `json:"full_text_search" xml:"full_text_search" mapstructure:"full_text_search"` Tags ocsBool `json:"tags" xml:"tags"` @@ -207,7 +271,8 @@ type CapabilitiesFiles struct { // CapabilitiesDav holds dav endpoint config type CapabilitiesDav struct { - Chunking string `json:"chunking" xml:"chunking"` + Chunking string `json:"chunking" xml:"chunking"` + // Deprecated: use capabilities.providers[].trash. Trashbin string `json:"trashbin" xml:"trashbin"` Reports []string `json:"reports" xml:"reports>element" mapstructure:"reports"` ChunkingParallelUploadDisabled bool `json:"chunkingParallelUploadDisabled" xml:"chunkingParallelUploadDisabled"` diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/datatx.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/datatx.go index 40246112cf..c2d53e1a43 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/datatx.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/datatx.go @@ -28,12 +28,15 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/upload" "github.com/opencloud-eu/reva/v2/pkg/utils" ) // DataTX provides an abstraction around various data transfer protocols. type DataTX interface { - Handler(fs storage.FS) (http.Handler, error) + // Handler serves the protocol's data path. Uploads go through coord, which + // owns the upload lifecycle for every driver; downloads read from driver. + Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error) } // EmitFileUploadedEvent is a helper function which publishes a FileUploaded event diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go index f1dbe801f7..52c63fdabe 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/simple/simple.go @@ -40,6 +40,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/cache" "github.com/opencloud-eu/reva/v2/pkg/storagespace" + "github.com/opencloud-eu/reva/v2/pkg/upload" "github.com/opencloud-eu/reva/v2/pkg/utils" ) @@ -78,7 +79,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("path", r.URL.Path).Logger() r = r.WithContext(appctx.WithLogger(r.Context(), &sublog)) @@ -92,7 +93,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, "") + download.GetOrHeadFile(w, r, driver, "") case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -114,7 +115,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { ctx = ctxpkg.ContextSetLockID(ctx, lockID) } - info, err := fs.Upload(ctx, storage.UploadRequest{ + info, err := coord.Upload(ctx, upload.Request{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go index 133120a89d..50fb54eff6 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/spaces/spaces.go @@ -42,6 +42,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/cache" "github.com/opencloud-eu/reva/v2/pkg/storagespace" + "github.com/opencloud-eu/reva/v2/pkg/upload" "github.com/opencloud-eu/reva/v2/pkg/utils" ) @@ -80,7 +81,7 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { +func (m *manager) Handler(coord upload.Coordinator, driver storage.FS) (http.Handler, error) { h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var spaceID string spaceID, r.URL.Path = router.ShiftPath(r.URL.Path) @@ -97,7 +98,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.DownloadsActive.Sub(1) }() } - download.GetOrHeadFile(w, r, fs, spaceID) + download.GetOrHeadFile(w, r, driver, spaceID) case "PUT": metrics.UploadsActive.Add(1) defer func() { @@ -117,7 +118,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { Path: fn, } var info *provider.ResourceInfo - info, err = fs.Upload(ctx, storage.UploadRequest{ + info, err = coord.Upload(ctx, upload.Request{ Ref: ref, Body: r.Body, Length: r.ContentLength, diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go index ae478a30ec..59aa291f85 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/tus/tus.go @@ -35,13 +35,13 @@ import ( "github.com/opencloud-eu/reva/v2/internal/http/services/owncloud/ocdav/net" "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx" "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/manager/registry" "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storagespace" + "github.com/opencloud-eu/reva/v2/pkg/upload" ) func init() { @@ -89,20 +89,19 @@ func New(m map[string]interface{}, publisher events.Publisher, log *zerolog.Logg }, nil } -func (m *manager) Handler(fs storage.FS) (http.Handler, error) { - composable, ok := fs.(storage.ComposableFS) - if !ok { - return nil, errtypes.NotSupported("file system does not support the tus protocol") - } +func (m *manager) Handler(coord upload.Coordinator, fs storage.FS) (http.Handler, error) { + // such a driver creates a new file under the id minted at initiate, so that id is + // already the file's + nc, ok := fs.(storage.NodeCreator) + createsNode := ok && nc.PrepareCreatesNode() // A storage backend for tusd may consist of multiple different parts which // handle upload creation, locking, termination and so on. The composer is a - // place where all those separated pieces are joined together. In this example - // we only use the file store but you may plug in multiple. + // place where all those separated pieces are joined together. composer := tusd.NewStoreComposer() - // let the composable storage tell tus which extensions it supports - composable.UseIn(composer) + // the coordinator serves the tus protocol on behalf of every driver + coord.UseIn(composer) config := tusd.Config{ StoreComposer: composer, @@ -110,7 +109,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { Logger: slog.New(tusdLogger{log: m.log}), // Add the finalized resource's etag and permissions to the last chunk's response // (see preFinishResponseCallback). https://github.com/opencloud-eu/opencloud/issues/2409 - PreFinishResponseCallback: m.preFinishResponseCallback(fs), + PreFinishResponseCallback: m.preFinishResponseCallback(coord), } if m.conf.CorsEnabled { @@ -135,33 +134,29 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { return nil, err } - if usl, ok := fs.(storage.UploadSessionLister); ok { - // We can currently only send updates if the fs is decomposedfs as we read very specific keys from the storage map of the tus info - go func() { - for { - ev := <-handler.CompleteUploads - // We should be able to get the upload progress with fs.GetUploadProgress, but currently tus will erase the info files - // so we create a Progress instance here that is used to read the correct properties - ups, err := usl.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) - if err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") - } else { - if len(ups) < 1 { - appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") - continue - } - up := ups[0] - executant := up.Executant() - ref := up.Reference() - if m.publisher != nil { - if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { - appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") - } - } + go func() { + for { + ev := <-handler.CompleteUploads + // tus erases its info files, so read the session back for the event's properties + ups, err := coord.ListUploadSessions(context.Background(), storage.UploadSessionFilter{ID: &ev.Upload.ID}) + if err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Str("session", ev.Upload.ID).Msg("failed to list upload session") + continue + } + if len(ups) < 1 { + appctx.GetLogger(context.Background()).Error().Str("session", ev.Upload.ID).Msg("upload session not found") + continue + } + up := ups[0] + executant := up.Executant() + ref := up.Reference() + if m.publisher != nil { + if err := datatx.EmitFileUploadedEvent(up.SpaceOwner(), &executant, &ref, m.publisher); err != nil { + appctx.GetLogger(context.Background()).Error().Err(err).Msg("failed to publish FileUploaded event") } } - }() - } + } + }() h := handler.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { sublog := m.log.With().Str("uploadid", r.URL.Path).Logger() @@ -179,7 +174,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, createsNode, w, r) handler.PostFile(w, r) case "HEAD": handler.HeadFile(w, r) @@ -189,7 +184,7 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { metrics.UploadsActive.Sub(1) }() // set etag, mtime and file id - setHeaders(fs, w, r) + setHeaders(coord, createsNode, w, r) handler.PatchFile(w, r) case "DELETE": handler.DelFile(w, r) @@ -214,16 +209,16 @@ func (m *manager) Handler(fs storage.FS) (http.Handler, error) { // the final response is written, it looks up the finalized resource and attaches its etag and // WebDAV permissions to the response headers, so clients do not need a follow-up PROPFIND after // the last chunk. It mirrors the etag and permissions the ocdav gateway already produces for the -// creation-with-upload path. The storage driver resolves the resource as the upload's executant -// (the data path's transfer token carries no user identity), so this layer does not reconstruct -// the user. It is best effort: it logs any lookup failure and still completes the upload (the +// creation-with-upload path. The upload coordinator resolves the resource as the upload's +// executant (the data path's transfer token carries no user identity), so this layer does not +// reconstruct the user. It is best effort: it logs any lookup failure and still completes the upload (the // client falls back to a PROPFIND). https://github.com/opencloud-eu/opencloud/issues/2409 -func (m *manager) preFinishResponseCallback(fs storage.FS) func(tusd.HookEvent) (tusd.HTTPResponse, error) { - resolver, ok := fs.(storage.UploadSessionResolver) +func (m *manager) preFinishResponseCallback(r any) func(tusd.HookEvent) (tusd.HTTPResponse, error) { + resolver, ok := r.(storage.UploadSessionResolver) return func(hook tusd.HookEvent) (tusd.HTTPResponse, error) { resp := tusd.HTTPResponse{Header: tusd.HTTPHeader{}} if !ok { - // the storage driver cannot resolve the upload; the client falls back to a PROPFIND + // nothing resolves the upload; the client falls back to a PROPFIND return resp, nil } @@ -246,15 +241,10 @@ func (m *manager) preFinishResponseCallback(fs storage.FS) func(tusd.HookEvent) } } -func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { +func setHeaders(coord upload.Coordinator, createsNode bool, w http.ResponseWriter, r *http.Request) { ctx := r.Context() id := path.Base(r.URL.Path) - datastore, ok := fs.(tusd.DataStore) - if !ok { - appctx.GetLogger(ctx).Error().Interface("fs", fs).Msg("storage is not a tus datastore") - return - } - upload, err := datastore.GetUpload(ctx, id) + upload, err := coord.GetUpload(ctx, id) if err != nil { appctx.GetLogger(ctx).Error().Err(err).Msg("could not get upload from storage") return @@ -268,12 +258,23 @@ func setHeaders(fs storage.FS, w http.ResponseWriter, r *http.Request) { if expires != "" { w.Header().Set(net.HeaderTusUploadExpires, expires) } - resourceid := &provider.ResourceId{ + if id := fileID(info, createsNode); id != "" { + w.Header().Set(net.HeaderOCFileID, id) + } +} + +// fileID returns the id of the file an upload writes, or "" while it is not known yet: +// a new file's id is only valid once the upload commits, unless the driver creates the +// file under the id minted at initiate. +func fileID(info tusd.FileInfo, createsNode bool) string { + if info.Storage["NodeExists"] != "true" && !createsNode { + return "" + } + return storagespace.FormatResourceID(&provider.ResourceId{ StorageId: info.MetaData["providerID"], SpaceId: info.Storage["SpaceRoot"], OpaqueId: info.Storage["NodeId"], - } - w.Header().Set(net.HeaderOCFileID, storagespace.FormatResourceID(resourceid)) + }) } // tusdLogger is a logger implementation (slog) for tusd that uses zerolog. diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics/metrics.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics/metrics.go index 80ca36c2c1..f1875483ca 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics/metrics.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics/metrics.go @@ -1,4 +1,4 @@ -// Package metrics provides prometheus metrics for the data managers.. +// Package metrics provides prometheus metrics for the data managers. package metrics import ( @@ -25,36 +25,43 @@ var ( // UploadSessionsInitiated is the number of upload sessions that have been initiated UploadSessionsInitiated = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_initiated", - Help: "Number of uploads sessions that were initiated", + Help: "Number of upload sessions that were initiated", }) - // UploadSessionsBytesReceived is the number of upload sessions that have received all bytes + // UploadSessionsBytesReceived counts upload sessions whose data has been fully received and is + // safely stored, before postprocessing decides their outcome; it counts sessions, not bytes, + // despite its name (kept for compatibility, see https://github.com/opencloud-eu/reva/issues/930) UploadSessionsBytesReceived = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_bytes_received", - Help: "Number of uploads sessions that have received all bytes", + Help: "Number of upload sessions whose data has been fully received, before postprocessing (counts sessions, not bytes)", }) - // UploadSessionsFinalized is the number of upload sessions that have received all bytes + // UploadSessionsFinalized is the number of upload sessions that have successfully completed UploadSessionsFinalized = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_finalized", - Help: "Number of uploads sessions that have successfully completed", + Help: "Number of upload sessions that have successfully completed", }) // UploadSessionsAborted is the number of upload sessions that have been aborted UploadSessionsAborted = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_aborted", - Help: "Number of uploads sessions that have aborted by postprocessing", + Help: "Number of upload sessions that have been aborted by postprocessing", + }) + // UploadSessionsCommitFailed is the number of upload sessions whose blob commit failed after retries and were reverted + UploadSessionsCommitFailed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "reva_upload_sessions_commit_failed", + Help: "Number of upload sessions whose blob commit failed after retries and were reverted", }) // UploadSessionsDeleted is the number of upload sessions that have been deleted UploadSessionsDeleted = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_deleted", - Help: "Number of uploads sessions that have been deleted by postprocessing", + Help: "Number of upload sessions that have been deleted by postprocessing", }) // UploadSessionsRestarted is the number of upload sessions that have been restarted UploadSessionsRestarted = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_restarted", - Help: "Number of uploads sessions that have been restarted by postprocessing", + Help: "Number of upload sessions that have been restarted by postprocessing", }) // UploadSessionsScanned is the number of upload sessions that have been scanned by antivirus UploadSessionsScanned = promauto.NewCounter(prometheus.CounterOpts{ Name: "reva_upload_sessions_scanned", - Help: "Number of uploads sessions that have been scanned by antivirus", + Help: "Number of upload sessions that have been scanned by antivirus", }) ) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/providercache/providercache.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/providercache/providercache.go index 7ed0f40bf3..a8149d983a 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/providercache/providercache.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/providercache/providercache.go @@ -33,7 +33,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go index b8e536b082..be39b7c990 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/receivedsharecache/receivedsharecache.go @@ -32,7 +32,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata" "github.com/opencloud-eu/reva/v2/pkg/utils" "go.opentelemetry.io/otel/attribute" diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/sharecache/sharecache.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/sharecache/sharecache.go index c5a812dc16..ae85b17d22 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/sharecache/sharecache.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/sharecache/sharecache.go @@ -35,7 +35,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/share/manager/jsoncs3/shareid" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/mtimesyncedcache" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/metadata" "github.com/opencloud-eu/reva/v2/pkg/utils" ) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/hello/unimplemented.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/hello/unimplemented.go index 631fb0d069..0f244040d3 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/hello/unimplemented.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/hello/unimplemented.go @@ -75,16 +75,6 @@ func (fs *hellofs) Move(ctx context.Context, oldRef, newRef *provider.Reference) return nil, errtypes.NotSupported("unimplemented") } -// Upload creates or updates a resource of type file with a new revision -func (fs *hellofs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - return nil, errtypes.NotSupported("hellofs: upload not supported") -} - -// InitiateUpload returns a list of protocols with urls that can be used to append bytes to a new upload session -func (fs *hellofs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - return nil, errtypes.NotSupported("hellofs: initiate upload not supported") -} - // MarkProcessing toggles a processing flag on the resource. func (fs *hellofs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { return errtypes.NotSupported("hellofs: mark processing not supported") @@ -95,6 +85,14 @@ func (fs *hellofs) CommitUpload(_ context.Context, _ *provider.Reference, _ stri return errtypes.NotSupported("hellofs: commit upload not supported") } +func (fs *hellofs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil +} + +func (fs *hellofs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { + return nil +} + // grants // DenyGrant marks a resource as denied for a recipient diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/loader/loader.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/loader/loader.go index c3e05a4f4c..e73d0a4ac3 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/loader/loader.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/loader/loader.go @@ -31,9 +31,7 @@ import ( _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/local" _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/localhome" _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud" - _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis" _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql" _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix" - _ "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng" // Add your own here ) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud.go index 6b152c2cda..52b151cdea 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud.go @@ -21,7 +21,6 @@ package nextcloud import ( "context" "encoding/json" - "fmt" "io" "net/http" "net/url" @@ -271,7 +270,20 @@ func (nc *StorageDriver) CreateDir(ctx context.Context, ref *provider.Reference) // TouchFile as defined in the storage.FS interface func (nc *StorageDriver) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error { - return fmt.Errorf("unimplemented: TouchFile") + type paramsObj struct { + Ref *provider.Reference `json:"ref"` + MarkProcessing bool `json:"markprocessing"` + MTime string `json:"mtime"` + } + bodyStr, err := json.Marshal(¶msObj{Ref: ref, MarkProcessing: markprocessing, MTime: mtime}) + if err != nil { + return err + } + log := appctx.GetLogger(ctx) + log.Info().Msgf("TouchFile %s", bodyStr) + + _, _, err = nc.do(ctx, Action{"TouchFile", string(bodyStr)}) + return err } // Delete as defined in the storage.FS interface @@ -372,58 +384,28 @@ func (nc *StorageDriver) ListFolder(ctx context.Context, ref *provider.Reference return pointers, err } -// InitiateUpload as defined in the storage.FS interface -func (nc *StorageDriver) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - type paramsObj struct { - Ref *provider.Reference `json:"ref"` - UploadLength int64 `json:"uploadLength"` - Metadata map[string]string `json:"metadata"` - } - bodyObj := ¶msObj{ - Ref: ref, - UploadLength: uploadLength, - Metadata: metadata, - } - bodyStr, _ := json.Marshal(bodyObj) - log := appctx.GetLogger(ctx) - log.Info().Msgf("InitiateUpload %s", bodyStr) - - _, respBody, err := nc.do(ctx, Action{"InitiateUpload", string(bodyStr)}) - if err != nil { - return nil, err - } - respMap := make(map[string]string) - err = json.Unmarshal(respBody, &respMap) - if err != nil { - return nil, err - } - return respMap, err -} - -// MarkProcessing as defined in the storage.FS interface -func (nc *StorageDriver) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { - return errtypes.NotSupported("nextcloud: mark processing not supported") +// MarkProcessing as defined in the storage.FS interface. +// No sciencemesh endpoint toggles the flag alone. A no-op, not NotSupported: +// the coordinator treats a failed mark as fatal. +func (nc *StorageDriver) MarkProcessing(_ context.Context, _ *provider.Reference, _ bool, _ string) error { + return nil } // CommitUpload as defined in the storage.FS interface -func (nc *StorageDriver) CommitUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.UploadSource) error { - return errtypes.NotSupported("nextcloud: commit upload not supported") +func (nc *StorageDriver) CommitUpload(ctx context.Context, ref *provider.Reference, _ string, source storage.UploadSource) error { + if source.Body == nil { + // A nil body would PUT an empty request and truncate the file. + return errtypes.BadRequest("nextcloud: source body is nil") + } + return nc.doUpload(ctx, ref.GetPath(), source.Body) } -// Upload as defined in the storage.FS interface -func (nc *StorageDriver) Upload(ctx context.Context, req storage.UploadRequest, _ storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - err := nc.doUpload(ctx, req.Ref.Path, req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } +func (nc *StorageDriver) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil +} - // return id, etag and mtime - ri, err := nc.GetMD(ctx, req.Ref, []string{}, []string{"id", "etag", "mtime"}) - if err != nil { - return &provider.ResourceInfo{}, err - } - - return ri, nil +func (nc *StorageDriver) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { + return nil } // Download as defined in the storage.FS interface diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go index 5bfdfe8b5c..bd3207cd34 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud/nextcloud_server_mock.go @@ -74,6 +74,11 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/EmptyRecycle `: {200, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetQuota `: {200, `{"totalBytes":456,"usedBytes":123}`, serverStateEmpty}, + + // the parent of an upload target, stated for its permissions and its id + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":[]} EMPTY`: {200, `{"opaque":{},"type":2,"id":{"opaque_id":"fileid-/"},"checksum":{},"etag":"deadbeef","mime_type":"httpd/unix-directory","mtime":{"seconds":1234567890},"path":"/","permission_set":{"initiate_file_upload":true,"stat":true,"list_container":true},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":null} EMPTY`: {404, ``, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/"},"mdKeys":null} HOME`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateHome}, @@ -109,9 +114,21 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/versionedFile"},"mdKeys":null} EMPTY`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":2,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/versionedFile"},"mdKeys":null} FILE-RESTORED`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":1,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}`, serverStateFileRestored}, + // The coordinator resolves the upload target itself, so it stats the file with the + // full reference before every upload. Reporting the file as existing keeps it on the + // overwrite path, which needs neither a parent stat nor a TouchFile (whose mtime is + // wall-clock derived and could never match this table's exact-body keys). + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"mdKeys":[]}`: {200, `{"opaque":{},"type":1,"id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","opaque_id":"fileid-/versionedFile"},"parent_id":{"opaque_id":"fileid-/"},"name":"versionedFile","checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{"initiate_file_upload":true,"stat":true,"list_container":true,"list_file_versions":true,"restore_file_version":true},"size":1,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetPathByID {"storage_id":"00000000-0000-0000-0000-000000000000","opaque_id":"fileid-/some/path"} EMPTY`: {200, "/subdir", serverStateEmpty}, - `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":null}`: {404, ``, serverStateEmpty}, + // the coordinator resolves the upload target itself: the file does not exist yet, + // so it stats the parent for the permissions and the parent id + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"path":"/file"},"mdKeys":[]}`: {404, ``, serverStateEmpty}, + // a zero-length upload finishes at once, so the node is created and read back here + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/TouchFile {"ref":{"resource_id":{"opaque_id":"fileid-/"},"path":"file"},"markprocessing":false,"mtime":"1234567890"}`: {200, ``, serverStateEmpty}, + `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/GetMD {"ref":{"resource_id":{"opaque_id":"fileid-/"},"path":"file"},"mdKeys":null}`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/file"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/file","permission_set":{},"size":0,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{}}}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"path":"/file"},"uploadLength":0,"metadata":{"providerID":""}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":1,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, `POST /apps/sciencemesh/~f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"path":"/versionedFile"},"uploadLength":2,"metadata":{}}`: {200, `{"simple": "yes","tus": "yes"}`, serverStateEmpty}, @@ -160,6 +177,7 @@ var responses = map[string]Response{ `POST /apps/sciencemesh/~tester/api/storage/ListFolder {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some"},"mdKeys":["val1","val2","val3"]}`: {200, `[{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/some/path","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}]`, serverStateEmpty}, // `POST /apps/sciencemesh/~tester/api/storage/ListFolder {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some"},"mdKeys":["val1","val2","val3"]}`: {200, `[{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/path","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"da":"ta","some":"arbi","trary":"meta"}}}]`, serverStateEmpty}, `POST /apps/sciencemesh/~tester/api/storage/InitiateUpload {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"/some/path"},"uploadLength":12345,"metadata":{"key1":"val1","key2":"val2","key3":"val3"}}`: {200, `{ "not":"sure", "what": "should be", "returned": "here" }`, serverStateEmpty}, + `POST /apps/sciencemesh/~tester/api/storage/TouchFile {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"some/file/path.txt"},"markprocessing":false,"mtime":"1234567890"}`: {200, ``, serverStateEmpty}, `PUT /apps/sciencemesh/~tester/api/storage/Upload/some/file/path.txt shiny!`: {200, ``, serverStateEmpty}, `POST /apps/sciencemesh/~tester/api/storage/GetMD {"ref":{"resource_id":{"storage_id":"storage-id","opaque_id":"opaque-id"},"path":"some/file/path.txt"},"mdKeys":[]}`: {200, `{"opaque":{},"type":1,"id":{"opaque_id":"fileid-/some/path"},"checksum":{},"etag":"deadbeef","mime_type":"text/plain","mtime":{"seconds":1234567890},"path":"/versionedFile","permission_set":{},"size":12345,"canonical_metadata":{},"arbitrary_metadata":{"metadata":{"key1":"val1","key2":"val2","key3":"val3"}}}`, serverStateEmpty}, `GET /apps/sciencemesh/~tester/api/storage/Download/some/file/path.txt `: {200, `the contents of the file`, serverStateEmpty}, diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go deleted file mode 100644 index 2d21a94cb9..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore/blobstore.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore - -import ( - "bufio" - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" -) - -// ErrBlobIDEmpty is returned when the BlobID is empty -var ErrBlobIDEmpty = fmt.Errorf("blobstore: BlobID is empty") - -// Blobstore provides an interface to an filesystem based blobstore -type Blobstore struct { - root string -} - -// New returns a new Blobstore -func New(root string) (*Blobstore, error) { - err := os.MkdirAll(root, 0700) - if err != nil { - return nil, err - } - - return &Blobstore{ - root: root, - }, nil -} - -// Upload stores some data in the blobstore under the given key -func (bs *Blobstore) Upload(node *node.Node, source string) (err error) { - if node.BlobID == "" { - return ErrBlobIDEmpty - } - - dest := bs.Path(node) - - // ensure parent path exists - if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob") - } - - if err := os.Rename(source, dest); err == nil { - return nil - } - - // Rename failed, file needs to be copied. - file, err := os.Open(source) - if err != nil { - return errors.Wrap(err, "Decomposedfs: oCIS blobstore: Can not open source file to upload") - } - defer file.Close() - - f, err := os.OpenFile(dest, os.O_CREATE|os.O_WRONLY, 0700) - if err != nil { - return errors.Wrapf(err, "could not open blob '%s' for writing", dest) - } - defer func() { - if cerr := f.Close(); cerr != nil && err == nil { - err = errors.Wrapf(cerr, "could not close blob '%s'", dest) - } - }() - - w := bufio.NewWriter(f) - if _, err = w.ReadFrom(file); err != nil { - return errors.Wrapf(err, "could not write blob '%s'", dest) - } - - if err = w.Flush(); err != nil { - return errors.Wrapf(err, "could not flush blob '%s'", dest) - } - - // Ensure data is synced to disk before returning - if err = f.Sync(); err != nil { - return errors.Wrapf(err, "could not sync blob '%s' to disk", dest) - } - - return nil -} - -// UploadFromReader stores data from a reader in the blobstore under the given key. -// It buffers to a temp file on the same filesystem as the destination so that the -// final placement is an atomic os.Rename. -func (bs *Blobstore) UploadFromReader(node *node.Node, r io.Reader, size int64) error { - if node.BlobID == "" { - return ErrBlobIDEmpty - } - - dest := bs.Path(node) - - if err := os.MkdirAll(filepath.Dir(dest), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating parent folders for blob") - } - - tmp, err := os.CreateTemp(filepath.Dir(dest), "reva-blobstore-upload-*") - if err != nil { - return errors.Wrap(err, "Decomposedfs: oCIS blobstore: error creating temp file for upload") - } - tmpName := tmp.Name() - defer func() { - tmp.Close() - os.Remove(tmpName) - }() - - w := bufio.NewWriter(tmp) - if _, err = io.Copy(w, r); err != nil { - return errors.Wrapf(err, "could not write blob to temp file '%s'", tmpName) - } - if err = w.Flush(); err != nil { - return errors.Wrapf(err, "could not flush blob temp file '%s'", tmpName) - } - if err = tmp.Sync(); err != nil { - return errors.Wrapf(err, "could not sync blob temp file '%s'", tmpName) - } - tmp.Close() - - if err = os.Rename(tmpName, dest); err != nil { - return errors.Wrapf(err, "could not rename blob temp file to '%s'", dest) - } - return nil -} - -// Download retrieves a blob from the blobstore for reading -func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { - if node.BlobID == "" { - return nil, ErrBlobIDEmpty - } - - dest := bs.Path(node) - file, err := os.Open(dest) - if err != nil { - return nil, errors.Wrapf(err, "could not read blob '%s'", dest) - } - return file, nil -} - -// Delete deletes a blob from the blobstore -func (bs *Blobstore) Delete(node *node.Node) error { - if node.BlobID == "" { - return ErrBlobIDEmpty - } - dest := bs.Path(node) - if err := utils.RemoveItem(dest); err != nil { - return errors.Wrapf(err, "could not delete blob '%s'", dest) - } - return nil -} - -// List lists all blobs in the Blobstore -func (bs *Blobstore) List() ([]*node.Node, error) { - dirs, err := filepath.Glob(filepath.Join(bs.root, "spaces", "*", "*", "blobs", "*", "*", "*", "*", "*")) - if err != nil { - return nil, err - } - blobids := make([]*node.Node, 0, len(dirs)) - for _, d := range dirs { - _, s, _ := strings.Cut(d, "spaces") - spaceraw, blobraw, _ := strings.Cut(s, "blobs") - blobids = append(blobids, &node.Node{ - SpaceID: strings.ReplaceAll(spaceraw, "/", ""), - BlobID: strings.ReplaceAll(blobraw, "/", ""), - }) - } - return blobids, nil -} - -func (bs *Blobstore) Path(node *node.Node) string { - return filepath.Join( - bs.root, - filepath.Clean(filepath.Join( - "/", "spaces", lookup.Pathify(node.SpaceID, 1, 2), "blobs", lookup.Pathify(node.BlobID, 4, 2)), - ), - ) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/ocis.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/ocis.go deleted file mode 100644 index 3b6687c1a7..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/ocis.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package ocis - -import ( - "path" - - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore" - "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/rs/zerolog" -) - -func init() { - registry.Register("ocis", New) -} - -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. -func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) { - o, err := options.New(m) - if err != nil { - return nil, err - } - - bs, err := blobstore.New(path.Join(o.Root)) - if err != nil { - return nil, err - } - - return decomposedfs.NewDefault(m, bs, stream, log) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql/upload.go index 46b1cb932a..08c790fb94 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql/upload.go @@ -20,155 +20,12 @@ package owncloudsql import ( "context" - "encoding/json" - "fmt" - "io" - iofs "io/fs" - "os" - "path/filepath" - "strconv" - "time" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/conversions" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/logger" - "github.com/opencloud-eu/reva/v2/pkg/mime" "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog/log" - tusd "github.com/tus/tusd/v2/pkg/handler" ) -var defaultFilePerm = os.FileMode(0664) - -func (fs *owncloudsqlfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - upload, err := fs.GetUpload(ctx, req.Ref.GetPath()) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error retrieving upload") - } - - uploadInfo := upload.(*fileUpload) - - p := uploadInfo.info.Storage["InternalDestination"] - if chunking.IsChunked(p) { - var assembledFile string - p, assembledFile, err = fs.chunkHandler.WriteChunk(p, req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } - if p == "" { - if err = uploadInfo.Terminate(ctx); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error removing auxiliary files") - } - return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) - } - uploadInfo.info.Storage["InternalDestination"] = p - fd, err := os.Open(assembledFile) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error opening assembled file") - } - defer fd.Close() - defer os.RemoveAll(assembledFile) - req.Body = fd - } - - if _, err := uploadInfo.WriteChunk(ctx, 0, req.Body); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "owncloudsql: error writing to binary file") - } - - if err := uploadInfo.FinishUpload(ctx); err != nil { - return &provider.ResourceInfo{}, err - } - - if uff != nil { - info := uploadInfo.info - uploadRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: info.MetaData["providerID"], - SpaceId: info.Storage["SpaceRoot"], - OpaqueId: info.Storage["SpaceRoot"], - }, - Path: utils.MakeRelativePath(filepath.Join(info.MetaData["dir"], info.MetaData["filename"])), - } - owner, ok := ctxpkg.ContextGetUser(uploadInfo.ctx) - if !ok { - return &provider.ResourceInfo{}, errtypes.PreconditionFailed("error getting user from uploadinfo context") - } - // spaces support in localfs needs to be revisited: - // * info.Storage["SpaceRoot"] is never set - // * there is no space owner or manager that could be passed to the UploadFinishedFunc - uff(owner.Id, owner.Id, uploadRef) - } - - ri := &provider.ResourceInfo{ - // fill with at least fileid, mtime and etag - Id: &provider.ResourceId{ - StorageId: uploadInfo.info.MetaData["providerID"], - SpaceId: uploadInfo.info.Storage["StorageId"], - OpaqueId: uploadInfo.info.Storage["fileid"], - }, - Etag: uploadInfo.info.MetaData["etag"], - } - - if mtime, err := utils.MTimeToTS(uploadInfo.info.MetaData["mtime"]); err == nil { - ri.Mtime = &mtime - } - - return ri, nil -} - -// InitiateUpload returns upload ids corresponding to different protocols it supports -// TODO read optional content for small files in this request -func (fs *owncloudsqlfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - ip, err := fs.resolve(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "owncloudsql: error resolving reference") - } - - // permissions are checked in NewUpload below - - p := fs.toStoragePath(ctx, ip) - - info := tusd.FileInfo{ - MetaData: tusd.MetaData{ - "filename": filepath.Base(p), - "dir": filepath.Dir(p), - "mtime": strconv.FormatInt(time.Now().Unix(), 10), - }, - Size: uploadLength, - } - - if metadata != nil { - info.MetaData["providerID"] = metadata["providerID"] - if metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] - } - if _, ok := metadata["sizedeferred"]; ok { - info.SizeIsDeferred = true - } - } - - upload, err := fs.NewUpload(ctx, info) - if err != nil { - return nil, err - } - - info, _ = upload.GetInfo(ctx) - - return map[string]string{ - "simple": info.ID, - "tus": info.ID, - }, nil -} - func (fs *owncloudsqlfs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { return errtypes.NotSupported("op not supported") } @@ -177,383 +34,10 @@ func (fs *owncloudsqlfs) CommitUpload(_ context.Context, _ *provider.Reference, return errtypes.NotSupported("op not supported") } -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *owncloudsqlfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - composer.UseConcater(fs) - composer.UseLengthDeferrer(fs) +func (fs *owncloudsqlfs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil } -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload - -func (fs *owncloudsqlfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { - - log := appctx.GetLogger(ctx) - log.Debug().Interface("info", info).Msg("owncloudsql: NewUpload") - - if info.MetaData["filename"] == "" { - return nil, errors.New("owncloudsql: missing filename in metadata") - } - info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) - - dir := info.MetaData["dir"] - if dir == "" { - return nil, errors.New("owncloudsql: missing dir in metadata") - } - info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) - - ip := fs.toInternalPath(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) - - // check permissions - var perm *provider.ResourcePermissions - var perr error - var fsInfo iofs.FileInfo - // if destination exists - if fsInfo, err = os.Stat(ip); err == nil { - // check permissions of file to be overwritten - perm, perr = fs.readPermissions(ctx, ip) - } else { - // check permissions of parent folder - perm, perr = fs.readPermissions(ctx, filepath.Dir(ip)) - } - if perr == nil { - if !perm.InitiateFileUpload { - return nil, errtypes.PermissionDenied("") - } - } else { - if os.IsNotExist(err) { - return nil, errtypes.NotFound(fs.toStoragePath(ctx, filepath.Dir(ip))) - } - return nil, errors.Wrap(err, "owncloudsql: error reading permissions") - } - - // if we are trying to overwriting a folder with a file - if fsInfo != nil && fsInfo.IsDir() { - return nil, errtypes.PreconditionFailed("resource is not a file") - } - - log.Debug().Interface("info", info).Msg("owncloudsql: resolved filename") - - info.ID = uuid.New().String() - - binPath, err := fs.getUploadPath(ctx, info.ID) - if err != nil { - return nil, errors.Wrap(err, "owncloudsql: error resolving upload path") - } - usr := ctxpkg.ContextMustGetUser(ctx) - storageID, err := fs.getStorage(ctx, ip) - if err != nil { - return nil, err - } - info.Storage = map[string]string{ - "Type": "OwnCloudStore", - "BinPath": binPath, - "InternalDestination": ip, - "Permissions": strconv.Itoa((int)(conversions.RoleFromResourcePermissions(perm, false).OCSPermissions())), - - "Idp": usr.Id.Idp, - "UserId": usr.Id.OpaqueId, - "UserName": usr.Username, - - "LogLevel": log.GetLevel().String(), - - "StorageId": strconv.Itoa(storageID), - } - // Create binary file in the upload folder with no content - log.Debug().Interface("info", info).Msg("owncloudsql: built storage info") - file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return nil, err - } - defer file.Close() - - u := &fileUpload{ - info: info, - binPath: binPath, - infoPath: filepath.Join(fs.c.UploadInfoDir, info.ID+".info"), - fs: fs, - ctx: ctx, - } - - // writeInfo creates the file by itself if necessary - err = u.writeInfo() - if err != nil { - return nil, err - } - - return u, nil -} - -func (fs *owncloudsqlfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - err := errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx") - return "", err - } - layout := templates.WithUser(u, fs.c.UserLayout) - return filepath.Join(fs.c.DataDirectory, layout, "uploads", uploadID), nil -} - -// GetUpload returns the Upload for the given upload id -func (fs *owncloudsqlfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - infoPath := filepath.Join(fs.c.UploadInfoDir, id+".info") - - info := tusd.FileInfo{} - data, err := os.ReadFile(infoPath) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - if err := json.Unmarshal(data, &info); err != nil { - return nil, err - } - - stat, err := os.Stat(info.Storage["BinPath"]) - if err != nil { - return nil, err - } - - info.Offset = stat.Size() - - u := &userpb.User{ - Id: &userpb.UserId{ - Idp: info.Storage["Idp"], - OpaqueId: info.Storage["UserId"], - }, - Username: info.Storage["UserName"], - } - - ctx = ctxpkg.ContextSetUser(ctx, u) - // TODO configure the logger the same way ... store and add traceid in file info - - opts := []logger.Option{ - logger.WithLevel(info.Storage["LogLevel"]), - logger.WithWriter(os.Stderr, logger.ConsoleMode), - } - l := logger.New(opts...) - - sub := l.With().Int("pid", os.Getpid()).Logger() - - ctx = appctx.WithLogger(ctx, &sub) - - return &fileUpload{ - info: info, - binPath: info.Storage["BinPath"], - infoPath: infoPath, - fs: fs, - ctx: ctx, - }, nil -} - -type fileUpload struct { - // info stores the current information about the upload - info tusd.FileInfo - // infoPath is the path to the .info file - infoPath string - // binPath is the path to the binary file (which has no extension) - binPath string - // only fs knows how to handle metadata and versions - fs *owncloudsqlfs - // a context with a user - // TODO add logger as well? - ctx context.Context -} - -// GetInfo returns the FileInfo -func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { - return upload.info, nil -} - -// WriteChunk writes the stream from the reader to the given offset of the upload -func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return 0, err - } - defer file.Close() - - n, err := io.Copy(file, src) - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for OwnCloudStore it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil { - if err != io.ErrUnexpectedEOF { - return n, err - } - } - - upload.info.Offset += n - err = upload.writeInfo() // TODO info is written here ... we need to truncate in DiscardChunk - - return n, err -} - -// GetReader returns an io.Reader for the upload -func (upload *fileUpload) GetReader(ctx context.Context) (io.ReadCloser, error) { - return os.Open(upload.binPath) -} - -// writeInfo updates the entire information. Everything will be overwritten. -func (upload *fileUpload) writeInfo() error { - log.Debug().Str("path", upload.infoPath).Msg("Writing info file") - data, err := json.Marshal(upload.info) - if err != nil { - return err - } - return os.WriteFile(upload.infoPath, data, defaultFilePerm) -} - -// FinishUpload finishes an upload and moves the file to the internal destination -func (upload *fileUpload) FinishUpload(ctx context.Context) error { - - ip := upload.info.Storage["InternalDestination"] - - // if destination exists - // TODO check etag with If-Match header - if _, err := os.Stat(ip); err == nil { - // create revision - if err := upload.fs.archiveRevision(upload.ctx, upload.fs.getVersionsPath(upload.ctx, ip), ip); err != nil { - return err - } - } - - sha1h, md5h, adler32h, err := upload.fs.HashFile(upload.binPath) - if err != nil { - log.Err(err).Msg("owncloudsql: could not open file for checksumming") - } - - err = os.Rename(upload.binPath, ip) - if err != nil { - log.Err(err).Interface("info", upload.info). - Str("binPath", upload.binPath). - Str("ipath", ip). - Msg("owncloudsql: could not rename") - return err - } - - var fi os.FileInfo - fi, err = os.Stat(ip) - if err != nil { - return err - } - - perms, err := strconv.Atoi(upload.info.Storage["Permissions"]) - if err != nil { - return err - } - - if upload.info.MetaData["mtime"] == "" { - upload.info.MetaData["mtime"] = fmt.Sprintf("%d", fi.ModTime().Unix()) - } - if upload.info.MetaData["etag"] == "" { - upload.info.MetaData["etag"] = calcEtag(upload.ctx, fi) - } - - data := map[string]interface{}{ - "path": upload.fs.toDatabasePath(ip), - "checksum": fmt.Sprintf("SHA1:%032x MD5:%032x ADLER32:%032x", sha1h, md5h, adler32h), - "etag": upload.info.MetaData["etag"], - "size": upload.info.Size, - "mimetype": mime.Detect(false, ip), - "permissions": perms, - "mtime": upload.info.MetaData["mtime"], - "storage_mtime": upload.info.MetaData["mtime"], - } - var fileid int - fileid, err = upload.fs.filecache.InsertOrUpdate(ctx, upload.info.Storage["StorageId"], data, false) - if err != nil { - return err - } - upload.info.Storage["fileid"] = fmt.Sprintf("%d", fileid) - - // only delete the upload if it was successfully written to the storage - if err := os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - log.Err(err).Interface("info", upload.info).Msg("owncloudsql: could not delete upload info") - return err - } - } - - return upload.fs.propagate(upload.ctx, ip) -} - -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// - the storage needs to implement AsTerminatableUpload -// - the upload needs to implement Terminate - -// AsTerminatableUpload returns a TerminatableUpload -func (fs *owncloudsqlfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { - return upload.(*fileUpload) -} - -// Terminate terminates the upload -func (upload *fileUpload) Terminate(ctx context.Context) error { - if err := os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - return err - } - } - if err := os.Remove(upload.binPath); err != nil { - if !os.IsNotExist(err) { - return err - } - } +func (fs *owncloudsqlfs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { return nil } - -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// - the storage needs to implement AsLengthDeclarableUpload -// - the upload needs to implement DeclareLength - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -func (fs *owncloudsqlfs) AsLengthDeclarableUpload(upload tusd.Upload) tusd.LengthDeclarableUpload { - return upload.(*fileUpload) -} - -// DeclareLength updates the upload length information -func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error { - upload.info.Size = length - upload.info.SizeIsDeferred = false - return upload.writeInfo() -} - -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// - the storage needs to implement AsConcatableUpload -// - the upload needs to implement ConcatUploads - -// AsConcatableUpload returns a ConcatableUpload -func (fs *owncloudsqlfs) AsConcatableUpload(upload tusd.Upload) tusd.ConcatableUpload { - return upload.(*fileUpload) -} - -// ConcatUploads concatenates multiple uploads -func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []tusd.Upload) (err error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*fileUpload) - - src, err := os.Open(fileUpload.binPath) - if err != nil { - return err - } - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - - return -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go index ebb554b947..7ab03e8b26 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go @@ -67,7 +67,10 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error { return err } - if bs.canUseRenameForUpload { + // a source staged in the tmp dir already is the temp file (see UploadFromReader) + staged := filepath.Clean(source) == tempName + + if !staged && bs.canUseRenameForUpload { err := os.Rename(source, tempName) switch { case err == nil: @@ -80,7 +83,7 @@ func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error { } } - if !bs.canUseRenameForUpload { + if !staged && !bs.canUseRenameForUpload { sourceFile, err := os.Open(source) if err != nil { return fmt.Errorf("failed to open source file '%s': %v", source, err) @@ -193,7 +196,12 @@ func (bs *Blobstore) UploadFromReader(n *node.Node, r io.Reader, size int64) err if n.SpaceRoot == nil { return errors.New("node has no space root") } - f, err := os.CreateTemp(n.SpaceRoot.InternalPath(), ".upload-*") + // stage in the space's tmp dir, which the watcher ignores + tmpDir := filepath.Join(n.SpaceRoot.InternalPath(), TMPDir) + if err := os.MkdirAll(tmpDir, 0700); err != nil { + return err + } + f, err := os.CreateTemp(tmpDir, ".upload-*") if err != nil { return errors.Wrap(err, "could not create temporary upload file") } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go index 3588b93719..73c28e81e6 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/posix.go @@ -28,9 +28,7 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/rs/zerolog" - tusd "github.com/tus/tusd/v2/pkg/handler" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" "github.com/opencloud-eu/reva/v2/pkg/storage" @@ -49,7 +47,6 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/permissions" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware" "github.com/pkg/errors" @@ -179,13 +176,12 @@ func New(o *options.Options, stream events.Stream, cache, historyCache *idcache. } aspects := aspects.Aspects{ - Lookup: lu, - Tree: tp, - Permissions: p, - EventStream: stream, - UserMapper: um, - DisableVersioning: o.DisableVersioning, - Trashbin: trashbin, + Lookup: lu, + Tree: tp, + Permissions: p, + EventStream: stream, + UserMapper: um, + Trashbin: trashbin, } dfs, err := decomposedfs.New(&o.Options, aspects, log) @@ -245,54 +241,14 @@ func (fs *posixFS) WarmupIDCache(root string, assimilate, onlyDirty bool) error return fs.tree.WarmupIDCache(root, assimilate, onlyDirty) } -// ListUploadSessions returns the upload sessions matching the given filter -func (fs *posixFS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { - return fs.FS.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) +// PrepareCreatesNode reports whether the wrapped driver's PrepareUpload creates a +// new file's node. +func (fs *posixFS) PrepareCreatesNode() bool { + nc, ok := fs.FS.(storage.NodeCreator) + return ok && nc.PrepareCreatesNode() } -// ResolveUpload resolves a finished upload to the resource it produced, read as the upload's -// executant. It is best effort: if the wrapped storage does not resolve uploads it returns -// NotSupported rather than panicking, so the tus finalize falls back to no headers. -func (fs *posixFS) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) { - r, ok := fs.FS.(storage.UploadSessionResolver) - if !ok { - return nil, ctx, errtypes.NotSupported("storage does not resolve upload sessions") - } - return r.ResolveUpload(ctx, info) -} - -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *posixFS) UseIn(composer *tusd.StoreComposer) { - fs.FS.(storage.ComposableFS).UseIn(composer) -} - -// NewUpload returns a new tus Upload instance -func (fs *posixFS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { - return fs.FS.(tusd.DataStore).NewUpload(ctx, info) -} - -// NewUpload returns a new tus Upload instance -func (fs *posixFS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { - return fs.FS.(tusd.DataStore).GetUpload(ctx, id) -} - -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (fs *posixFS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.DecomposedFsSession) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (fs *posixFS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.DecomposedFsSession) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (fs *posixFS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.DecomposedFsSession) +// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable. +func (fs *posixFS) IsOrphaned(ctx context.Context, ref *provider.Reference) bool { + return fs.FS.(storage.OrphanChecker).IsOrphaned(ctx, ref) } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go index 072c96a9b7..a2217ada15 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -41,8 +41,10 @@ import ( provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" + "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/idcache" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/ignore" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup" @@ -414,6 +416,9 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool, return err } + if storage.SkipTouchPropagation(ctx) { + return nil + } return t.Propagate(ctx, n, 0) } @@ -788,8 +793,12 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met _, span := tracer.Start(ctx, "InitNewNode") defer span.End() + + // the node may not be cached yet, so its path comes from the parent and the name + nodePath := filepath.Join(n.ParentPath(), n.Name) + // create folder structure (if needed) - if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { + if err := os.MkdirAll(filepath.Dir(nodePath), 0700); err != nil { return nil, err } @@ -800,36 +809,63 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met } // we also need to touch the actual node file here it stores the mtime of the resource - h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) + h, err := os.OpenFile(nodePath, os.O_CREATE|os.O_EXCL, 0600) if err != nil { + // the file is not ours: leave it, and its cache entry, alone + releaseNewNodeLock(ctx, n, unlock) if os.IsExist(err) { - return unlock, errtypes.AlreadyExists(n.InternalPath()) + // the name only: the message reaches the client, the path is the server's + return nil, errtypes.AlreadyExists(n.Name) } - return unlock, err + return nil, err + } + defer func() { _ = h.Close() }() + + // from here the file is ours. A failure removes it and our cache entry before + // unlocking, so nothing can open the node in between. + fail := func(err error) (metadata.UnlockFunc, error) { + if rErr := os.Remove(nodePath); rErr != nil { + t.log.Error().Err(rErr).Str("nodeid", n.ID).Msg("could not remove new node file") + } + if dErr := t.lookup.IDCache.Delete(ctx, n.SpaceID, n.ID); dErr != nil { + t.log.Error().Err(dErr).Str("nodeid", n.ID).Msg("could not remove new node from the id cache") + } + releaseNewNodeLock(ctx, n, unlock) + return nil, err + } + + // cached only now that the name is ours, so a losing upload cannot point it at its id + if err := t.lookup.CacheID(ctx, n.SpaceID, n.ID, nodePath); err != nil { + return fail(err) } // Set known mtime from filesystem to metadata to preven re-assimilation fi, err := h.Stat() if err != nil { - return nil, err + return fail(err) } mtime := fi.ModTime() err = n.SetXattrsWithContext(ctx, map[string][]byte{ prefixes.MTimeAttr: []byte(mtime.UTC().Format(time.RFC3339Nano)), }) if err != nil { - t.log.Error().Err(err).Str("path", n.InternalPath()).Msg("could not set mtime attribute on new node") + t.log.Error().Err(err).Str("path", nodePath).Msg("could not set mtime attribute on new node") } - _ = h.Close() - if _, err := node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize); err != nil { - return unlock, err + return fail(err) } return unlock, nil } +// releaseNewNodeLock releases the lock of a node InitNewNode failed to create. +func releaseNewNodeLock(ctx context.Context, n *node.Node, unlock metadata.UnlockFunc) { + if err := unlock(); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not release new node lock") + } +} + // TODO check if node exists? func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { if t.Ignorer.IsIgnored(filepath.Join(n.ParentPath(), n.Name)) { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go deleted file mode 100644 index 628105a183..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore/blobstore.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package blobstore - -import ( - "context" - "fmt" - "io" - "net/url" - "os" - "path/filepath" - "strings" - - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/pkg/errors" -) - -// Blobstore provides an interface to an s3 compatible blobstore -type Blobstore struct { - client *minio.Client - - defaultPutOptions Options - - bucket string -} - -type Options struct { - DisableContentSha256 bool - DisableMultipart bool - SendContentMd5 bool - ConcurrentStreamParts bool - NumThreads uint - PartSize uint64 -} - -// New returns a new Blobstore -func New(endpoint, region, bucket, accessKey, secretKey string, defaultPutOptions Options) (*Blobstore, error) { - u, err := url.Parse(endpoint) - if err != nil { - return nil, errors.Wrap(err, "failed to parse s3 endpoint") - } - - useSSL := u.Scheme != "http" - client, err := minio.New(u.Host, &minio.Options{ - Region: region, - Creds: credentials.NewStaticV4(accessKey, secretKey, ""), - Secure: useSSL, - }) - if err != nil { - return nil, errors.Wrap(err, "failed to setup s3 client") - } - - return &Blobstore{ - client: client, - bucket: bucket, - defaultPutOptions: defaultPutOptions, - }, nil -} - -// Upload stores some data in the blobstore under the given key -// TODO(OCISDEV-901): remove once FinishUploadDecomposed is replaced by the upload coordinator (OCISDEV-900). -func (bs *Blobstore) Upload(node *node.Node, source string) error { - reader, err := os.Open(source) - if err != nil { - return errors.Wrap(err, "can not open source file to upload") - } - defer reader.Close() - - _, err = bs.client.PutObject(context.Background(), bs.bucket, bs.Path(node), reader, node.Blobsize, minio.PutObjectOptions{ - ContentType: "application/octet-stream", - SendContentMd5: bs.defaultPutOptions.SendContentMd5, - ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts, - NumThreads: bs.defaultPutOptions.NumThreads, - PartSize: bs.defaultPutOptions.PartSize, - DisableMultipart: bs.defaultPutOptions.DisableMultipart, - DisableContentSha256: bs.defaultPutOptions.DisableContentSha256, - }) - - if err != nil { - return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.Path(node), bs.bucket) - } - return nil -} - -// UploadFromReader stores data from a reader directly into S3 without local buffering. -func (bs *Blobstore) UploadFromReader(node *node.Node, r io.Reader, size int64) error { - _, err := bs.client.PutObject(context.Background(), bs.bucket, bs.Path(node), r, size, minio.PutObjectOptions{ - ContentType: "application/octet-stream", - SendContentMd5: bs.defaultPutOptions.SendContentMd5, - ConcurrentStreamParts: bs.defaultPutOptions.ConcurrentStreamParts, - NumThreads: bs.defaultPutOptions.NumThreads, - PartSize: bs.defaultPutOptions.PartSize, - DisableMultipart: bs.defaultPutOptions.DisableMultipart, - DisableContentSha256: bs.defaultPutOptions.DisableContentSha256, - }) - if err != nil { - return errors.Wrapf(err, "could not store object '%s' into bucket '%s'", bs.Path(node), bs.bucket) - } - return nil -} - -// Download retrieves a blob from the blobstore for reading -func (bs *Blobstore) Download(node *node.Node) (io.ReadCloser, error) { - reader, err := bs.client.GetObject(context.Background(), bs.bucket, bs.Path(node), minio.GetObjectOptions{}) - if err != nil { - return nil, errors.Wrapf(err, "could not download object '%s' from bucket '%s'", bs.Path(node), bs.bucket) - } - - stat, err := reader.Stat() - if err != nil { - return nil, errors.Wrapf(err, "blob path: %s", bs.Path(node)) - } - - if node.Blobsize != stat.Size { - return nil, fmt.Errorf("blob has unexpected size. %d bytes expected, got %d bytes", node.Blobsize, stat.Size) - } - - return reader, nil -} - -// Delete deletes a blob from the blobstore -func (bs *Blobstore) Delete(node *node.Node) error { - err := bs.client.RemoveObject(context.Background(), bs.bucket, bs.Path(node), minio.RemoveObjectOptions{}) - if err != nil { - return errors.Wrapf(err, "could not delete object '%s' from bucket '%s'", bs.Path(node), bs.bucket) - } - return nil -} - -// List lists all blobs in the Blobstore -func (bs *Blobstore) List() ([]*node.Node, error) { - ch := bs.client.ListObjects(context.Background(), bs.bucket, minio.ListObjectsOptions{Recursive: true}) - - var err error - ids := make([]*node.Node, 0) - for oi := range ch { - if oi.Err != nil { - err = oi.Err - continue - } - spaceid, blobid, _ := strings.Cut(oi.Key, "/") - ids = append(ids, &node.Node{ - SpaceID: strings.ReplaceAll(spaceid, "/", ""), - BlobID: strings.ReplaceAll(blobid, "/", ""), - }) - } - return ids, err -} - -func (bs *Blobstore) Path(node *node.Node) string { - // https://aws.amazon.com/de/premiumsupport/knowledge-center/s3-prefix-nested-folders-difference/ - // Prefixes are used to partion a bucket. A prefix is everything except the filename. - // For a file `BucketName/foo/bar/lorem.ipsum`, `BucketName/foo/bar/` is the prefix. - // There are request limits per prefix, therefore you should have many prefixes. - // There are no limits to prefixes per bucket, so in general it's better to have more then less. - // - // Since the spaceID is always the same for a space, we don't need to pathify that, because it would - // not yield any performance gains - return filepath.Clean(filepath.Join(node.SpaceID, lookup.Pathify(node.BlobID, 4, 2))) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/option.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/option.go deleted file mode 100644 index 3f0f3201fd..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/option.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng - -import ( - "github.com/go-viper/mapstructure/v2" - "github.com/pkg/errors" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - - // Endpoint of the s3 blobstore - S3Endpoint string `mapstructure:"s3.endpoint"` - - // Region of the s3 blobstore - S3Region string `mapstructure:"s3.region"` - - // Bucket of the s3 blobstore - S3Bucket string `mapstructure:"s3.bucket"` - - // Access key for the s3 blobstore - S3AccessKey string `mapstructure:"s3.access_key"` - - // Secret key for the s3 blobstore - S3SecretKey string `mapstructure:"s3.secret_key"` - - // disable sending content sha256 - DisableContentSha256 bool `mapstructure:"s3.disable_content_sha254"` - - // disable multipart uploads - DisableMultipart bool `mapstructure:"s3.disable_multipart"` - - // enable sending content md5, defaults to true if unset - SendContentMd5 bool `mapstructure:"s3.send_content_md5"` - - // use concurrent stream parts - ConcurrentStreamParts bool `mapstructure:"s3.concurrent_stream_parts"` - - // number of concurrent uploads - NumThreads uint `mapstructure:"s3.num_threads"` - - // part size for concurrent uploads - PartSize uint64 `mapstructure:"s3.part_size"` -} - -// S3ConfigComplete return true if all required s3 fields are set -func (o *Options) S3ConfigComplete() bool { - return o.S3Endpoint != "" && - o.S3Region != "" && - o.S3Bucket != "" && - o.S3AccessKey != "" && - o.S3SecretKey != "" -} - -func parseConfig(m map[string]interface{}) (*Options, error) { - o := &Options{} - if err := mapstructure.Decode(m, o); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - - // if unset we set these defaults - if m["s3.send_content_md5"] == nil { - o.SendContentMd5 = true - } - if m["s3.concurrent_stream_parts"] == nil { - o.ConcurrentStreamParts = true - } - if m["s3.num_threads"] == nil { - o.NumThreads = 4 - } - return o, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/s3ng.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/s3ng.go deleted file mode 100644 index 903e224bd0..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/s3ng.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package s3ng - -import ( - "fmt" - - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry" - "github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs" - "github.com/rs/zerolog" -) - -func init() { - registry.Register("s3ng", New) -} - -// New returns an implementation to of the storage.FS interface that talk to -// a local filesystem. -func New(m map[string]interface{}, stream events.Stream, log *zerolog.Logger) (storage.FS, error) { - o, err := parseConfig(m) - if err != nil { - return nil, err - } - - if !o.S3ConfigComplete() { - return nil, fmt.Errorf("S3 configuration incomplete") - } - - defaultPutOptions := blobstore.Options{ - DisableContentSha256: o.DisableContentSha256, - DisableMultipart: o.DisableMultipart, - SendContentMd5: o.SendContentMd5, - ConcurrentStreamParts: o.ConcurrentStreamParts, - NumThreads: o.NumThreads, - PartSize: o.PartSize, - } - - bs, err := blobstore.New(o.S3Endpoint, o.S3Region, o.S3Bucket, o.S3AccessKey, o.S3SecretKey, defaultPutOptions) - if err != nil { - return nil, err - } - - return decomposedfs.NewDefault(m, bs, stream, log) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go index 69ebcc93f9..5f7290aa40 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects/aspects.go @@ -28,11 +28,10 @@ import ( // Aspects holds dependencies for handling aspects of the decomposedfs type Aspects struct { - Lookup node.PathLookup - Tree node.Tree - Trashbin trashbin.Trashbin - Permissions permissions.Permissions - EventStream events.Stream - DisableVersioning bool - UserMapper usermapper.Mapper + Lookup node.PathLookup + Tree node.Tree + Trashbin trashbin.Trashbin + Permissions permissions.Permissions + EventStream events.Stream + UserMapper usermapper.Mapper } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go index 462889799d..d0d744d98e 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/commit_upload.go @@ -19,6 +19,8 @@ package decomposedfs import ( "context" + "path/filepath" + "strings" "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" @@ -27,6 +29,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" ) @@ -62,20 +65,24 @@ func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Refere return n.RemoveXattr(ctx, prefixes.StatusPrefix) } - if n.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } + // a newer upload takes over the mark, the last writer wins return n.SetXattrsWithContext(ctx, node.Attributes{ prefixes.StatusPrefix: []byte(node.ProcessingStatus + sessionID), }) } // CommitUpload writes the staged bytes from source to the resource at ref. -// sessionID is used to identify the correct blob slot prepared before postprocessing. +// sessionID identifies the upload that PrepareUpload wrote onto the node. // Caller owns source.Body and must close it after CommitUpload returns. +// +// The bytes are written against the real node, so drivers that keep the content +// in the node itself (posix) can replace it atomically. If a newer upload took +// over the node in the meantime, the bytes go to the revision that upload created +// for this session instead. Without such a revision (versioning disabled) they are +// discarded: nothing references them anymore. func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source storage.UploadSource) error { - if source.Body == nil { - return errtypes.BadRequest("Decomposedfs: source body is nil") + if source.Body == nil && source.Path == "" { + return errtypes.BadRequest("Decomposedfs: source has neither body nor path") } if sessionID == "" { return errtypes.BadRequest("Decomposedfs: sessionID is empty") @@ -89,17 +96,75 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc if !n.Exists { return errtypes.NotFound(ref.String()) } + n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false) + if err != nil { + return err + } - blobNode := node.New(n.SpaceID, n.ID, "", "", source.Length, sessionID, - provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu) + unlock, err := fs.lu.MetadataBackend().Lock(n) + if err != nil { + return errors.Wrap(err, "Decomposedfs: could not lock node") + } + defer func() { + _ = unlock() + }() - if err := fs.tp.WriteBlobFromReader(blobNode, source.Body, source.Length); err != nil { - if derr := fs.tp.DeleteBlob(blobNode); derr != nil { + // re-read the blob id while holding the lock, a newer upload may have taken over. + // read the attribute itself: the posix lookup's ReadBlobIDAndSizeAttr reports no blob id + blobID, err := fs.lu.MetadataBackend().Get(ctx, n, prefixes.BlobIDAttr) + if err != nil { + return errors.Wrap(err, "Decomposedfs: could not read blob id") + } + n.BlobID = string(blobID) + + target := n + if n.BlobID != sessionID { + revision, revisionUnlock, err := fs.uploadRevision(ctx, n, sessionID) + if err != nil { + return err + } + if revision == nil { + appctx.GetLogger(ctx).Info().Str("nodeid", n.ID).Str("sessionid", sessionID).Msg("upload was superseded and has no revision, discarding its bytes") + return nil + } + defer func() { + _ = revisionUnlock() + }() + target = revision + } + + if source.Path != "" { + err = fs.tp.WriteBlob(target, source.Path) + } else { + err = fs.tp.WriteBlobFromReader(target, source.Body, source.Length) + } + if err != nil { + if derr := fs.tp.DeleteBlob(target); derr != nil { appctx.GetLogger(ctx).Error().Err(derr).Str("nodeid", n.ID).Str("blobid", sessionID).Msg("could not clean up orphaned blob after failed write") } return errors.Wrap(err, "Decomposedfs: failed to write blob") } + // on the node that holds the content, not the session: the session is deleted when the + // upload finishes + if !source.ScanDate.IsZero() { + if err := target.SetScanData(ctx, source.ScanResult, source.ScanDate); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", target.ID).Msg("could not set scan results") + } + } + + if target != n { + // the revision kept this upload's processing mark for a rollback of the newer + // upload; this upload is done, so restoring it must not mark the node again + if err := target.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", target.ID).Msg("could not remove the processing mark from the revision") + } + } else { + // revisions kept only for a rollback of this upload served their purpose; versions + // stay, also the ones created before versioning was disabled + fs.removeRevisions(ctx, n) + } + now := time.Now() if p, err := n.Parent(ctx); err != nil { appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not read parent for etag propagation") @@ -112,3 +177,72 @@ func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Referenc return nil } + +// removeRevisions deletes the revisions of n that were kept only for a rollback, with their +// blobs, while the caller holds the node lock: the ones of uploads that were rolled back, +// and the snapshots taken with versioning disabled. +func (fs *Decomposedfs) removeRevisions(ctx context.Context, n *node.Node) { + revPrefix := fs.lu.VersionPath(n.SpaceID, n.ID, "") + revisions, err := filepath.Glob(fs.lu.VersionPath(n.SpaceID, n.ID, "*")) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not list revisions") + return + } + for _, rev := range revisions { + if strings.HasSuffix(rev, ".mpk") || strings.HasSuffix(rev, ".mlock") { + continue + } + revisionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+strings.TrimPrefix(rev, revPrefix), fs.lu) + if !fs.keptForRollback(ctx, revisionNode) { + continue + } + blobID, err := fs.lu.MetadataBackend().Get(ctx, revisionNode, prefixes.BlobIDAttr) + if err == nil && len(blobID) > 0 && string(blobID) != n.BlobID { + blobNode := node.New(n.SpaceID, revisionNode.ID, "", "", 0, string(blobID), provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu) + if err := fs.tp.DeleteBlob(blobNode); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("revisionid", revisionNode.ID).Msg("could not delete the blob of a superseded revision") + } + } + removeRevision(ctx, fs.lu, revisionNode) + } +} + +// keptForRollback reports whether a revision exists only for a rollback, rather than as a +// version. +func (fs *Decomposedfs) keptForRollback(ctx context.Context, revision metadata.MetadataNode) bool { + for _, attr := range []string{purgeOnRestoreAttr, uploadSnapshotAttr} { + if _, err := fs.lu.MetadataBackend().Get(ctx, revision, attr); err == nil { + return true + } + } + return false +} + +// uploadRevision returns the locked revision of n that holds the upload sessionID, +// or nil if there is none. +func (fs *Decomposedfs) uploadRevision(ctx context.Context, n *node.Node, sessionID string) (*node.Node, metadata.UnlockFunc, error) { + revPrefix := fs.lu.VersionPath(n.SpaceID, n.ID, "") + revisions, err := filepath.Glob(fs.lu.VersionPath(n.SpaceID, n.ID, "*")) + if err != nil { + return nil, nil, err + } + for _, rev := range revisions { + if strings.HasSuffix(rev, ".mpk") || strings.HasSuffix(rev, ".mlock") { + continue + } + revisionID := n.ID + node.RevisionIDDelimiter + strings.TrimPrefix(rev, revPrefix) + blobID, err := fs.lu.MetadataBackend().Get(ctx, node.NewBaseNode(n.SpaceID, revisionID, fs.lu), prefixes.BlobIDAttr) + if err != nil || string(blobID) != sessionID { + continue + } + revision, unlock, err := node.LockAndReadNode(ctx, fs.lu, n.SpaceID, revisionID, "", false, n.SpaceRoot, false) + if err != nil { + return nil, nil, err + } + // ReadNode caches the attributes of the node, not of the revision. The posix + // blobstore copies them onto the file it writes, so read the revision's own. + revision.ResetXattrsCache() + return revision, unlock, nil + } + return nil, nil, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go index 22e4d65c45..2671274121 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/decomposedfs.go @@ -36,18 +36,17 @@ import ( "github.com/jellydator/ttlcache/v2" "github.com/pkg/errors" "github.com/rs/zerolog" - tusd "github.com/tus/tusd/v2/pkg/handler" microstore "go-micro.dev/v4/store" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" "github.com/opencloud-eu/reva/v2/pkg/appctx" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup" @@ -60,9 +59,7 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks" "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" "github.com/opencloud-eu/reva/v2/pkg/storagespace" @@ -80,11 +77,6 @@ var ( tracer trace.Tracer _registeredEvents = []events.Unmarshaller{ - events.PostprocessingFinished{}, - events.PostprocessingStepFinished{}, - events.RestartPostprocessing{}, - events.StartPostprocessingStep{}, - events.CleanUpload{}, events.DeleteRevision{}, } ) @@ -93,41 +85,19 @@ func init() { tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs") } -// Session is the interface that DecomposedfsSession implements. By combining tus.Upload, -// storage.UploadSession and custom functions we can reuse the same struct throughout -// the whole upload lifecycle. -// -// Some functions that are only used by decomposedfs are not yet part of this interface. -// They might be added after more refactoring. -type Session interface { - tusd.Upload - storage.UploadSession - upload.Session - LockID() string -} - type IDCachingTree interface { WarmupIDCache(root string, assimilate, onlyDirty bool) error } -type SessionStore interface { - New(ctx context.Context) *upload.DecomposedFsSession - SessionFromInfo(info tusd.FileInfo) *upload.DecomposedFsSession - List(ctx context.Context) ([]*upload.DecomposedFsSession, error) - Get(ctx context.Context, id string) (*upload.DecomposedFsSession, error) -} - // Decomposedfs provides the base for decomposed filesystem implementations type Decomposedfs struct { - lu node.PathLookup - tp node.Tree - trashbin trashbin.Trashbin - o *options.Options - p permissions.Permissions - um usermapper.Mapper - chunkHandler *chunking.ChunkHandler - stream events.Stream - sessionStore SessionStore + lu node.PathLookup + tp node.Tree + trashbin trashbin.Trashbin + o *options.Options + p permissions.Permissions + um usermapper.Mapper + stream events.Stream UserCache *ttlcache.Cache userSpaceIndex *spaceidindex.Index @@ -184,12 +154,11 @@ func NewDefault(m map[string]interface{}, bs node.Blobstore, es events.Stream, l ), log) aspects := aspects.Aspects{ - Lookup: lu, - Tree: tp, - Permissions: p, - EventStream: es, - DisableVersioning: o.DisableVersioning, - Trashbin: &DecomposedfsTrashbin{}, + Lookup: lu, + Tree: tp, + Permissions: p, + EventStream: es, + Trashbin: &DecomposedfsTrashbin{}, } return New(o, aspects, log) @@ -251,7 +220,6 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor o: o, p: aspects.Permissions, um: aspects.UserMapper, - chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")), stream: aspects.EventStream, UserCache: ttlcache.NewCache(), userSpaceIndex: userSpaceIndex, @@ -260,7 +228,6 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor spaceTypeIndex: spaceTypeIndex, log: log, } - fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens, log) if err = fs.trashbin.Setup(fs); err != nil { return nil, err } @@ -271,7 +238,9 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor return nil, errors.New("need nats for async file processing") } - ch, err := events.Consume(fs.stream, "dcfs", _registeredEvents...) + // a group of its own: the upload coordinator holds the "dcfs" group, and one + // group gets one copy of each event + ch, err := events.Consume(fs.stream, "dcfs-revisions", _registeredEvents...) if err != nil { return nil, err } @@ -281,201 +250,32 @@ func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (stor } for i := 0; i < o.Events.NumConsumers; i++ { - go fs.Postprocessing(ch) + go fs.ConsumeRevisionEvents(ch) } } return fs, nil } -// Postprocessing starts the postprocessing result collector -func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { +// ConsumeRevisionEvents starts the revision event collector. Upload postprocessing +// results are consumed by the upload coordinator. +func (fs *Decomposedfs) ConsumeRevisionEvents(ch <-chan events.Event) { for event := range ch { // derive a per-event context from the event's W3C trace context so the - // postprocessing work is linked to the originating request + // work is linked to the originating request ctx := event.GetTraceContext(context.Background()) ctx = ctxpkg.ContextSetInitiator(ctx, event.InitiatorID) - fs.handlePostprocessingEvent(ctx, event) + fs.handleRevisionEvent(ctx, event) } } -// handlePostprocessingEvent processes a single postprocessing event under a span -// scoped to that event's trace context. -func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event events.Event) { - ctx, span := tracer.Start(ctx, "Postprocessing/"+event.Type) +// handleRevisionEvent processes a single revision event under a span scoped to +// that event's trace context. +func (fs *Decomposedfs) handleRevisionEvent(ctx context.Context, event events.Event) { + ctx, span := tracer.Start(ctx, "ConsumeRevisionEvents/"+event.Type) defer span.End() log := fs.log switch ev := event.Event.(type) { - case events.PostprocessingFinished: - sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - if !n.Exists { - sublog.Debug().Msg("node no longer exists") - session.Cleanup(false, true, true, false) - return - } - - var ( - failed bool - revertNodeMetadata bool - keepUpload bool - ) - unmarkPostprocessing := true - - switch ev.Outcome { - default: - sublog.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - revertNodeMetadata = true - keepUpload = true - metrics.UploadSessionsAborted.Inc() - case events.PPOutcomeContinue: - if err := session.Finalize(ctx); err != nil { - sublog.Error().Err(err).Msg("could not finalize upload") - failed = true - revertNodeMetadata = false - keepUpload = true - // keep postprocessing status so the upload is not deleted during housekeeping - unmarkPostprocessing = false - } else { - metrics.UploadSessionsFinalized.Inc() - } - case events.PPOutcomeDelete: - failed = true - revertNodeMetadata = true - metrics.UploadSessionsDeleted.Inc() - } - - getParent := func() *node.Node { - p, err := n.Parent(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read parent") - return nil - } - return p - } - - now := time.Now() - var parentId *provider.ResourceId - if failed { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - sublog.Error().Err(err).Msg("reading node for session failed") - } - if latestSession == session.ID() { - // propagate reverted sizeDiff after failed postprocessing - if err := fs.tp.Propagate(ctx, n, -session.SizeDiff()); err != nil { - sublog.Error().Err(err).Msg("could not propagate tree size change") - } - } - } else if p := getParent(); p != nil { - parentId = &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: p.ID, - } - // update parent tmtime to propagate etag change after successful postprocessing - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - sublog.Error().Err(err).Msg("could not propagate etag change") - } - } - - session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) - - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionID"] != "" { - isVersion = true - } - } - - if err := events.Publish( - ctx, - fs.stream, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - }, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - ParentID: parentId, - Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), - IsVersion: isVersion, - ImpersonatingUser: ev.ImpersonatingUser, - }, - ); err != nil { - sublog.Error().Err(err).Msg("Failed to publish UploadReady event") - } - case events.RestartPostprocessing: - sublog := log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - s, err := session.URL(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not create url") - return - } - - metrics.UploadSessionsRestarted.Inc() - - // restart postprocessing - if err := events.Publish(ctx, fs.stream, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(ctx), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - }); err != nil { - sublog.Error().Err(err).Msg("Failed to publish BytesReceived event") - } - case events.CleanUpload: - sublog := log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) case events.DeleteRevision: sublog := log.With().Str("event", "DeleteRevision").Interface("nodeid", ev.ResourceID).Logger() n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) @@ -501,152 +301,6 @@ func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event eve if deleteErr != nil { sublog.Error().Err(deleteErr).Msg("Failed to delete revision") } - case events.StartPostprocessingStep: - sublog := log.With().Str("event", "StartPostprocessingStep").Str("uploadid", ev.UploadID).Logger() - if ev.UploadID == "" { - sublog.Error().Msg("UploadID is empty, cannot start postprocessing step") - return - } - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - session.SetStatus(upload.SessionStatusProcessing, "started postprocessing step: "+string(ev.StepToStart)) - err = session.Persist(ctx) - if err != nil { - sublog.Error().Err(err).Msg("Failed to persist upload session after starting postprocessing step") - } - case events.PostprocessingStepFinished: - sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() - if ev.FinishedStep != events.PPStepAntivirus { - // atm we are only interested in antivirus results - return - } - - res := ev.Result.(events.VirusscanResult) - if res.ErrorMsg != "" { - // scan failed somehow - // Should we handle this here? - return - } - sublog = log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() - - var n *node.Node - switch ev.UploadID { - case "": - // uploadid is empty -> this was an on-demand scan - /* ON DEMAND SCANNING NOT SUPPORTED ATM - ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) - ref := &provider.Reference{ResourceId: ev.ResourceID} - - no, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") - continue - - } - n = no - if ev.Outcome == events.PPOutcomeDelete { - // antivir wants us to delete the file. We must obey and need to - - // check if there a previous versions existing - revs, err := fs.ListRevisions(ctx, ref) - if len(revs) == 0 { - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") - } - - // no versions -> trash file - err := fs.Delete(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") - continue - } - - // now purge it from the recycle bin - if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - - // we have versions - find the newest - versions := make(map[uint64]string) // remember all versions - we need them later - var nv uint64 - for _, v := range revs { - versions[v.Mtime] = v.Key - if v.Mtime > nv { - nv = v.Mtime - } - } - - // restore newest version - if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") - continue - } - - // now find infected version - revs, err = fs.ListRevisions(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") - } - - for _, v := range revs { - // we looking for a version that was previously not there - if _, ok := versions[v.Mtime]; ok { - continue - } - - if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") - } - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - */ - default: - // uploadid is not empty -> this is an async upload - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - - n, err = session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node after scan") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - - session.SetScanData(res.Description, res.Scandate) - var status string - switch ev.Outcome { - case events.PPOutcomeAbort, events.PPOutcomeDelete: - status = upload.SessionStatusFailed - default: - status = upload.SessionStatusProcessing - } - session.SetStatus(status, "finished postprocessing step: "+string(ev.FinishedStep)) - if err := session.Persist(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to persist scan results") - } - } - - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - sublog.Error().Err(err).Msg("Failed to set scan results") - return - } - - metrics.UploadSessionsScanned.Inc() default: log.Error().Interface("event", ev).Msg("Unknown event") } @@ -664,6 +318,12 @@ func (fs *Decomposedfs) Shutdown(ctx context.Context) error { return nil } +// Capabilities declares decomposedfs fully writeable, stated explicitly so it is +// a decision here and not an inherited default. +func (fs *Decomposedfs) Capabilities(_ context.Context) storage.Capabilities { + return storage.FullCapabilities() +} + // GetQuota returns the quota available // TODO Document in the cs3 should we return quota or free space? func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, remaining uint64, err error) { @@ -1113,7 +773,8 @@ func (fs *Decomposedfs) ListFolder(ctx context.Context, ref *provider.Reference, for i := 0; i < numWorkers; i++ { g.Go(func() error { for child := range work { - np := rp + // copy the parent's permissions: the workers must not share them + np := proto.Clone(rp).(*provider.ResourcePermissions) // add this childs permissions pset, _ := child.PermissionSet(ctx) node.AddPermissions(np, pset) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/hybrid_backend.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/hybrid_backend.go index 040740d6f3..80d6e9c588 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/hybrid_backend.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/hybrid_backend.go @@ -14,7 +14,6 @@ import ( "github.com/google/renameio/v2" "github.com/pkg/errors" "github.com/pkg/xattr" - "github.com/rogpeppe/go-internal/lockedfile" "github.com/vmihailenco/msgpack/v5" "github.com/opencloud-eu/reva/v2/pkg/storage/cache" @@ -30,6 +29,7 @@ type HybridBackend struct { offloadLimit int metaCache cache.FileMetadataCache metadataPathFunc MetadataPathFunc + metrics *backendMetrics } // NewMessageBackend returns a new HybridBackend instance @@ -38,6 +38,7 @@ func NewHybridBackend(offloadLimit int, metadataPathFunc MetadataPathFunc, o cac offloadLimit: offloadLimit, metaCache: cache.GetFileMetadataCache(o), metadataPathFunc: metadataPathFunc, + metrics: newBackendMetrics("hybrid"), } } @@ -46,6 +47,7 @@ func (HybridBackend) Name() string { return "hybrid" } // IdentifyPath returns the space id, node id and mtime of a file func (b HybridBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) { + b.metrics.reads.Inc() spaceID, err := xattr.Get(path, prefixes.SpaceIDAttr) if err != nil { if IsNotExist(err) { @@ -66,10 +68,13 @@ func (b HybridBackend) IdentifyPath(_ context.Context, path string) (string, str func (b HybridBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) { attribs := map[string][]byte{} err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs) - if err == nil && len(attribs[key]) > 0 { + found := err == nil && len(attribs[key]) > 0 + b.metrics.cacheLookup(found) + if found { return attribs[key], err } + b.metrics.reads.Inc() if isOffloadingAttribute(key) { // check if key is offloaded offloaded, err := xattr.Get(n.InternalPath(), metadataOffloadedAttr()) @@ -122,6 +127,7 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by attribs := map[string][]byte{} err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs) + b.metrics.cacheLookup(err == nil) if err == nil { return attribs, err } @@ -138,6 +144,7 @@ func (b HybridBackend) All(ctx context.Context, n MetadataNode) (map[string][]by } func (b HybridBackend) getAll(ctx context.Context, n MetadataNode, skipOffloaded bool) (map[string][]byte, error) { + b.metrics.reads.Inc() attribs := map[string][]byte{} attrNames, err := b.list(ctx, n) if err != nil { @@ -217,6 +224,8 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs defer func() { _ = unlock() }() } + // counts once together with reading the offloaded metadata below + b.metrics.reads.Inc() offloadAttr, err := xattr.Get(path, metadataOffloadedAttr()) offloaded := err == nil && string(offloadAttr) == "1" @@ -254,6 +263,7 @@ func (b HybridBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs } } + b.metrics.writes.Inc() if offloaded { metaPath := b.MetadataPath(n) var msgBytes []byte @@ -340,6 +350,7 @@ func (b HybridBackend) offloadMetadata(ctx context.Context, n MetadataNode) erro return err } + b.metrics.writes.Inc() err = renameio.WriteFile(b.MetadataPath(n), d, 0600) if err != nil { return err @@ -376,6 +387,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e } if isOffloadingAttribute(key) { + b.metrics.reads.Inc() offloadAttr, err := xattr.Get(path, metadataOffloadedAttr()) offloaded := err == nil && string(offloadAttr) == "1" if offloaded { @@ -412,12 +424,14 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e return err } + b.metrics.writes.Inc() err = renameio.WriteFile(b.MetadataPath(n), d, 0600) if err != nil { return err } } else { // remove from xattrs + b.metrics.writes.Inc() err := xattr.Remove(path, key) if err != nil { return err @@ -425,6 +439,7 @@ func (b HybridBackend) Remove(ctx context.Context, n MetadataNode, key string) e } } else { // remove from xattrs + b.metrics.writes.Inc() err := xattr.Remove(path, key) if err != nil { return err @@ -452,6 +467,7 @@ func (b HybridBackend) Purge(ctx context.Context, n MetadataNode) error { attribs, err := b.getAll(ctx, n, false) if err == nil { + b.metrics.writes.Inc() for attr := range attribs { if strings.HasPrefix(attr, prefixes.OcPrefix) { _ = xattr.Remove(path, attr) @@ -497,7 +513,7 @@ func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) { } metaLockPath := b.LockfilePath(n) - mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics) if err != nil { if errors.Is(err, os.ErrNotExist) { // The lock file's parent directories don't exist yet (or anymore). @@ -517,7 +533,7 @@ func (b HybridBackend) Lock(n MetadataNode) (UnlockFunc, error) { return nil, err } } - mlock, err = lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + mlock, err = lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics) if err != nil { return nil, err } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/lock.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/lock.go new file mode 100644 index 0000000000..0ddf3a46cc --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/lock.go @@ -0,0 +1,41 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package metadata + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/rogpeppe/go-internal/lockedfile" +) + +// lockFile opens and locks the lock file of a node like lockedfile.OpenFile, recording in the metrics how long +// it waits for the lock and, once the returned lock is closed, how long the lock has been held +func lockFile(n MetadataNode, path string, flag int, m *backendMetrics) (*heldLock, error) { + lm := m.locks(n) + + m.locksWaiting.Inc() + start := time.Now() + f, err := lockedfile.OpenFile(path, flag, 0600) + m.locksWaiting.Dec() + if err != nil { + return nil, err + } + locked := time.Now() + lm.wait.Observe(locked.Sub(start).Seconds()) + return &heldLock{f: f, held: lm.held, since: locked}, nil +} + +// heldLock is a locked lock file +type heldLock struct { + f *lockedfile.File + held prometheus.Observer + since time.Time +} + +// Close unlocks and closes the lock file +func (l *heldLock) Close() error { + l.held.Observe(time.Since(l.since).Seconds()) + return l.f.Close() +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/messagepack_backend.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/messagepack_backend.go index af719f4ac1..3ab894c39d 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/messagepack_backend.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/messagepack_backend.go @@ -33,7 +33,6 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/cache" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" "github.com/pkg/xattr" - "github.com/rogpeppe/go-internal/lockedfile" "github.com/vmihailenco/msgpack/v5" "go.opentelemetry.io/otel/codes" ) @@ -41,12 +40,14 @@ import ( // MessagePackBackend persists the attributes in messagepack format inside the file type MessagePackBackend struct { metaCache cache.FileMetadataCache + metrics *backendMetrics } // NewMessagePackBackend returns a new MessagePackBackend instance func NewMessagePackBackend(o cache.Config) MessagePackBackend { return MessagePackBackend{ metaCache: cache.GetFileMetadataCache(o), + metrics: newBackendMetrics("messagepack"), } } @@ -56,6 +57,7 @@ func (MessagePackBackend) Name() string { return "messagepack" } // IdentifyPath returns the id and mtime of a file func (b MessagePackBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) { metaPath := filepath.Clean(path + ".mpk") + b.metrics.reads.Inc() source, err := os.Open(metaPath) // // No cached entry found. Read from storage and store in cache if err != nil { @@ -157,6 +159,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, defer func() { _ = unlock() }() } // Read current state + b.metrics.reads.Inc() _, subspan := tracer.Start(ctx, "os.ReadFile") var msgBytes []byte msgBytes, err = os.ReadFile(metaPath) @@ -192,6 +195,7 @@ func (b MessagePackBackend) saveAttributes(ctx context.Context, n MetadataNode, } // overwrite file atomically + b.metrics.writes.Inc() _, subspan = tracer.Start(ctx, "renameio.Writefile") err = renameio.WriteFile(metaPath, d, 0600) if err != nil { @@ -210,6 +214,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode) defer span.End() attribs := map[string][]byte{} err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs) + b.metrics.cacheLookup(err == nil) if err == nil { return attribs, err } @@ -217,6 +222,7 @@ func (b MessagePackBackend) loadAttributes(ctx context.Context, n MetadataNode) metaPath := b.MetadataPath(n) // No cached entry found. Read from storage and store in cache + b.metrics.reads.Inc() _, subspan := tracer.Start(ctx, "os.Open") source, err := os.Open(metaPath) subspan.End() @@ -271,6 +277,7 @@ func (b MessagePackBackend) Purge(_ context.Context, n MetadataNode) error { return err } + b.metrics.writes.Inc() return os.Remove(b.MetadataPath(n)) } @@ -289,6 +296,7 @@ func (b MessagePackBackend) Rename(oldNode, newNode MetadataNode) error { return err } + b.metrics.writes.Inc() return os.Rename(b.MetadataPath(oldNode), b.MetadataPath(newNode)) } @@ -305,7 +313,7 @@ func (b MessagePackBackend) Lock(n MetadataNode) (UnlockFunc, error) { } metaLockPath := b.LockfilePath(n) - mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics) if err != nil { return nil, err } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/metrics.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/metrics.go new file mode 100644 index 0000000000..ee50ae29d6 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/metrics.go @@ -0,0 +1,92 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package metadata + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// the metrics of the backends, by the name of the backend, see backendMetrics. A read or write of the +// metadata of a node counts once, however many attributes or files it involves. +var ( + reads = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_metadata_reads_total", + Help: "Number of times the metadata of a node has been read from the filesystem", + }, []string{"backend"}) + writes = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_metadata_writes_total", + Help: "Number of times the metadata of a node has been written to the filesystem", + }, []string{"backend"}) + cacheLookups = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_metadata_cache_lookups_total", + Help: "Number of times the metadata of a node has been looked up in the cache, by whether it has been found (result hit) or not (result miss)", + }, []string{"backend", "result"}) + locksWaiting = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "reva_metadata_lock_waiting", + Help: "Number of metadata locks being waited for", + }, []string{"backend"}) + lockWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "reva_metadata_lock_wait_seconds", + Help: "Time spent waiting for metadata locks, by whether the node is a space root", + Buckets: prometheus.ExponentialBuckets(.0001, 4, 10), + }, []string{"backend", "space_root"}) + lockHeld = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "reva_metadata_lock_held_seconds", + Help: "Time metadata locks have been held, by whether the node is a space root", + Buckets: prometheus.ExponentialBuckets(.0001, 4, 10), + }, []string{"backend", "space_root"}) +) + +// backendMetrics are the metrics of a backend +type backendMetrics struct { + reads prometheus.Counter + writes prometheus.Counter + cacheHits prometheus.Counter + cacheMisses prometheus.Counter + locksWaiting prometheus.Gauge + // the locks of space roots are the ones every propagation in a space ends at + spaceRootLocks lockMetrics + otherLocks lockMetrics +} + +type lockMetrics struct { + wait prometheus.Observer + held prometheus.Observer +} + +func newBackendMetrics(backend string) *backendMetrics { + return &backendMetrics{ + reads: reads.WithLabelValues(backend), + writes: writes.WithLabelValues(backend), + cacheHits: cacheLookups.WithLabelValues(backend, "hit"), + cacheMisses: cacheLookups.WithLabelValues(backend, "miss"), + locksWaiting: locksWaiting.WithLabelValues(backend), + spaceRootLocks: lockMetrics{ + wait: lockWait.WithLabelValues(backend, "true"), + held: lockHeld.WithLabelValues(backend, "true"), + }, + otherLocks: lockMetrics{ + wait: lockWait.WithLabelValues(backend, "false"), + held: lockHeld.WithLabelValues(backend, "false"), + }, + } +} + +// cacheLookup counts a lookup in the cache +func (m *backendMetrics) cacheLookup(found bool) { + if found { + m.cacheHits.Inc() + } else { + m.cacheMisses.Inc() + } +} + +// locks returns the lock metrics for the locks of a node +func (m *backendMetrics) locks(n MetadataNode) lockMetrics { + if id := n.GetID(); id != "" && id == n.GetSpaceID() { + return m.spaceRootLocks + } + return m.otherLocks +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/xattrs_backend.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/xattrs_backend.go index 0111a72a23..629072b448 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/xattrs_backend.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/xattrs_backend.go @@ -31,18 +31,19 @@ import ( "github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks" "github.com/pkg/errors" "github.com/pkg/xattr" - "github.com/rogpeppe/go-internal/lockedfile" ) // XattrsBackend stores the file attributes in extended attributes type XattrsBackend struct { metaCache cache.FileMetadataCache + metrics *backendMetrics } // NewMessageBackend returns a new XattrsBackend instance func NewXattrsBackend(o cache.Config) XattrsBackend { return XattrsBackend{ metaCache: cache.GetFileMetadataCache(o), + metrics: newBackendMetrics("xattrs"), } } @@ -51,6 +52,7 @@ func (XattrsBackend) Name() string { return "xattrs" } // IdentifyPath returns the space id, node id and mtime of a file func (b XattrsBackend) IdentifyPath(_ context.Context, path string) (string, string, string, time.Time, error) { + b.metrics.reads.Inc() spaceID, _ := xattr.Get(path, prefixes.SpaceIDAttr) id, _ := xattr.Get(path, prefixes.IDAttr) parentID, _ := xattr.Get(path, prefixes.ParentidAttr) @@ -66,10 +68,13 @@ func (b XattrsBackend) IdentifyPath(_ context.Context, path string) (string, str func (b XattrsBackend) Get(ctx context.Context, n MetadataNode, key string) ([]byte, error) { attribs := map[string][]byte{} err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs) - if err == nil && len(attribs[key]) > 0 { + found := err == nil && len(attribs[key]) > 0 + b.metrics.cacheLookup(found) + if found { return attribs[key], err } + b.metrics.reads.Inc() return xattr.Get(n.InternalPath(), key) } @@ -102,7 +107,7 @@ func (b XattrsBackend) list(ctx context.Context, n MetadataNode, acquireLock boo // listing xattrs failed, try again, either with lock or without if acquireLock { - f, err := lockedfile.OpenFile(filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) + f, err := lockFile(n, filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, b.metrics) if err != nil { return nil, err } @@ -128,11 +133,13 @@ func (b XattrsBackend) getAll(ctx context.Context, n MetadataNode, skipCache, ac if !skipCache { err := b.metaCache.PullFromCache(b.cacheKey(n), &attribs) + b.metrics.cacheLookup(err == nil) if err == nil { return attribs, err } } + b.metrics.reads.Inc() attrNames, err := b.list(ctx, n, acquireLock) if err != nil { return nil, err @@ -184,7 +191,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs if err != nil { return err } - lockedFile, err := lockedfile.OpenFile(b.LockfilePath(n), os.O_CREATE|os.O_WRONLY, 0600) + lockedFile, err := lockFile(n, b.LockfilePath(n), os.O_CREATE|os.O_WRONLY, b.metrics) if err != nil { return err } @@ -202,6 +209,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs xerrs = 0 xerr error ) + b.metrics.writes.Inc() for key, val := range attribs { if xerr = xattr.Set(path, key, val); xerr != nil { // log @@ -223,7 +231,7 @@ func (b XattrsBackend) SetMultiple(ctx context.Context, n MetadataNode, attribs func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) error { path := n.InternalPath() if !n.LockHeld() { - lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) + lockedFile, err := lockFile(n, path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, b.metrics) if err != nil { return err } @@ -235,6 +243,7 @@ func (b XattrsBackend) Remove(ctx context.Context, n MetadataNode, key string) e }() } + b.metrics.writes.Inc() err := xattr.Remove(path, key) if err != nil { return err @@ -259,6 +268,7 @@ func (b XattrsBackend) Purge(ctx context.Context, n MetadataNode) error { return err } + b.metrics.writes.Inc() for attr := range attribs { if strings.HasPrefix(attr, prefixes.OcPrefix) { err := xattr.Remove(path, attr) @@ -298,7 +308,7 @@ func (b XattrsBackend) Lock(n MetadataNode) (UnlockFunc, error) { } metaLockPath := b.LockfilePath(n) - mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) + mlock, err := lockFile(n, metaLockPath, os.O_RDWR|os.O_CREATE, b.metrics) if err != nil { return nil, err } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go index 79f6623371..a941a92dfe 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node/node.go @@ -40,6 +40,7 @@ import ( "github.com/google/uuid" "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/conversions" ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/mime" @@ -57,6 +58,10 @@ import ( var tracer trace.Tracer +// ErrMissingParentID is returned when a node exists on disk but has no parent id, +// which means its metadata is gone and the node can no longer be resolved. +var ErrMissingParentID = errtypes.InternalError("Missing parent ID on node") + func init() { tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node") } @@ -524,7 +529,7 @@ func readNode(ctx context.Context, lu PathLookup, spaceID, nodeID, internalPath if _, ok := lu.MetadataBackend().(metadata.MessagePackBackend); ok { appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Str("nodeid", n.ID).Interface("attrs", attrs).Bytes("messagepack", d).Msg("missing parent id") } - return nil, errtypes.InternalError("Missing parent ID on node") + return nil, ErrMissingParentID } if n.Type(ctx) == provider.ResourceType_RESOURCE_TYPE_FILE { @@ -952,8 +957,8 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi // share indicator if _, ok := fieldMaskKeysMap["share-types"]; returnAllFields || ok { - granteeTypes := n.getGranteeTypes(ctx) - if len(granteeTypes) > 0 { + granteeTypes, hasGuestShares := n.getGranteeTypes(ctx) + if len(granteeTypes) > 0 || hasGuestShares { // TODO add optional property to CS3 ResourceInfo to transport grants? var s strings.Builder first := true @@ -965,6 +970,12 @@ func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissi } s.WriteString(strconv.Itoa(int(t))) } + if hasGuestShares { + if !first { + s.WriteString(",") + } + s.WriteString(strconv.Itoa(int(conversions.ShareTypeGuest))) + } ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "share-types", s.String()) } } @@ -1369,17 +1380,25 @@ func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) { return grants, nil } -func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { +// getGranteeTypes returns the grantee types of the node's grants. The second +// return value reports whether the node has guest shares, which are stored as +// user grants with a mail ACE prefix. +func (n *Node) getGranteeTypes(ctx context.Context) ([]provider.GranteeType, bool) { types := []provider.GranteeType{} + hasGuestShares := false if g, err := n.ListGrantees(ctx); err == nil { hasUserShares, hasGroupShares := false, false for i := range g { switch { - case !hasUserShares && (strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix) || strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix)): + case !hasGuestShares && strings.HasPrefix(g[i], prefixes.GrantMailAcePrefix): + hasGuestShares = true + case !hasUserShares && strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix): hasUserShares = true case !hasGroupShares && strings.HasPrefix(g[i], prefixes.GrantGroupAcePrefix): hasGroupShares = true - case hasUserShares && hasGroupShares: + } + + if hasUserShares && hasGroupShares && hasGuestShares { break } } @@ -1390,7 +1409,7 @@ func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { types = append(types, provider.GranteeType_GRANTEE_TYPE_GROUP) } } - return types + return types, hasGuestShares } // ancestorGrants returns the active grants on the node and all its ancestors. The grants on the @@ -1659,12 +1678,12 @@ func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, er return n.deleteRevision(ctx, n.ID+RevisionIDDelimiter+versionID) } - revisionPath, err := n.getLatestRevision(ctx) + reverted, err := n.RevertLatestRevision(ctx) if err != nil { return "", err } - if revisionPath == "" { + if !reverted { // there is no revision - delete the node unlock, err := n.lu.MetadataBackend().Lock(n) if err != nil { @@ -1680,11 +1699,6 @@ func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, er return "", nil } - latestID := strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, "")) - if err := n.RevertUpload(ctx, latestID); err != nil { - return "", err - } - // we just deleted the current revision - remove processing flag if set if uploadid, err := n.ProcessingID(ctx); err == nil { return "", n.UnmarkProcessing(ctx, uploadid) @@ -1735,6 +1749,32 @@ func (n *Node) deleteRevision(ctx context.Context, revisionID string) (string, e return blobID, nil } +// RevertLatestRevision reverts the upload that created the node's latest +// revision: the revision is restored onto the node and removed. Unlike +// DeleteRevision it neither purges the node when there is no revision nor +// touches the processing flag. It returns false if there is no revision to +// restore. +func (n *Node) RevertLatestRevision(ctx context.Context) (bool, error) { + revisionPath, err := n.getLatestRevision(ctx) + if err != nil { + return false, err + } + if revisionPath == "" { + return false, nil + } + return true, n.RevertUpload(ctx, strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, ""))) +} + +// LatestRevisionKey returns the version key of the node's latest revision, or "" if it has +// none. +func (n *Node) LatestRevisionKey(ctx context.Context) (string, error) { + revisionPath, err := n.getLatestRevision(ctx) + if err != nil || revisionPath == "" { + return "", err + } + return strings.TrimPrefix(revisionPath, n.lu.VersionPath(n.SpaceID, n.ID, "")), nil +} + func (n *Node) getLatestRevision(ctx context.Context) (string, error) { revPrefix := n.lu.VersionPath(n.SpaceID, n.ID, "") revisions, err := filepath.Glob(n.lu.VersionPath(n.SpaceID, n.ID, "*")) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go index f2827eccc2..9b785a1b53 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options/options.go @@ -45,9 +45,9 @@ type Options struct { // the attribute key prefix of the metadata on disk, process wide, see prefixes.SetOcPrefix MetadataPrefix string `mapstructure:"metadata_prefix"` - // the propagator to use for this fs. currently only `sync` is fully supported, `async` is available as an experimental feature + // the propagator to use for this fs. currently only `sync` is fully supported, `async` is deprecated and will be removed in a future release Propagator string `mapstructure:"propagator"` - // Options specific to the async propagator + // Options specific to the async propagator, which is deprecated AsyncPropagatorOptions AsyncPropagatorOptions `mapstructure:"async_propagator_options"` // decomposedfs fs works on top of a dir of uuid nodes diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/prepare_upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/prepare_upload.go new file mode 100644 index 0000000000..90ec27f5fa --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/prepare_upload.go @@ -0,0 +1,552 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package decomposedfs + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/pkg/errors" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" + "github.com/opencloud-eu/reva/v2/pkg/storagespace" +) + +// PrepareCreatesNode reports that PrepareUpload creates a new file's node. +func (fs *Decomposedfs) PrepareCreatesNode() bool { + return true +} + +// PrepareUpload finalizes node metadata after bytes are received, before postprocessing, +// and marks the node as processing for sessionID. A new file's node is created here, under +// the id the ref carries. CommitUpload is called after postprocessing completes. +// +// The previous version is snapshotted through the tree's CreateRevision, like the legacy +// upload path does: posix copies the current content, decomposedfs only the metadata. When +// another upload is still processing the node, it is snapshotted even with versioning +// disabled, together with that upload's processing mark: if this upload is rolled back, the +// other one is restored and can still finish. +func (fs *Decomposedfs) PrepareUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + ctx, span := tracer.Start(ctx, "PrepareUpload") + defer span.End() + + n, err := fs.lu.NodeFromResource(ctx, ref) + if err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("PrepareUpload: unexpected NodeFromResource failure") + return nil, errtypes.InternalError("PrepareUpload: node lookup failed unexpectedly") + } + // a new file nobody has created yet: PrepareUpload creates it + create := !n.Exists && !info.NodeExisted && info.ParentID != "" && info.Name != "" + if !n.Exists && !create { + return nil, errtypes.NotFound(ref.String()) + } + if create { + if err := fs.checkNewFileParent(ctx, n.SpaceID, info); err != nil { + return nil, err + } + n.ParentID = info.ParentID + n.Name = info.Name + } + + var ( + committed bool + // only a node this upload created is purged, one it is creating only once + // the name is its own: Purge removes the name + purge = !info.NodeExisted && !create + ) + // registered before the lock is taken, so it runs once the lock is released. + // Purge, not trash: the file never had content, and Purge needs no Delete permission + defer func() { + if committed || !purge { + return + } + if err := n.Purge(ctx); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not purge new node after failed prepare") + } + }() + + n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false) + if err != nil { + return nil, err + } + + if create { + // a node still to be created holds no lock, so any lock id is a mismatch + if lockID, _ := ctxpkg.ContextGetLockID(ctx); lockID != "" { + return nil, errtypes.Aborted("not locked") + } + } else if err := n.CheckLock(ctx); err != nil { + return nil, err + } + + // scope to space owner GID for posix deployments; no-op with NullMapper + if spaceGID, ok := ctx.Value(CtxKeySpaceGID).(uint32); ok { + unscope, err := fs.um.ScopeUserByIds(-1, int(spaceGID)) + if err != nil { + return nil, errors.Wrap(err, "failed to scope user") + } + if unscope != nil { + defer func() { _ = unscope() }() + } + } + + var unlock metadata.UnlockFunc + if create { + // InitNewNode refuses a name already taken, checks the quota, and on failure + // removes what it made and releases its lock. It returns the node lock. + unlock, err = fs.tp.InitNewNode(ctx, n, uint64(info.Size)) + if err != nil { + return nil, err + } + purge = true + } else { + unlock, err = fs.lu.MetadataBackend().Lock(n) + if err != nil { + return nil, err + } + } + defer func() { + if err := unlock(); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not close lock") + } + }() + + var ( + sizeDiff int64 + versionCreated bool + versionNode *node.BaseNode + oldAttrs node.Attributes + oldMtime time.Time + ) + + defer func() { + if committed { + return + } + if versionCreated { + removeRevision(ctx, fs.lu, versionNode) + } + if info.NodeExisted && oldAttrs != nil { + // mtime goes in the same batch: we still hold the metadata lock and + // SetMTime would try to retake it + if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &oldAttrs, oldMtime); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not restore node mtime during rollback") + } + if err := n.SetXattrsWithContext(ctx, oldAttrs); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not restore node xattrs during rollback") + } + // restoring only rewrites the old keys, so the mark this upload added would stay + if _, ok := oldAttrs[prefixes.StatusPrefix]; !ok { + if err := n.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not remove processing mark during rollback") + } + } + } + }() + + var ( + old *node.Node + overwrite bool + oldBlobsize int64 + ) + if info.NodeExisted { + old, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.ID, "", false, nil, false) + if err != nil { + return nil, errors.Wrap(err, "PrepareUpload: failed to read existing node") + } + overwrite = old.BlobID != "" + oldBlobsize = old.Blobsize + } + + // also for new files: the coordinator's check fails open without GetQuota + // permission, and CheckQuota guards disk space too. InitNewNode has checked a + // node it created. + if !create { + if _, err := node.CheckQuota(ctx, n.SpaceRoot, overwrite, uint64(oldBlobsize), uint64(info.Size)); err != nil { + return nil, err + } + } + + if info.NodeExisted { + oldMtime, err = old.GetMTime(ctx) + if err != nil { + return nil, err + } + oldEtag, err := node.CalculateEtag(old.ID, oldMtime) + if err != nil { + return nil, err + } + + if info.IfMatch != "" && info.IfMatch != oldEtag { + return nil, errtypes.Aborted("etag mismatch") + } + if info.IfNoneMatch != "" { + if info.IfNoneMatch == "*" { + return nil, errtypes.Aborted("etag mismatch, resource exists") + } + for _, tag := range strings.Split(info.IfNoneMatch, ",") { + if tag == oldEtag { + return nil, errtypes.Aborted("etag mismatch") + } + } + } + if !info.IfUnmodifiedSince.IsZero() && oldMtime.After(info.IfUnmodifiedSince) { + return nil, errtypes.Aborted("if-unmodified-since mismatch") + } + + // capture full node xattrs for rollback before any write + oldAttrs, err = fs.lu.MetadataBackend().All(ctx, n) + if err != nil { + return nil, err + } + + // another upload is still processing the node: keep its state, so a rollback + // of this upload can hand the node back to it + supersededMark, superseding := oldAttrs[prefixes.StatusPrefix] + if !fs.o.DisableVersioning || superseding { + timestamp := oldMtime.UTC().Format(time.RFC3339Nano) + versionPath, err := fs.tp.CreateRevision(ctx, n, timestamp) + if err != nil { + return nil, err + } + versionNode = node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+timestamp, fs.lu) + versionCreated = true + if err := os.Chtimes(versionPath, oldMtime, oldMtime); err != nil { + return nil, errtypes.InternalError(fmt.Sprintf("failed to set mtime on version node: %s", err)) + } + if superseding { + revisionAttrs := map[string][]byte{prefixes.StatusPrefix: supersededMark} + if fs.o.DisableVersioning { + // not a version, only kept for a rollback: a live commit removes it + revisionAttrs[uploadSnapshotAttr] = []byte("1") + } + if err := fs.lu.MetadataBackend().SetMultiple(ctx, versionNode, revisionAttrs); err != nil { + return nil, err + } + } + } + + sizeDiff = info.Size - old.Blobsize + } else { + // InitNewNode has cached a node it created + if c, ok := fs.lu.(node.IDCacher); ok && !create { + if err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id") + } + } + sizeDiff = info.Size + } + + attrs := node.Attributes{} + attrs.SetString(prefixes.IDAttr, n.ID) + attrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) + attrs.SetString(prefixes.ParentidAttr, n.ParentID) + attrs.SetString(prefixes.NameAttr, n.Name) + attrs.SetString(prefixes.BlobIDAttr, sessionID) + attrs.SetInt64(prefixes.BlobsizeAttr, info.Size) + attrs[prefixes.ChecksumPrefix+"sha1"] = info.Checksums.SHA1 + attrs[prefixes.ChecksumPrefix+"md5"] = info.Checksums.MD5 + attrs[prefixes.ChecksumPrefix+"adler32"] = info.Checksums.Adler32 + // in the same batch, so marking the node costs no write of its own + attrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+sessionID) + + mtime := time.Now() + if !info.MTime.IsZero() { + mtime = info.MTime + } + if err := fs.lu.TimeManager().OverrideMtime(ctx, n, &attrs, mtime); err != nil { + return nil, errors.Wrap(err, "failed to set mtime") + } + + if err := n.SetXattrsWithContext(ctx, attrs); err != nil { + return nil, errors.Wrap(err, "could not write metadata") + } + + if err := fs.tp.Propagate(ctx, n, sizeDiff); err != nil { + return nil, errors.Wrap(err, "could not propagate size change") + } + committed = true + + result := &storage.PrepareUploadResult{VersionCreated: versionCreated, SizeDiff: sizeDiff} + if !info.NodeExisted { + // read off the disk, not through ListGrants: an uploader who cannot see the + // space root still needs the manager of a project space reported + result.SpaceOwner = n.SpaceOwnerOrManager(ctx) + } + return result, nil +} + +// checkNewFileParent re-checks, for a file PrepareUpload is about to create, what +// TouchFile would have checked: the parent still exists, and the user may still upload +// into it. Either may have changed while the bytes were in flight. +func (fs *Decomposedfs) checkNewFileParent(ctx context.Context, spaceID string, info storage.UploadInfo) error { + parentID := &provider.ResourceId{SpaceId: spaceID, OpaqueId: info.ParentID} + parent, err := fs.lu.NodeFromID(ctx, parentID) + if err != nil { + return errtypes.InternalError(err.Error()) + } + f, _ := storagespace.FormatReference(&provider.Reference{ResourceId: parentID, Path: info.Name}) + if !parent.Exists { + return errtypes.NotFound(f) + } + + rp, err := fs.p.AssemblePermissions(ctx, parent) + switch { + case err != nil: + return err + case !rp.InitiateFileUpload: + if rp.Stat { + return errtypes.PermissionDenied(f) + } + return errtypes.NotFound(f) + } + return nil +} + +// removeRevision deletes a revision PrepareUpload created, with its metadata, while the +// caller holds the node lock (Node.DeleteRevision would try to take it again). Its blob +// is kept: the revision shares it with the node, whose old metadata is being restored. +func removeRevision(ctx context.Context, lu node.PathLookup, revisionNode *node.BaseNode) { + log := appctx.GetLogger(ctx).With().Str("revisionid", revisionNode.ID).Logger() + if err := os.Remove(revisionNode.InternalPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Msg("could not remove revision during rollback") + } + if err := lu.MetadataBackend().Purge(ctx, revisionNode); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Msg("could not purge revision metadata during rollback") + } + if err := os.Remove(lu.MetadataBackend().LockfilePath(revisionNode)); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Msg("could not remove revision lockfile during rollback") + } +} + +// RollbackUpload reverts the node state written by PrepareUpload after a failed or aborted +// postprocessing run. It restores the previous revision and reverts the optimistic size +// propagation. Unlike upstream it never purges an existing node: without a revision (versioning +// disabled) the previous metadata cannot be restored, so it returns an error instead. On posix +// this also keeps the file itself, which still holds the previous content until CommitUpload. +// When the node metadata has become unreadable the ids recorded in info are used to release +// its quota, and to remove the node if the upload created it. +func (fs *Decomposedfs) RollbackUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.RollbackInfo) error { + n, err := fs.lu.NodeFromResource(ctx, ref) + if err != nil { + if !errors.Is(err, node.ErrMissingParentID) { + // the node might only be temporarily unreadable, e.g. because a cache + // is unavailable. Do not treat it as orphaned. + return err + } + // The node metadata is unreadable, so the upload can never finish and its + // quota would stay consumed forever. + return fs.rollbackOrphaned(ctx, ref, sessionID, info, err) + } + if !n.Exists { + return nil // nothing was written yet + } + n.SpaceRoot, err = node.ReadNode(ctx, fs.lu, n.SpaceID, n.SpaceID, "", false, nil, false) + if err != nil { + return err + } + + curProcessingID, err := n.ProcessingID(ctx) + if err != nil { + return fmt.Errorf("RollbackUpload: could not read processing ID: %w", err) + } + if curProcessingID != sessionID { + // a newer upload took over the node. The revision it kept of this upload would + // hand the node back to an upload that is gone. + fs.retireUploadRevision(ctx, n, sessionID, info.NodeExisted) + return nil + } + + sizeDiff := info.SizeDiff + if info.NodeExisted { + restored, purge, err := fs.restoreLatestRevision(ctx, n) + if err != nil { + return err + } + if !restored { + return errtypes.PreconditionFailed("RollbackUpload: no revision to restore for node " + n.ID) + } + if purge { + // the file was created by an upload that was rolled back while this one + // was processing, so nothing is left to keep + if err := n.Purge(ctx); err != nil { + return fmt.Errorf("RollbackUpload: could not purge node: %w", err) + } + sizeDiff += n.Blobsize + } + } else { + // the upload created this node, so undo it. Purge, not trash: the file never + // became visible, and Purge needs no Delete permission + if err := n.Purge(ctx); err != nil { + return fmt.Errorf("RollbackUpload: could not purge node: %w", err) + } + } + + if sizeDiff != 0 { + if err := fs.tp.Propagate(ctx, n, -sizeDiff); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("RollbackUpload: could not revert propagate") + } + } + return nil +} + +// restoreLatestRevision restores the node from the revision PrepareUpload created, its +// content included on posix, and removes the revision. If the revision kept the processing +// mark of an upload this one superseded, the mark is restored too, so that upload can still +// finish. It reports false if the node has no revision, and purge if the revision says the +// file should not exist anymore (see retireUploadRevision); n then describes the restored +// state, with the size the file accounts for. +func (fs *Decomposedfs) restoreLatestRevision(ctx context.Context, n *node.Node) (restored, purge bool, err error) { + unlock, err := fs.lu.MetadataBackend().Lock(n) + if err != nil { + return false, false, err + } + defer func() { + if err := unlock(); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not close lock") + } + }() + + key, err := n.LatestRevisionKey(ctx) + if err != nil || key == "" { + return false, false, err + } + revisionNode := node.NewBaseNode(n.SpaceID, n.ID+node.RevisionIDDelimiter+key, fs.lu) + revisionAttrs, err := fs.lu.MetadataBackend().All(ctx, revisionNode) + if err != nil { + return false, false, err + } + + var mtime time.Time + if v, ok := revisionAttrs[prefixes.MTimeAttr]; ok { + if mtime, err = time.Parse(time.RFC3339Nano, string(v)); err != nil { + return false, false, err + } + } + if err := fs.tp.RestoreRevision(ctx, revisionNode, n, mtime); err != nil { + return false, false, err + } + if mark, ok := revisionAttrs[prefixes.StatusPrefix]; ok { + if err := n.SetXattrsWithContext(ctx, node.Attributes{prefixes.StatusPrefix: mark}); err != nil { + return false, false, err + } + } + if v, ok := revisionAttrs[prefixes.BlobsizeAttr]; ok { + n.Blobsize, _ = strconv.ParseInt(string(v), 10, 64) + } + _, purge = revisionAttrs[purgeOnRestoreAttr] + + removeRevision(ctx, fs.lu, revisionNode) + return true, purge, nil +} + +// purgeOnRestoreAttr marks a revision kept for an upload that created the file and was +// rolled back while a newer upload was processing: restoring it removes the file. +var purgeOnRestoreAttr = prefixes.OcPrefix + "purgeonrestore" + +// uploadSnapshotAttr marks a revision PrepareUpload kept with versioning disabled, only so +// that a rollback can hand the node back to the upload it superseded. It is not a version. +var uploadSnapshotAttr = prefixes.OcPrefix + "uploadsnapshot" + +// retireUploadRevision handles the revision a newer upload kept of the upload sessionID, +// which is being rolled back. If that upload overwrote the file, the revision is dropped: +// the one it kept itself describes the file before it. If it created the file, there is +// nothing before it, so the revision is marked to remove the file should the newer upload +// be rolled back too. +func (fs *Decomposedfs) retireUploadRevision(ctx context.Context, n *node.Node, sessionID string, nodeExisted bool) { + log := appctx.GetLogger(ctx).With().Str("nodeid", n.ID).Str("sessionid", sessionID).Logger() + unlock, err := fs.lu.MetadataBackend().Lock(n) + if err != nil { + log.Error().Err(err).Msg("could not lock node") + return + } + defer func() { + if err := unlock(); err != nil { + log.Error().Err(err).Msg("could not close lock") + } + }() + + revision, revisionUnlock, err := fs.uploadRevision(ctx, n, sessionID) + if err != nil { + log.Error().Err(err).Msg("could not look up the upload's revision") + return + } + if revision == nil { + return + } + defer func() { _ = revisionUnlock() }() + + if nodeExisted { + removeRevision(ctx, fs.lu, &revision.BaseNode) + return + } + if err := revision.SetXattrsWithContext(ctx, node.Attributes{purgeOnRestoreAttr: []byte("1")}); err != nil { + log.Error().Err(err).Msg("could not mark the upload's revision") + } + if err := revision.RemoveXattr(ctx, prefixes.StatusPrefix); err != nil && !metadata.IsAttrUnset(err) { + log.Error().Err(err).Msg("could not remove the processing mark from the upload's revision") + } +} + +// rollbackOrphaned releases the quota of an upload whose node metadata is unreadable, using +// the node and parent ids the upload session recorded. It removes the node only if the upload +// created it and it is still empty. +func (fs *Decomposedfs) rollbackOrphaned(ctx context.Context, ref *provider.Reference, sessionID string, info storage.RollbackInfo, lookupErr error) error { + spaceID := ref.GetResourceId().GetSpaceId() + if info.NodeID == "" || info.ParentID == "" || spaceID == "" { + return fmt.Errorf("RollbackUpload: node lookup failed: %w", lookupErr) + } + log := appctx.GetLogger(ctx).With().Str("sessionid", sessionID).Str("nodeid", info.NodeID).Logger() + log.Info().Err(lookupErr).Msg("node unreadable, rolling back orphaned upload") + + n := node.New(spaceID, info.NodeID, info.ParentID, info.Filename, info.Size, sessionID, + provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu) + spaceRoot, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, "", false, nil, false) + if err != nil { + return fmt.Errorf("RollbackUpload: space root lookup failed: %w", err) + } + n.SpaceRoot = spaceRoot + + if info.SizeDiff != 0 { + // an error keeps the session, so the rollback can be retried + if err := fs.tp.Propagate(ctx, n, -info.SizeDiff); err != nil { + return fmt.Errorf("RollbackUpload: could not revert propagate: %w", err) + } + } + + // Only a node this upload created may be removed. An existing node holds the + // previous version, on posix the user's file itself, so it is left alone. + if info.NodeExisted { + return nil + } + // The node file stays empty until the upload is committed. If it has content, + // something outside of us wrote it, e.g. an editor saving atomically on posix, + // so keep it. + if fi, err := os.Stat(n.InternalPath()); err == nil && fi.Size() > 0 { + log.Info().Str("nodepath", n.InternalPath()).Msg("orphaned node has content, keeping it") + return nil + } + + // the orphaned node cannot be resolved by any other means, remove it together + // with its metadata. A missing node is fine: it may never have been created. + if err := fs.lu.PurgeNode(n); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("RollbackUpload: removing orphaned node failed") + } + if err := fs.lu.MetadataBackend().Purge(ctx, n); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("RollbackUpload: purging orphaned node metadata failed") + } + return nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/async.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/async.go index 5348b620c7..76a930388a 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/async.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/async.go @@ -44,6 +44,8 @@ const ( var _propagationGracePeriod = 3 * time.Minute // AsyncPropagator implements asynchronous treetime & treesize propagation +// +// Deprecated: the async propagator will be removed in a future release. Use the sync propagator instead. type AsyncPropagator struct { treeSizeAccounting bool treeTimeAccounting bool @@ -59,6 +61,8 @@ type Change struct { } // NewAsyncPropagator returns a new AsyncPropagator instance +// +// Deprecated: the async propagator will be removed in a future release. Use the sync propagator instead. func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.AsyncPropagatorOptions, lookup node.PathLookup, log *zerolog.Logger) AsyncPropagator { p := AsyncPropagator{ treeSizeAccounting: treeSizeAccounting, @@ -69,6 +73,7 @@ func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.A } log.Info().Msg("async propagator starting up...") + log.Warn().Msg("the async propagator is deprecated and will be removed in a future release, please switch to the sync propagator") // spawn a goroutine that watches for stale .processing dirs and fixes them go func() { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/metrics.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/metrics.go new file mode 100644 index 0000000000..c60d5cf885 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/metrics.go @@ -0,0 +1,40 @@ +// Copyright 2026 OpenCloud GmbH +// SPDX-License-Identifier: Apache-2.0 + +package propagator + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// the metrics of the propagation, by the internal root of the storage, see metrics +var ( + queuedChanges = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_propagation_changes_total", + Help: "Number of changes to be propagated", + }, []string{"root"}) + directoryUpdates = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_propagation_directory_updates_total", + Help: "Number of directory updates written by the propagation", + }, []string{"root"}) + failures = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "reva_propagation_failures_total", + Help: "Number of failed attempts to apply changes to a directory", + }, []string{"root"}) +) + +// metrics are the metrics of a propagator +type metrics struct { + changes prometheus.Counter + updates prometheus.Counter + failures prometheus.Counter +} + +func newMetrics(internalRoot string) metrics { + return metrics{ + changes: queuedChanges.WithLabelValues(internalRoot), + updates: directoryUpdates.WithLabelValues(internalRoot), + failures: failures.WithLabelValues(internalRoot), + } +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/sync.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/sync.go index 99aed8eefe..ebfb077970 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/sync.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator/sync.go @@ -35,6 +35,7 @@ type SyncPropagator struct { treeSizeAccounting bool treeTimeAccounting bool lookup node.PathLookup + metrics metrics } // NewSyncPropagator returns a new AsyncPropagator instance @@ -43,6 +44,7 @@ func NewSyncPropagator(treeSizeAccounting, treeTimeAccounting bool, lookup node. treeSizeAccounting: treeSizeAccounting, treeTimeAccounting: treeTimeAccounting, lookup: lookup, + metrics: newMetrics(lookup.InternalRoot()), } } @@ -62,6 +64,7 @@ func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff in sublog.Debug().Msg("propagation disabled or nothing to propagate") return nil } + p.metrics.changes.Inc() // is propagation enabled for the parent node? root := n.SpaceRoot @@ -80,6 +83,7 @@ func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff in } if err != nil { + p.metrics.failures.Inc() sublog.Error().Err(err).Msg("error propagating") return err } @@ -195,6 +199,7 @@ func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime t log.Error().Err(err).Msg("Failed to update extend attributes of parent node") return n, true, err } + p.metrics.updates.Inc() return n, false, nil } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go index 9bb49bfc25..4219859527 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/tree.go @@ -34,6 +34,7 @@ import ( "github.com/google/uuid" "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" + "github.com/opencloud-eu/reva/v2/pkg/storage" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" @@ -177,6 +178,9 @@ func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool, } } + if storage.SkipTouchPropagation(ctx) { + return nil + } return t.Propagate(ctx, n, 0) } @@ -485,14 +489,13 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { trashPath := trashNode.InternalPath() err = os.Rename(nodePath, trashPath) if err != nil { - // To roll back changes - // TODO remove symlink - // Roll back changes + _ = os.Remove(trashLink) _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr) return } err = t.lookup.MetadataBackend().Rename(n, trashNode) if err != nil { + _ = os.Remove(trashLink) _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr) _ = os.Rename(trashPath, nodePath) return @@ -506,10 +509,9 @@ func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { // finally remove the entry from the parent dir if err = os.Remove(path); err != nil { - // To roll back changes - // TODO revert the rename - // TODO remove symlink - // Roll back changes + _ = t.lookup.MetadataBackend().Rename(trashNode, n) + _ = os.Rename(trashPath, nodePath) + _ = os.Remove(trashLink) _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr) return } @@ -670,15 +672,31 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) subspan.End() if err != nil { - return unlock, err + // the node file is not ours, so there is nothing of ours to remove + releaseNewNodeLock(ctx, n, unlock) + if errors.Is(err, fs.ErrExist) { + return nil, errtypes.AlreadyExists(n.Name) + } + return nil, err } h.Close() + // from here the node file is ours, but the name may not be: a failure removes the + // node file and leaves the name to whichever upload linked it. Removed before + // unlocking, so nothing can open the node in between. + fail := func(err error) (metadata.UnlockFunc, error) { + if rErr := os.Remove(n.InternalPath()); rErr != nil { + appctx.GetLogger(ctx).Error().Err(rErr).Str("nodeid", n.ID).Msg("could not remove new node file") + } + releaseNewNodeLock(ctx, n, unlock) + return nil, err + } + _, subspan = tracer.Start(ctx, "node.CheckQuota") _, err = node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize) subspan.End() if err != nil { - return unlock, err + return fail(err) } // link child name to parent if it is new @@ -694,15 +712,22 @@ func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (met log.Info().Err(err).Msg("initNewNode: symlink failed") if errors.Is(err, fs.ErrExist) { log.Info().Err(err).Msg("initNewNode: symlink already exists") - return unlock, errtypes.AlreadyExists(n.Name) + return fail(errtypes.AlreadyExists(n.Name)) } - return unlock, errors.Wrap(err, "Decomposedfs: could not symlink child entry") + return fail(errors.Wrap(err, "Decomposedfs: could not symlink child entry")) } log.Info().Msg("initNewNode: symlink created") return unlock, nil } +// releaseNewNodeLock releases the lock of a node InitNewNode failed to create. +func releaseNewNodeLock(ctx context.Context, n *node.Node, unlock metadata.UnlockFunc) { + if err := unlock(); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not release new node lock") + } +} + func (t *Tree) removeNode(ctx context.Context, n *node.Node) error { path := n.InternalPath() logger := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go index 06b24db830..ec25cf28f1 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload.go @@ -20,441 +20,12 @@ package decomposedfs import ( "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/pkg/errors" - tusd "github.com/tus/tusd/v2/pkg/handler" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" ) -// Upload uploads data to the given resource -// TODO Upload (and InitiateUpload) needs a way to receive the expected checksum. -// Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - _, span := tracer.Start(ctx, "Upload") - defer span.End() - up, err := fs.GetUpload(ctx, req.Ref.GetPath()) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload") - } - - session := up.(*upload.DecomposedFsSession) - - ctx = session.Context(ctx) - - if session.Chunk() != "" { // check chunking v1 - p, assembledFile, err := fs.chunkHandler.WriteChunk(session.Chunk(), req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } - if p == "" { - if err = session.Terminate(ctx); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error removing auxiliary files") - } - return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) - } - fd, err := os.Open(assembledFile) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error opening assembled file") - } - defer func() { - _ = fd.Close() - }() - defer func() { - _ = os.RemoveAll(assembledFile) - }() - req.Body = fd - - size, err := session.WriteChunk(ctx, 0, req.Body) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") - } - session.SetSize(size) - } else { - size, err := session.WriteChunk(ctx, 0, req.Body) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") - } - if size != req.Length { - return &provider.ResourceInfo{}, errtypes.PartialContent("Decomposedfs: unexpected end of stream") - } - } - - if err := session.FinishUploadDecomposed(ctx); err != nil { - return &provider.ResourceInfo{}, err - } - - if uff != nil { - uploadRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - } - executant := session.Executant() - uff(session.SpaceOwner(), &executant, uploadRef) - } - - ri := &provider.ResourceInfo{ - // fill with at least fileid, mtime and etag - Id: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - } - - // add etag to metadata - ri.Etag, _ = node.CalculateEtag(session.NodeID(), session.MTime()) - - if !session.MTime().IsZero() { - ri.Mtime = utils.TimeToTS(session.MTime()) - } - - return ri, nil -} - -// InitiateUpload returns upload ids corresponding to different protocols it supports -// TODO read optional content for small files in this request -// TODO InitiateUpload (and Upload) needs a way to receive the expected checksum. Maybe in metadata as 'checksum' => 'sha1 aeosvp45w5xaeoe' = lowercase, space separated? -func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - _, span := tracer.Start(ctx, "InitiateUpload") - defer span.End() - log := appctx.GetLogger(ctx) - - // remember the path from the reference - refpath := ref.GetPath() - var chunk *chunking.ChunkBLOBInfo - var err error - if chunking.IsChunked(refpath) { // check chunking v1 - chunk, err = chunking.GetChunkBLOBInfo(refpath) - if err != nil { - return nil, errtypes.BadRequest(err.Error()) - } - ref.Path = chunk.Path - } - n, err := fs.lu.NodeFromResource(ctx, ref) - switch err.(type) { - case nil: - // ok - case errtypes.IsNotFound: - return nil, errtypes.PreconditionFailed(err.Error()) - default: - return nil, err - } - - // permissions are checked in NewUpload below - - relative, err := fs.lu.Path(ctx, n, node.NoCheck) - // TODO why do we need the path here? - // jfd: it is used later when emitting the UploadReady event ... - // AAAND refPath might be . when accessing with an id / relative reference ... which causes NodeName to become . But then dir will also always be . - // That is why we still have to read the path here: so that the event we emit contains a relative reference with a path relative to the space root. WTF - if err != nil { - return nil, err - } - - lockID, _ := ctxpkg.ContextGetLockID(ctx) - - session := fs.sessionStore.New(ctx) - session.SetMetadata("filename", n.Name) - session.SetStorageValue("NodeName", n.Name) - if chunk != nil { - session.SetStorageValue("Chunk", filepath.Base(refpath)) - } - session.SetMetadata("dir", filepath.Dir(relative)) - session.SetStorageValue("Dir", filepath.Dir(relative)) - session.SetMetadata("lockid", lockID) - - session.SetSize(uploadLength) - session.SetStorageValue("SpaceRoot", n.SpaceRoot.ID) // TODO SpaceRoot -> SpaceID - session.SetStorageValue("SpaceOwnerOrManager", n.SpaceOwnerOrManager(ctx).GetOpaqueId()) // TODO needed for what? - - spaceGID, ok := ctx.Value(CtxKeySpaceGID).(uint32) - if ok { - session.SetStorageValue("SpaceGid", fmt.Sprintf("%d", spaceGID)) - } - - iid, _ := ctxpkg.ContextGetInitiator(ctx) - session.SetMetadata("initiatorid", iid) - - if metadata != nil { - session.SetMetadata("providerID", metadata["providerID"]) - if mtime, ok := metadata["mtime"]; ok { - if mtime != "null" { - session.SetMetadata("mtime", metadata["mtime"]) - } - } - if expiration, ok := metadata["expires"]; ok { - if expiration != "null" { - session.SetMetadata("expires", metadata["expires"]) - } - } - if _, ok := metadata["sizedeferred"]; ok { - session.SetSizeIsDeferred(true) - } - if checksum, ok := metadata["checksum"]; ok { - parts := strings.SplitN(checksum, " ", 2) - if len(parts) != 2 { - return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1", "md5", "adler32": - session.SetMetadata("checksum", checksum) - default: - return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - } - - // only check preconditions if they are not empty // TODO or is this a bad request? - if metadata["if-match"] != "" { - session.SetMetadata("if-match", metadata["if-match"]) - } - if metadata["if-none-match"] != "" { - session.SetMetadata("if-none-match", metadata["if-none-match"]) - } - if metadata["if-unmodified-since"] != "" { - session.SetMetadata("if-unmodified-since", metadata["if-unmodified-since"]) - } - } - - if session.MTime().IsZero() { - session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now())) - } - - log.Debug().Str("uploadid", session.ID()).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename") - - _, err = node.CheckQuota(ctx, n.SpaceRoot, n.Exists, uint64(n.Blobsize), uint64(session.Size())) - if err != nil { - return nil, err - } - - if session.Filename() == "" { - return nil, errors.New("Decomposedfs: missing filename in metadata") - } - if session.Dir() == "" { - return nil, errors.New("Decomposedfs: missing dir in metadata") - } - - // the parent owner will become the new owner - parent, perr := n.Parent(ctx) - if perr != nil { - return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID) - } - - // check permissions - var ( - checkNode *node.Node - path string - ) - if n.Exists { - // check permissions of file to be overwritten - checkNode = n - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }}) - } else { - // check permissions of parent - checkNode = parent - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }, Path: n.Name}) - } - rp, err := fs.p.AssemblePermissions(ctx, checkNode) - switch { - case err != nil: - return nil, err - case !rp.InitiateFileUpload: - return nil, errtypes.PermissionDenied(path) - } - - // are we trying to overwriting a folder with a file? - if n.Exists && n.IsDir(ctx) { - return nil, errtypes.PreconditionFailed("resource is not a file") - } - - // check lock - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - usr := ctxpkg.ContextMustGetUser(ctx) - - // fill future node info - if n.Exists { - if session.HeaderIfNoneMatch() == "*" { - return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", n.ParentID, n.Name, n.ID)) - } - session.SetStorageValue("NodeId", n.ID) - session.SetStorageValue("NodeExists", "true") - } else { - session.SetStorageValue("NodeId", uuid.New().String()) - } - session.SetStorageValue("NodeParentId", n.ParentID) - session.SetExecutant(usr) - session.SetStorageValue("LogLevel", log.GetLevel().String()) - - log.Debug().Interface("session", session).Msg("Decomposedfs: built session info") - - err = fs.um.RunInBaseScope(func() error { - // Create binary file in the upload folder with no content - // It will be used when determining the current offset of an upload - err := session.TouchBin() - if err != nil { - return err - } - - return session.Persist(ctx) - }) - if err != nil { - return nil, err - } - metrics.UploadSessionsInitiated.Inc() - - if uploadLength == 0 { - // Directly finish this upload - err = session.FinishUploadDecomposed(ctx) - if err != nil { - return nil, err - } - } - - return map[string]string{ - "simple": session.ID(), - "tus": session.ID(), - }, nil -} - -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *Decomposedfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - composer.UseConcater(fs) - composer.UseLengthDeferrer(fs) - composer.UseContentServer(fs) -} - -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload - -// NewUpload returns a new tus Upload instance -func (fs *Decomposedfs) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { - return nil, fmt.Errorf("not implemented, use InitiateUpload on the CS3 API to start a new upload") -} - -// AsServableUpload returns a ServableUpload -// which implements the tusd.ServableUpload interface and -func (fs *Decomposedfs) AsServableUpload(u tusd.Upload) tusd.ServableUpload { - return u.(*upload.DecomposedFsSession) -} - -// GetUpload returns the Upload for the given upload id -func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - var ul tusd.Upload - var err error - _ = fs.um.RunInBaseScope(func() error { - ul, err = fs.sessionStore.Get(ctx, id) - return nil - }) - return ul, err -} - -// ResolveUpload stats the resource produced by a finished upload, reading it as the upload's -// executant. The session is reconstructed from the given tus FileInfo rather than the session -// store, so it still resolves after a synchronous upload has cleaned up its session. It returns -// the resource info and the executant context, so the caller can derive identity dependent -// values such as the WebDAV permissions. https://github.com/opencloud-eu/opencloud/issues/2409 -func (fs *Decomposedfs) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) { - session := fs.sessionStore.SessionFromInfo(info) - ctx = session.Context(ctx) - ref := session.Reference() - ri, err := fs.GetMD(ctx, &ref, nil, nil) - return ri, ctx, err -} - -// ListUploadSessions returns the upload sessions for the given filter -func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { - var sessions []*upload.DecomposedFsSession - if filter.ID != nil && *filter.ID != "" { - session, err := fs.sessionStore.Get(ctx, *filter.ID) - if err != nil { - return nil, err - } - sessions = []*upload.DecomposedFsSession{session} - } else { - var err error - sessions, err = fs.sessionStore.List(ctx) - if err != nil { - return nil, err - } - } - filteredSessions := []storage.UploadSession{} - now := time.Now() - for _, session := range sessions { - if filter.Processing != nil && *filter.Processing != session.IsProcessing() { - continue - } - if filter.Expired != nil { - if *filter.Expired { - if now.Before(session.Expires()) { - continue - } - } else { - if now.After(session.Expires()) { - continue - } - } - } - if filter.HasVirus != nil { - sr, _ := session.ScanData() - infected := sr != "" - if *filter.HasVirus != infected { - continue - } - } - filteredSessions = append(filteredSessions, session) - } - return filteredSessions, nil -} - -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (fs *Decomposedfs) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.DecomposedFsSession) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (fs *Decomposedfs) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.DecomposedFsSession) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (fs *Decomposedfs) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.DecomposedFsSession) +// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable. +func (fs *Decomposedfs) IsOrphaned(ctx context.Context, ref *provider.Reference) bool { + _, err := fs.lu.NodeFromResource(ctx, ref) + return err != nil } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/session.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/session.go deleted file mode 100644 index f6eb460ce9..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/session.go +++ /dev/null @@ -1,364 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/google/renameio/v2" - tusd "github.com/tus/tusd/v2/pkg/handler" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -// DecomposedFsSession extends tus upload lifecycle with postprocessing steps. -type DecomposedFsSession struct { - store DecomposedFsStore - // for now, we keep the json files in the uploads folder - info tusd.FileInfo -} - -const ( - SessionStatusUploading = "uploading" - SessionStatusProcessing = "processing" - SessionStatusFailed = "failed" -) - -// Context returns a context with the user, logger and lockid used when initiating the upload session -func (session *DecomposedFsSession) Context(ctx context.Context) context.Context { // restore logger from file info - sub := session.store.log.With().Int("pid", os.Getpid()).Logger() - ctx = appctx.WithLogger(ctx, &sub) - ctx = ctxpkg.ContextSetLockID(ctx, session.lockID()) - ctx = ctxpkg.ContextSetUser(ctx, session.executantUser()) - return ctxpkg.ContextSetInitiator(ctx, session.InitiatorID()) -} - -func (session *DecomposedFsSession) lockID() string { - return session.info.MetaData["lockid"] -} -func (session *DecomposedFsSession) executantUser() *userpb.User { - var o *typespb.Opaque - _ = json.Unmarshal([]byte(session.info.Storage["UserOpaque"]), &o) - return &userpb.User{ - Id: &userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[session.info.Storage["UserType"]]), - Idp: session.info.Storage["Idp"], - OpaqueId: session.info.Storage["UserId"], - }, - Username: session.info.Storage["UserName"], - DisplayName: session.info.Storage["UserDisplayName"], - Opaque: o, - } -} - -// Purge deletes the upload session metadata and written binary data -func (session *DecomposedFsSession) Purge(ctx context.Context) { - _, span := tracer.Start(ctx, "Purge") - defer span.End() - session.Cleanup(true, true, true, true) -} - -// TouchBin creates a file to contain the binary data. It's size will be used to keep track of the tus upload offset. -func (session *DecomposedFsSession) TouchBin() error { - file, err := os.OpenFile(session.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return err - } - return file.Close() -} - -// Persist writes the upload session metadata to disk -// events can update the scan outcome and the finished event might read an empty file because of race conditions -// so we need to lock the file while writing and use atomic writes -func (session *DecomposedFsSession) Persist(ctx context.Context) error { - _, span := tracer.Start(ctx, "Persist") - defer span.End() - infoPath := session.infoPath() - // create folder structure (if needed) - if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil { - return err - } - - var d []byte - d, err := json.Marshal(session.info) - if err != nil { - return err - } - return renameio.WriteFile(infoPath, d, 0600) -} - -// ToFileInfo returns tus compatible FileInfo so the tus handler can access the upload offset -func (session *DecomposedFsSession) ToFileInfo() tusd.FileInfo { - return session.info -} - -// ProviderID returns the provider id -func (session *DecomposedFsSession) ProviderID() string { - return session.info.MetaData["providerID"] -} - -// SpaceID returns the space id -func (session *DecomposedFsSession) SpaceID() string { - return session.info.Storage["SpaceRoot"] -} - -// NodeID returns the node id -func (session *DecomposedFsSession) NodeID() string { - return session.info.Storage["NodeId"] -} - -// NodeParentID returns the nodes parent id -func (session *DecomposedFsSession) NodeParentID() string { - return session.info.Storage["NodeParentId"] -} - -// NodeExists returns wether or not the node existed during InitiateUpload. -// FIXME If two requests try to write the same file they both will store a new -// random node id in the session and try to initialize a new node when -// finishing the upload. The second request will fail with an already exists -// error when trying to create the symlink for the node in the parent directory. -// A node should be created as part of InitiateUpload. When listing a directory -// we can decide if we want to skip the entry, or expose uploed progress -// information. But that is a bigger change and might involve client work. -func (session *DecomposedFsSession) NodeExists() bool { - return session.info.Storage["NodeExists"] == "true" -} - -// HeaderIfMatch returns the if-match header for the upload session -func (session *DecomposedFsSession) HeaderIfMatch() string { - return session.info.MetaData["if-match"] -} - -// HeaderIfNoneMatch returns the if-none-match header for the upload session -func (session *DecomposedFsSession) HeaderIfNoneMatch() string { - return session.info.MetaData["if-none-match"] -} - -// HeaderIfUnmodifiedSince returns the if-unmodified-since header for the upload session -func (session *DecomposedFsSession) HeaderIfUnmodifiedSince() string { - return session.info.MetaData["if-unmodified-since"] -} - -// Node returns the node for the session -func (session *DecomposedFsSession) Node(ctx context.Context) (*node.Node, error) { - return node.ReadNode(ctx, session.store.lu, session.SpaceID(), session.info.Storage["NodeId"], "", false, nil, true) -} - -// ID returns the upload session id -func (session *DecomposedFsSession) ID() string { - return session.info.ID -} - -// Filename returns the name of the node which is not the same as the name af the file being uploaded for legacy chunked uploads -func (session *DecomposedFsSession) Filename() string { - return session.info.Storage["NodeName"] -} - -// Chunk returns the chunk name when a legacy chunked upload was started -func (session *DecomposedFsSession) Chunk() string { - return session.info.Storage["Chunk"] -} - -// SetMetadata is used to fill the upload metadata that will be exposed to the end user -func (session *DecomposedFsSession) SetMetadata(key, value string) { - session.info.MetaData[key] = value -} - -// SetStorageValue is used to set metadata only relevant for the upload session implementation -func (session *DecomposedFsSession) SetStorageValue(key, value string) { - session.info.Storage[key] = value -} - -// SetSize will set the upload size of the underlying tus info. -func (session *DecomposedFsSession) SetSize(size int64) { - session.info.Size = size -} - -// SetSizeIsDeferred is uset to change the SizeIsDeferred property of the underlying tus info. -func (session *DecomposedFsSession) SetSizeIsDeferred(value bool) { - session.info.SizeIsDeferred = value -} - -// Dir returns the directory to which the upload is made -// TODO get rid of Dir(), whoever consumes the reference should be able to deal -// with a relative reference. -// Dir is only used to: -// - fill the Path property when emitting the UploadReady event after -// postprocessing finished. I wonder why the UploadReady contains a finished -// flag ... maybe multiple distinct events would make more sense. -// - build the reference that is passed to the FileUploaded event in the -// UploadFinishedFunc callback passed to the Upload call used for simple -// datatx put requests -// -// AFAICT only search and audit services consume the path. -// - search needs to index from the root anyway. And it only needs the most -// recent path to put it in the index. So it should already be able to deal -// with an id based reference. -// - audit on the other hand needs to log events with the path at the state of -// the event ... so it does need the full path. -// -// I think we can safely determine the path later, right before emitting the -// event. And maybe make it configurable, because only audit needs it, anyway. -func (session *DecomposedFsSession) Dir() string { - return session.info.Storage["Dir"] -} - -// Size returns the upload size -func (session *DecomposedFsSession) Size() int64 { - return session.info.Size -} - -// SizeDiff returns the size diff that was calculated after postprocessing -func (session *DecomposedFsSession) SizeDiff() int64 { - sizeDiff, _ := strconv.ParseInt(session.info.MetaData["sizeDiff"], 10, 64) - return sizeDiff -} - -// Reference returns a reference that can be used to access the uploaded resource -func (session *DecomposedFsSession) Reference() provider.Reference { - return provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.info.MetaData["providerID"], - SpaceId: session.info.Storage["SpaceRoot"], - OpaqueId: session.info.Storage["NodeId"], - }, - // Path is not used - } -} - -// Executant returns the id of the user that initiated the upload session -func (session *DecomposedFsSession) Executant() userpb.UserId { - return userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[session.info.Storage["UserType"]]), - Idp: session.info.Storage["Idp"], - OpaqueId: session.info.Storage["UserId"], - } -} - -// SetExecutant is used to remember the user that initiated the upload session -func (session *DecomposedFsSession) SetExecutant(u *userpb.User) { - session.info.Storage["Idp"] = u.GetId().GetIdp() - session.info.Storage["UserId"] = u.GetId().GetOpaqueId() - session.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type) - session.info.Storage["UserName"] = u.GetUsername() - session.info.Storage["UserDisplayName"] = u.GetDisplayName() - - b, _ := json.Marshal(u.GetOpaque()) - session.info.Storage["UserOpaque"] = string(b) -} - -// Offset returns the current upload offset -func (session *DecomposedFsSession) Offset() int64 { - return session.info.Offset -} - -// SpaceOwner returns the id of the space owner -func (session *DecomposedFsSession) SpaceOwner() *userpb.UserId { - return &userpb.UserId{ - // idp and type do not seem to be consumed and the node currently only stores the user id anyway - OpaqueId: session.info.Storage["SpaceOwnerOrManager"], - } -} - -// Expires returns the time the upload session expires -func (session *DecomposedFsSession) Expires() time.Time { - var t time.Time - if value, ok := session.info.MetaData["expires"]; ok { - t, _ = utils.MTimeToTime(value) - } - return t -} - -// MTime returns the mtime to use for the uploaded file -func (session *DecomposedFsSession) MTime() time.Time { - var t time.Time - if value, ok := session.info.MetaData["mtime"]; ok { - t, _ = utils.MTimeToTime(value) - } - return t -} - -// IsProcessing returns true if the upload is in the processing state, meaning that the upload has finished and postprocessing is still running. -func (session *DecomposedFsSession) IsProcessing() bool { - return session.info.MetaData["status"] == SessionStatusProcessing -} - -// binPath returns the path to the file storing the binary data. -func (session *DecomposedFsSession) binPath() string { - return filepath.Join(session.store.root, "uploads", session.info.ID) -} - -// infoPath returns the path to the .info file storing the file's info. -func (session *DecomposedFsSession) infoPath() string { - return sessionPath(session.store.root, session.info.ID) -} - -// InitiatorID returns the id of the initiating client -func (session *DecomposedFsSession) InitiatorID() string { - return session.info.MetaData["initiatorid"] -} - -// SetScanData sets virus scan data to the upload session -func (session *DecomposedFsSession) SetScanData(result string, date time.Time) { - session.info.MetaData["scanResult"] = result - session.info.MetaData["scanDate"] = date.Format(time.RFC3339) -} - -// ScanData returns the virus scan data -func (session *DecomposedFsSession) ScanData() (string, time.Time) { - date := session.info.MetaData["scanDate"] - if date == "" { - return "", time.Time{} - } - d, _ := time.Parse(time.RFC3339, date) - return session.info.MetaData["scanResult"], d -} - -// SetStatus sets the status of the upload session -func (session *DecomposedFsSession) SetStatus(status, msg string) { - session.info.MetaData["status"] = status - session.info.MetaData["statusMessage"] = msg -} - -// Status returns the status of the upload session -func (session *DecomposedFsSession) Status() string { - return session.info.MetaData["status"] -} - -// StatusMessage returns the status message of the upload session -func (session *DecomposedFsSession) StatusMessage() string { - return session.info.MetaData["statusMessage"] -} - -// sessionPath returns the path to the .info file storing the file's info. -func sessionPath(root, id string) string { - return filepath.Join(root, "uploads", id+".info") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go deleted file mode 100644 index 6de0976f47..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/store.go +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright 2018-2022 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/json" - "fmt" - iofs "io/fs" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "syscall" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/options" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper" - "github.com/pkg/errors" - "github.com/rs/zerolog" - tusd "github.com/tus/tusd/v2/pkg/handler" -) - -var _idRegexp = regexp.MustCompile(".*/([^/]+).info") - -// PermissionsChecker defines an interface for checking permissions on a Node -type PermissionsChecker interface { - AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error) -} - -// DecomposedFsStore manages upload sessions -type DecomposedFsStore struct { - fs storage.FS - lu node.PathLookup - tp node.Tree - um usermapper.Mapper - root string - pub events.Publisher - async bool - tknopts options.TokenOptions - disableVersioning bool - log *zerolog.Logger -} - -// NewSessionStore returns a new DecomposedFsStore -func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async bool, tknopts options.TokenOptions, log *zerolog.Logger) *DecomposedFsStore { - return &DecomposedFsStore{ - fs: fs, - lu: aspects.Lookup, - tp: aspects.Tree, - root: root, - pub: aspects.EventStream, - async: async, - tknopts: tknopts, - disableVersioning: aspects.DisableVersioning, - um: aspects.UserMapper, - log: log, - } -} - -// New returns a new upload session -func (store DecomposedFsStore) New(ctx context.Context) *DecomposedFsSession { - return &DecomposedFsSession{ - store: store, - info: tusd.FileInfo{ - ID: uuid.New().String(), - Storage: map[string]string{ - "Type": "DecomposedFsStore", - }, - MetaData: tusd.MetaData{ - "status": string(SessionStatusUploading), - }, - }, - } -} - -// SessionFromInfo wraps an existing tus FileInfo into an upload session without reading the -// session store. It lets callers use an upload's session, e.g. its reference and -// executant context, after the store entry has already been cleaned up, as happens for a -// synchronous upload in its finish response callback. -func (store DecomposedFsStore) SessionFromInfo(info tusd.FileInfo) *DecomposedFsSession { - return &DecomposedFsSession{ - store: store, - info: info, - } -} - -// List lists all upload sessions -func (store DecomposedFsStore) List(ctx context.Context) ([]*DecomposedFsSession, error) { - uploads := []*DecomposedFsSession{} - infoFiles, err := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) - if err != nil { - return nil, err - } - - for _, info := range infoFiles { - id := strings.TrimSuffix(filepath.Base(info), filepath.Ext(info)) - progress, err := store.Get(ctx, id) - if err != nil { - appctx.GetLogger(ctx).Error().Interface("path", info).Msg("Decomposedfs: could not getUploadSession") - continue - } - - uploads = append(uploads, progress) - } - return uploads, nil -} - -// Get returns the upload session for the given upload id -func (store DecomposedFsStore) Get(ctx context.Context, id string) (*DecomposedFsSession, error) { - sessionPath := sessionPath(store.root, id) - match := _idRegexp.FindStringSubmatch(sessionPath) - if len(match) < 2 { - return nil, fmt.Errorf("invalid upload path") - } - - session := DecomposedFsSession{ - store: store, - info: tusd.FileInfo{}, - } - data, err := os.ReadFile(sessionPath) - if err != nil { - // handle stale NFS file handles that can occur when the file is deleted betwenn the ATTR and FOPEN call of os.ReadFile - if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE { - appctx.GetLogger(ctx).Info().Str("session", id).Err(err).Msg("treating stale file handle as not found") - err = tusd.ErrNotFound - } - if errors.Is(err, iofs.ErrNotExist) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - - if err := json.Unmarshal(data, &session.info); err != nil { - return nil, err - } - - stat, err := os.Stat(session.binPath()) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - - session.info.Offset = stat.Size() - - return &session, nil -} - -// Session is the interface used by the Cleanup call -type Session interface { - ID() string - Node(ctx context.Context) (*node.Node, error) - Context(ctx context.Context) context.Context - Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool) -} - -// CreateNodeForUpload will create the target node for the Upload -// TODO move this to the node package as NodeFromUpload? -// should we in InitiateUpload create the node first? and then the upload? -func (store DecomposedFsStore) CreateNodeForUpload(ctx context.Context, session *DecomposedFsSession, initAttrs node.Attributes) (*node.Node, error) { - ctx, span := tracer.Start(session.Context(ctx), "CreateNodeForUpload") - defer span.End() - n := node.New( - session.SpaceID(), - session.NodeID(), - session.NodeParentID(), - session.Filename(), - session.Size(), - session.ID(), - provider.ResourceType_RESOURCE_TYPE_FILE, - nil, - store.lu, - ) - var err error - n.SpaceRoot, err = node.ReadNode(ctx, store.lu, session.SpaceID(), session.SpaceID(), "", false, nil, false) - if err != nil { - return nil, err - } - - // check lock - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - var unlock metadata.UnlockFunc - if session.NodeExists() { // TODO this is wrong. The node should be created when the upload starts, the revisions should be created independently of the node - // we do not need to propagate a change when a node is created, only when the upload is ready. - // that still creates problems for desktop clients because if another change causes propagation it will detects an empty file - // so the first upload has to point to the first revision with the expected size. The file cannot be downloaded, but it can be overwritten (which will create a new revision and make the node reflect the latest revision) - // any finished postprocessing will not affect the node metadata. - // *thinking* but then initializing an upload will lock the file until the upload has finished. That sucks. - // so we have to check if the node has been created meanwhile (well, only in case the upload does not know the nodeid ... or the NodeExists array that is checked by session.NodeExists()) - // FIXME look at the disk again to see if the file has been created in between, or just try initializing a new node and do the update existing node as a fallback. <- the latter! - - unlock, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size())) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("failed to update existing node") - } - } else { - if c, ok := store.lu.(node.IDCacher); ok { - err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id") - } - } - - unlock, err = store.tp.InitNewNode(ctx, n, uint64(session.Size())) - if err != nil { - appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Err(err).Msg("failed to init new node") - } - session.info.MetaData["sizeDiff"] = strconv.FormatInt(session.Size(), 10) - } - defer func() { - if unlock == nil { - appctx.GetLogger(ctx).Info().Msg("did not get a unlockfunc, not unlocking") - return - } - - if err := unlock(); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock") - } - }() - if err != nil { - return nil, err - } - - // overwrite technical information - initAttrs.SetString(prefixes.IDAttr, n.ID) - initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) - initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) - initAttrs.SetString(prefixes.NameAttr, n.Name) - initAttrs.SetString(prefixes.BlobIDAttr, n.BlobID) - initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) - initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+session.ID()) - - // set mtime on the new node - mtime := time.Now() - if !session.MTime().IsZero() { - // overwrite mtime if requested - mtime = session.MTime() - } - err = store.lu.TimeManager().OverrideMtime(ctx, n, &initAttrs, mtime) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") - } - - // update node metadata with new blobid etc - err = n.SetXattrsWithContext(ctx, initAttrs) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: could not write metadata") - } - - err = store.um.RunInBaseScope(func() error { - return session.Persist(ctx) - }) - if err != nil { - return nil, err - } - - return n, nil -} - -func (store DecomposedFsStore) updateExistingNode(ctx context.Context, session *DecomposedFsSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) { - _, span := tracer.Start(ctx, "updateExistingNode") - defer span.End() - - // write lock existing node before reading any metadata - unlock, err := store.lu.MetadataBackend().Lock(n) - if err != nil { - return nil, err - } - - old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, "", false, nil, false) - if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil { - return unlock, err - } - - oldNodeMtime, err := old.GetMTime(ctx) - if err != nil { - return unlock, err - } - oldNodeEtag, err := node.CalculateEtag(old.ID, oldNodeMtime) - if err != nil { - return unlock, err - } - - // When the if-match header was set we need to check if the - // etag still matches before finishing the upload. - if session.HeaderIfMatch() != "" && session.HeaderIfMatch() != oldNodeEtag { - return unlock, errtypes.Aborted("etag mismatch") - } - - // When the if-none-match header was set we need to check if any of the - // etags matches before finishing the upload. - if session.HeaderIfNoneMatch() != "" { - if session.HeaderIfNoneMatch() == "*" { - return unlock, errtypes.Aborted("etag mismatch, resource exists") - } - for _, ifNoneMatchTag := range strings.Split(session.HeaderIfNoneMatch(), ",") { - if ifNoneMatchTag == oldNodeEtag { - return unlock, errtypes.Aborted("etag mismatch") - } - } - } - - // When the if-unmodified-since header was set we need to check if the - // etag still matches before finishing the upload. - if session.HeaderIfUnmodifiedSince() != "" { - ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, session.HeaderIfUnmodifiedSince()) - if err != nil { - return unlock, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) - } - - if oldNodeMtime.After(ifUnmodifiedSince) { - return unlock, errtypes.Aborted("if-unmodified-since mismatch") - } - } - - if !store.disableVersioning { - span.AddEvent("CreateVersion") - timestamp := oldNodeMtime.UTC().Format(time.RFC3339Nano) - versionID := n.ID + node.RevisionIDDelimiter + timestamp - versionPath, err := session.store.tp.CreateRevision(ctx, n, timestamp) - if err != nil { - if !errors.Is(err, os.ErrExist) { - return unlock, err - } - - // a revision with this mtime does already exist. - // If the blobs are the same we can just delete the old one - versionNode := node.NewBaseNode(n.SpaceID, versionID, session.store.lu) - if err := validateChecksums(ctx, old, session, versionNode); err != nil { - return unlock, err - } - - // delete old blob - bID, _, err := session.store.lu.ReadBlobIDAndSizeAttr(ctx, versionNode, nil) - if err != nil { - return unlock, err - } - if err := session.store.tp.DeleteBlob(&node.Node{BaseNode: node.BaseNode{SpaceID: n.SpaceID}, BlobID: bID}); err != nil { - return unlock, err - } - - // clean revision file - if versionPath, err = session.store.tp.CreateRevision(ctx, n, timestamp); err != nil { - return unlock, err - } - } - - session.info.MetaData["versionID"] = versionID - // keep mtime from previous version - span.AddEvent("os.Chtimes") - if err := os.Chtimes(versionPath, oldNodeMtime, oldNodeMtime); err != nil { - return unlock, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err)) - } - } - - session.info.MetaData["sizeDiff"] = strconv.FormatInt((int64(fsize) - old.Blobsize), 10) - - return unlock, nil -} - -func validateChecksums(ctx context.Context, n *node.Node, session *DecomposedFsSession, versionNode metadata.MetadataNode) error { - for _, t := range []string{"md5", "sha1", "adler32"} { - key := prefixes.ChecksumPrefix + t - - checksum, err := n.Xattr(ctx, key) - if err != nil { - return err - } - - revisionChecksum, err := session.store.lu.MetadataBackend().Get(ctx, versionNode, key) - if err != nil { - return err - } - - if string(checksum) == "" || string(revisionChecksum) == "" { - return errors.New("checksum not found") - } - - if string(checksum) != string(revisionChecksum) { - return errors.New("checksum mismatch") - } - } - - return nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go deleted file mode 100644 index 813c2e57f4..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go +++ /dev/null @@ -1,605 +0,0 @@ -// Copyright 2018-2022 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/hex" - "fmt" - "hash" - "io" - "io/fs" - "net/http" - "os" - "strconv" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/golang-jwt/jwt/v5" - "github.com/pkg/errors" - tusd "github.com/tus/tusd/v2/pkg/handler" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/utils/download" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -var ( - tracer trace.Tracer - defaultFilePerm = os.FileMode(0664) -) - -func init() { - tracer = otel.Tracer("github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload") -} - -// WriteChunk writes the stream from the reader to the given offset of the upload -func (session *DecomposedFsSession) WriteChunk(ctx context.Context, _ int64, src io.Reader) (int64, error) { - ctx, span := tracer.Start(session.Context(ctx), "WriteChunk") - defer span.End() - _, subspan := tracer.Start(ctx, "os.OpenFile") - file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - subspan.End() - - log := appctx.GetLogger(ctx) - if err != nil { - log.Error().Err(err).Msg("WriteChunk: error opening upload file") - return 0, err - } - defer func() { - // sync the written chunk to disk. This ensures that the upload can be resumed, - // and helps to prevent issues with filesystem/journal freezes at the end of the upload - // when committing a large fsync operation on slow disks. - _ = disk.Fdatasync(file) - _ = file.Close() - }() - - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... - _, subspan = tracer.Start(ctx, "io.Copy") - n, err := io.Copy(file, src) - subspan.End() - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the decompsedfs driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil && err != io.ErrUnexpectedEOF { - log.Error().Err(err).Msg("WriteChunk: error copying data to upload file") - return n, err - } - - // update upload.Session.Offset so subsequent code flow can use it. - // No need to persist the session as the offset is determined by stating the blob in the GetUpload / ReadSession codepath. - // The session offset is written to disk in FinishUpload - session.info.Offset += n - return n, nil -} - -// GetInfo returns the FileInfo -func (session *DecomposedFsSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { - return session.ToFileInfo(), nil -} - -// GetReader returns an io.Reader for the upload -func (session *DecomposedFsSession) GetReader(ctx context.Context) (io.ReadCloser, error) { - _, span := tracer.Start(session.Context(ctx), "GetReader") - defer span.End() - return os.Open(session.binPath()) -} - -// FinishUpload finishes an upload and moves the file to the internal destination -// implements tusd.DataStore interface -// returns tusd errors -func (session *DecomposedFsSession) FinishUpload(ctx context.Context) error { - err := session.FinishUploadDecomposed(ctx) - - if err != nil { - // this is part of the tusd integration and we might be able to - // log the error in another place - log := appctx.GetLogger(ctx) - log.Error().Err(err).Msg("failed to finish upload") - } - - // we need to return a tusd error here to make the tusd handler return the correct status code - switch err.(type) { - case errtypes.AlreadyExists: - return tusd.NewError("ERR_ALREADY_EXISTS", err.Error(), http.StatusConflict) - case errtypes.Aborted: - return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed) - case errtypes.InsufficientStorage: - return tusd.NewError("ERR_INSUFFICIENT_STORAGE", err.Error(), http.StatusInsufficientStorage) - default: - return err - } -} - -// FinishUploadDecomposed finishes an upload and moves the file to the internal destination -// retures errtypes errors -func (session *DecomposedFsSession) FinishUploadDecomposed(ctx context.Context) error { - ctx, span := tracer.Start(session.Context(ctx), "FinishUpload") - defer span.End() - log := appctx.GetLogger(ctx) - - ctx = ctxpkg.ContextSetInitiator(ctx, session.InitiatorID()) - - ctx = context.WithoutCancel(ctx) // Do not cancel the finish process, we unconditionally want to complete the upload. - sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) - if err != nil { - return err - } - - // compare if they match the sent checksum - // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - if session.info.MetaData["checksum"] != "" { - var err error - parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = checkHash(parts[1], sha1h) - case "md5": - err = checkHash(parts[1], md5h) - case "adler32": - err = checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if err != nil { - session.Cleanup(false, true, true, false) - return err - } - } - - // update checksums - attrs := node.Attributes{ - prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), - prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), - prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), - } - - // At this point we scope by the space to create the final file in the final location - if session.store.um != nil && session.info.Storage["SpaceGid"] != "" { - gid, err := strconv.Atoi(session.info.Storage["SpaceGid"]) - if err != nil { - return errors.Wrap(err, "failed to parse space gid") - } - - unscope, err := session.store.um.ScopeUserByIds(-1, gid) - if err != nil { - return errors.Wrap(err, "failed to scope user") - } - if unscope != nil { - defer func() { _ = unscope() }() - } - } - - n, err := session.store.CreateNodeForUpload(ctx, session, attrs) - if err != nil { - session.SetStatus(SessionStatusFailed, err.Error()) - if perr := session.Persist(ctx); perr != nil { - log.Error().Err(perr).Msg("failed to persist upload session after setting status to failed") - } - return err - } - - session.SetStatus(SessionStatusProcessing, "") - if err = session.Persist(ctx); err != nil { - log.Error().Err(err).Msg("failed to persist upload session after setting status to processing") - } - - // increase the processing counter for every started processing - // will be decreased in Cleanup() - metrics.UploadProcessing.Inc() - metrics.UploadSessionsBytesReceived.Inc() - - if session.store.pub != nil && session.info.Size > 0 { - u, _ := ctxpkg.ContextGetUser(ctx) - s, err := session.URL(ctx) - if err != nil { - return err - } - - var iu *userpb.User - if utils.ExistsInOpaque(u.Opaque, "impersonating-user") { - iu = &userpb.User{} - if err := utils.ReadJSONFromOpaque(u.Opaque, "impersonating-user", iu); err != nil { - return err - } - } - - if err := events.Publish(ctx, session.store.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(session.Context(ctx)), - ExecutingUser: u, - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - ImpersonatingUser: iu, - }); err != nil { - return err - } - } - - // if the upload is synchronous or the upload is empty, finalize it now - // for 0-byte uploads we take a shortcut and finalize isn't called elsewhere - if !session.store.async || session.info.Size == 0 { - // handle postprocessing synchronously - err = session.Finalize(ctx) - session.Cleanup(err != nil, err == nil, true, true) - if err != nil { - log.Error().Err(err).Msg("failed to upload") - return err - } - metrics.UploadSessionsFinalized.Inc() - } - - return session.store.tp.Propagate(ctx, n, session.SizeDiff()) -} - -// Terminate terminates the upload -func (session *DecomposedFsSession) Terminate(_ context.Context) error { - session.Cleanup(true, true, true, true) - return nil -} - -// DeclareLength updates the upload length information -func (session *DecomposedFsSession) DeclareLength(ctx context.Context, length int64) error { - session.info.Size = length - session.info.SizeIsDeferred = false - return session.store.um.RunInBaseScope(func() error { - return session.Persist(session.Context(ctx)) - }) -} - -// ConcatUploads concatenates multiple uploads -func (session *DecomposedFsSession) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) { - file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer func() { - _ = file.Close() - }() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*DecomposedFsSession) - - src, err := os.Open(fileUpload.binPath()) - if err != nil { - return err - } - defer func() { - _ = src.Close() - }() - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - - return -} - -// Finalize finalizes the upload (eg moves the file to the internal destination) -func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) { - ctx, span := tracer.Start(session.Context(ctx), "Finalize") - defer span.End() - - n := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(), - provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu) - - var ( - spaceRoot *node.Node - ) - switch spaceRoot, err = session.store.lu.NodeFromSpaceID(ctx, session.SpaceID()); { - case err != nil: - return fmt.Errorf("failed to get space root for space id %s: %v", session.SpaceID(), err) - case spaceRoot == nil: - return fmt.Errorf("space root for space id %s not found", session.SpaceID()) - case spaceRoot.InternalPath() == "": - return fmt.Errorf("space root for space id %s has no valid internal path", session.SpaceID()) - default: - n.SpaceRoot = spaceRoot - } - - // lock the node before reading its metadata and writing the blob - unlock, err := session.store.lu.MetadataBackend().Lock(n) - if err != nil { - return err - } - defer func() { - _ = unlock() - }() - - // Read the node attributes while holding the lock acquired above. - attribs, err := n.Xattrs(ctx) - if err != nil { - return err - } - status := attribs.String(prefixes.StatusPrefix) - isProcessing := strings.HasPrefix(status, node.ProcessingStatus) - processingID := strings.TrimPrefix(status, node.ProcessingStatus) - - // another upload on this node is in progress or has finished since we started - if !isProcessing || processingID != session.ID() { - // The node's current content is already this upload's content (e.g. a - // newer upload was reverted in the meantime) - no revision is needed, - // upload the blob to the node itself. - if attribs.String(prefixes.BlobIDAttr) == session.ID() { - appctx.GetLogger(ctx).Debug().Str("nodepath", n.InternalPath()).Msg("node already contains this upload's content, no revision needed") - } else { - versionID := n.ID + node.RevisionIDDelimiter + session.MTime().UTC().Format(time.RFC3339Nano) - // There should be a revision node (created by the other upload that finished before us), read it and upload our blob there. - existingRevisionNode, revisionNodeUnlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, spaceRoot, false) - if err != nil || !existingRevisionNode.Exists { - // The revision node has not been created. Likely because the file on disk was modified externally and re-assilimated (watchfs == true) - // Let's create the revision node now and upload the blob to it. - n, revisionNodeUnlock, err = session.createRevisionNodeForUpload(ctx, n, session.MTime().UTC().Format(time.RFC3339Nano)) - if err != nil { - appctx.GetLogger(ctx).Debug().Err(err).Str("versionID", session.MTime().UTC().Format(time.RFC3339Nano)).Msg("failed to create revision node for upload finalization") - return err - } - } else { - n = existingRevisionNode - } - appctx.GetLogger(ctx).Debug().Str("new nodepath", n.InternalPath()).Msg("uploading to revision node, that was created for us by another upload") - defer func() { _ = revisionNodeUnlock() }() - } - } - - // upload the data to the blobstore - _, subspan := tracer.Start(ctx, "WriteBlob") - err = session.store.tp.WriteBlob(n, session.binPath()) - subspan.End() - if err != nil { - return errors.Wrap(err, "failed to upload file to blobstore") - } - - return nil -} - -func (session *DecomposedFsSession) createRevisionNodeForUpload(ctx context.Context, baseNode *node.Node, rev string) (*node.Node, func() error, error) { - versionID := baseNode.ID + node.RevisionIDDelimiter + rev - log := appctx.GetLogger(ctx) - _, err := session.store.tp.CreateRevision(ctx, baseNode, rev) - if err != nil { - log.Error().Err(err).Str("versionID", versionID).Msg("failed to create revision node for upload") - return nil, nil, err - } - - // FIXME: We already calculated the checksums in FinishUpload, we should maybe pass them via the session instead of recalculating them here - sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) - if err != nil { - return nil, nil, err - } - - // update checksums - attrs := node.Attributes{ - prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), - prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), - prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), - } - revisionNode, unlock, err := node.LockAndReadNode(ctx, session.store.lu, session.SpaceID(), versionID, "", false, baseNode.SpaceRoot, false) - if err == nil { - mtime := session.MTime() - attrs.SetString(prefixes.BlobIDAttr, session.ID()) - attrs.SetInt64(prefixes.BlobsizeAttr, session.Size()) - revisionNode.BlobID = session.ID() - revisionNode.Blobsize = session.Size() - - err = session.store.lu.TimeManager().OverrideMtime(ctx, revisionNode, &attrs, mtime) - if err != nil { - _ = unlock() - return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") - } - err = revisionNode.SetXattrsWithContext(ctx, attrs) - if err != nil { - _ = unlock() - return nil, nil, errors.Wrap(err, "Decomposedfs: failed to set node attributes") - } - } - return revisionNode, unlock, err -} - -func checkHash(expected string, h hash.Hash) error { - shash := hex.EncodeToString(h.Sum(nil)) - if expected != shash { - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, shash)) - } - return nil -} - -// cleanup cleans up after the upload is finished -func (session *DecomposedFsSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) { - ctx := session.Context(context.Background()) - sublog := session.store.log.With().Str("cleanup sessionid", session.ID()).Bool("revertNodeMetadata", revertNodeMetadata).Bool("cleanBin", cleanBin). - Bool("cleanInfo", cleanInfo).Bool("unmarkPostprocessing", unmarkPostprocessing).Logger() - - if revertNodeMetadata { - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("reading node for session failed") - } else { - versionID := strings.TrimPrefix(session.info.MetaData["versionID"], n.ID+node.RevisionIDDelimiter) - if session.NodeExists() && versionID != "" { - sublog.Debug().Str("nodepath", n.InternalPath()).Str("versionID", versionID).Msg("restoring revision") - if err := n.RevertUpload(ctx, versionID); err != nil { - sublog.Error().Err(err).Str("versionID", versionID).Msg("reverting node metadata failed") - return - } - } else { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - sublog.Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("uploadid", session.ID()).Msg("reading processingid for session failed") - } - if latestSession == session.ID() { - // actually delete the node - if err := n.Purge(ctx); err != nil { - sublog.Error().Err(err).Str("nodepath", n.InternalPath()).Msg("purging node failed") - } - } - // FIXME else if the upload has become a revision, delete the revision, or if it is the last one, delete the node - } - } - } - - if cleanBin { - if err := os.Remove(session.binPath()); err != nil && !errors.Is(err, fs.ErrNotExist) { - appctx.GetLogger(ctx).Error().Str("path", session.binPath()).Err(err).Msg("removing upload failed") - } - } - - if cleanInfo { - if err := os.Remove(session.infoPath()); err != nil { - sublog.Error().Err(err).Msg("removing upload info failed") - return - } - } - - if unmarkPostprocessing { - n, err := session.Node(ctx) - if err != nil { - sublog.Info().Err(err).Msg("could not read node") - return - } - // FIXME: after cleanup the node might already be deleted ... - if n != nil { // node can be nil when there was an error before it was created (eg. checksum-mismatch) - if err := n.UnmarkProcessing(ctx, session.ID()); err != nil { - sublog.Info().Err(err).Str("path", n.InternalPath()).Msg("unmarking processing failed") - } - } - } -} - -// URL returns a url to download an upload -func (session *DecomposedFsSession) URL(_ context.Context) (string, error) { - - u := joinurl(session.store.tknopts.DownloadEndpoint, "tus/", session.ID()) - if session.store.tknopts.DataGatewayEndpoint == "" { - return u, nil - } - - // we need to create a token - type transferClaims struct { - jwt.RegisteredClaims - Target string `json:"target"` - } - - ttl := time.Duration(session.store.tknopts.TransferExpires) * time.Second - claims := transferClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), - Audience: jwt.ClaimStrings{"reva"}, - IssuedAt: jwt.NewNumericDate(time.Now()), - }, - Target: u, - } - - t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) - - tkn, err := t.SignedString([]byte(session.store.tknopts.TransferSharedSecret)) - if err != nil { - return "", errors.Wrapf(err, "error signing token with claims %+v", claims) - } - - return joinurl(session.store.tknopts.DataGatewayEndpoint, tkn), nil -} - -// ServeContent serves the content of the upload and implements the http.ServeContent interface needed by tusd, -// it is used by the tusd handler to serve the content of the upload and supports range requests -func (session *DecomposedFsSession) ServeContent(ctx context.Context, w http.ResponseWriter, req *http.Request) error { - _, span := tracer.Start(session.Context(ctx), "ServeContent") - defer span.End() - - f, err := os.Open(session.binPath()) - if err != nil { - return err - } - defer func() { - _ = f.Close() - }() - - info, err := f.Stat() - if err != nil { - return err - } - - var r io.Reader = f - if err := func() error { - if req.Header.Get("Range") == "" { - return nil - } - - ranges, err := download.ParseRange(req.Header.Get("Range"), info.Size()) - switch { - case len(ranges) == 0: - fallthrough - case errors.Is(err, download.ErrInvalidRange): - // ignore invalid range and return the whole file - return nil - case err != nil: - return err - } - - r = io.NewSectionReader(f, ranges[0].Start, ranges[0].Length) - w.WriteHeader(http.StatusPartialContent) - w.Header().Set("Content-Range", ranges[0].ContentRange(info.Size())) - return nil - }(); err != nil { - return err - } - - if _, err := io.Copy(w, r); err != nil { - return err - } - - return nil -} - -// replace with url.JoinPath after switching to go1.19 -func joinurl(paths ...string) string { - var s strings.Builder - l := len(paths) - for i, p := range paths { - s.WriteString(p) - if !strings.HasSuffix(p, "/") && i != l-1 { - s.WriteString("/") - } - } - - return s.String() -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/storage.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/storage.go index 7a44931400..fc6fc3edc0 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/storage.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/storage.go @@ -22,11 +22,11 @@ import ( "context" "io" "net/url" + "time" user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" registry "github.com/cs3org/go-cs3apis/cs3/storage/registry/v1beta1" - tusd "github.com/tus/tusd/v2/pkg/handler" ) // MoveResult carries the resolved references returned by a successful Move. @@ -36,6 +36,44 @@ type MoveResult struct { NewReference *provider.Reference } +type UploadChecksums struct { + SHA1 []byte + MD5 []byte + Adler32 []byte +} + +type UploadInfo struct { + NodeExisted bool // true when the target node existed before the upload started + Size int64 + MTime time.Time + Checksums UploadChecksums + IfMatch string + IfNoneMatch string + IfUnmodifiedSince time.Time + // Where a new file goes, for a driver whose PrepareUpload creates it. Empty + // leaves a missing node NotFound. + ParentID string + Name string +} + +type PrepareUploadResult struct { + VersionCreated bool + SizeDiff int64 + SpaceOwner *user.UserId // the space owner or a manager, for a new node; nil keeps the one already known +} + +// RollbackInfo carries what a driver needs to undo PrepareUpload. NodeID and +// ParentID come from the upload session rather than the node, so a rollback can +// still release the quota of a node whose own metadata has become unreadable. +type RollbackInfo struct { + NodeExisted bool // true when the target node had a prior version; drivers with nothing to undo for new nodes may no-op + SizeDiff int64 + NodeID string + ParentID string + Filename string + Size int64 +} + // FS is the interface to implement access to the storage. type FS interface { // Minimal set for a readonly storage driver @@ -69,7 +107,7 @@ type FS interface { // CreateDir creates a resource of type container CreateDir(ctx context.Context, ref *provider.Reference) error // TouchFile sets the mtime of a resource, creating an empty file if it does not exist - // FIXME(OCISDEV-900) remove markprocessing bool: coordinator calls MarkProcessing(true) explicitly after TouchFile + // FIXME(OCISDEV-900) remove markprocessing bool: PrepareUpload marks the node the coordinator touched // FIXME the mtime should either be a time.Time or a CS3 Timestamp, not a string TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error // Delete deletes a resource. @@ -79,15 +117,24 @@ type FS interface { // space owner for event publication. Drivers without this metadata may return // a nil result on success. Move(ctx context.Context, oldRef, newRef *provider.Reference) (*MoveResult, error) - // InitiateUpload returns a list of protocols with urls that can be used to append bytes to a new upload session - InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) - // Upload creates or updates a resource of type file with a new revision - Upload(ctx context.Context, req UploadRequest, uploadFunc UploadFinishedFunc) (*provider.ResourceInfo, error) // MarkProcessing toggles a processing flag on the resource. MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error // CommitUpload writes the staged bytes from source to the resource at ref. - // Caller owns source.Body and must close it after CommitUpload returns. + // source carries a Body, a Path or both. Caller owns source.Body and must close it + // after CommitUpload returns. CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source UploadSource) error + // PrepareUpload is called after all bytes are received and before postprocessing begins. + // Implementations may lock the target node, snapshot the previous version, write new metadata, + // mark the node as processing for sessionID, and propagate size changes. Drivers that do not + // require any of these steps may return immediately. + PrepareUpload(ctx context.Context, ref *provider.Reference, sessionID string, info UploadInfo) (*PrepareUploadResult, error) + // RollbackUpload reverts node state after a failed or aborted postprocessing run. + // It is the inverse of PrepareUpload: restores previous metadata and reverts the optimistic + // size propagation. The caller (coordinator) is responsible for unmarking the processing flag + // and deleting the upload session files. Drivers that performed no work in PrepareUpload may return nil. + // Callers must keep the session on a returned error: the rollback is retryable and info + // carries state the driver cannot recover once the session files are gone. + RollbackUpload(ctx context.Context, ref *provider.Reference, sessionID string, info RollbackInfo) error // Revisions @@ -176,15 +223,57 @@ type SpaceDisabledListableFS interface { type UploadSource struct { Body io.ReadCloser Length int64 + // Path is the staged file, if the caller has one. Drivers may move it into + // place instead of copying Body; the file is gone after a successful commit. + Path string + + // ScanResult is the antivirus verdict: empty means clean. + ScanResult string + // ScanDate is zero when the upload was not scanned. + ScanDate time.Time } // UnscopeFunc is a function that unscopes a user type UnscopeFunc func() -// Composable is the interface that a struct needs to implement -// to be composable, so that it can support the TUS methods -type ComposableFS interface { - UseIn(composer *tusd.StoreComposer) +// CapabilitiesOpaqueKey is the StorageSpace/ProviderInfo Opaque key under which +// a provider's Capabilities are carried as JSON up to the OCS capabilities handler. +const CapabilitiesOpaqueKey = "storage-capabilities" + +// Capabilities is a storage driver's self-declaration of the write-shaped +// actions it supports. A false key means unsupported: no merge, no inheritance. +type Capabilities struct { + Upload bool + CreateContainer bool + Delete bool + Move bool + Versioning bool + Trash bool + Locking bool + Sharing bool + ArbitraryMetadata bool +} + +// FullCapabilities is the default for drivers that do not implement +// CapabilityProvider, so existing drivers keep their current behavior. +func FullCapabilities() Capabilities { + return Capabilities{ + Upload: true, + CreateContainer: true, + Delete: true, + Move: true, + Versioning: true, + Trash: true, + Locking: true, + Sharing: true, + ArbitraryMetadata: true, + } +} + +// CapabilityProvider is an optional interface a storage.FS may implement to +// declare its capabilities. +type CapabilityProvider interface { + Capabilities(ctx context.Context) Capabilities } // Registry is the interface that storage registries implement diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/uploads.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/uploads.go index 880739ef9e..01fd08e83f 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/uploads.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/uploads.go @@ -20,7 +20,6 @@ package storage import ( "context" - "io" "time" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -28,28 +27,17 @@ import ( tusd "github.com/tus/tusd/v2/pkg/handler" ) -// UploadFinishedFunc is a callback function used in storage drivers to indicate that an upload has finished -type UploadFinishedFunc func(spaceOwner, executant *userpb.UserId, ref *provider.Reference) +type skipTouchPropagationKey struct{} -// UploadRequest us used in FS.Upload() to carry required upload metadata -type UploadRequest struct { - Ref *provider.Reference - Body io.ReadCloser - Length int64 +// ContextSkipTouchPropagation tells TouchFile not to propagate. +func ContextSkipTouchPropagation(ctx context.Context) context.Context { + return context.WithValue(ctx, skipTouchPropagationKey{}, true) } -// UploadsManager defines the interface for storage drivers that allow for managing uploads -// -// Deprecated: No longer used. Storage drivers should implement the UploadSessionLister. -type UploadsManager interface { - ListUploads() ([]tusd.FileInfo, error) - PurgeExpiredUploads(chan<- tusd.FileInfo) error -} - -// UploadSessionLister defines the interface for FS implementations that allow listing and purging upload sessions -type UploadSessionLister interface { - // ListUploadSessions returns the upload sessions matching the given filter - ListUploadSessions(ctx context.Context, filter UploadSessionFilter) ([]UploadSession, error) +// SkipTouchPropagation reports whether TouchFile should skip propagation. +func SkipTouchPropagation(ctx context.Context) bool { + skip, _ := ctx.Value(skipTouchPropagationKey{}).(bool) + return skip } // UploadSessionResolver defines the interface for FS implementations that can resolve a finished @@ -66,6 +54,20 @@ type UploadSessionResolver interface { ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) } +// OrphanChecker defines the interface for FS implementations that can resolve a resource's metadata. +type OrphanChecker interface { + // IsOrphaned reports whether the referenced resource exists but its metadata is unreadable. + IsOrphaned(ctx context.Context, ref *provider.Reference) bool +} + +// NodeCreator defines the interface for FS implementations whose PrepareUpload +// creates a new file's node from UploadInfo.ParentID and Name, so an upload need +// not TouchFile it first. +type NodeCreator interface { + // PrepareCreatesNode reports whether PrepareUpload creates a missing node. + PrepareCreatesNode() bool +} + // UploadSession is the interface that storage drivers need to return whan listing upload sessions. type UploadSession interface { // ID returns the upload id @@ -107,4 +109,8 @@ type UploadSessionFilter struct { Processing *bool Expired *bool HasVirus *bool + // Orphaned filters sessions by whether their target node can still be + // resolved. Evaluating it requires reading the node metadata of every + // session, so it is only evaluated when set. + Orphaned *bool } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking/chunking.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking/chunking.go index 80513937a9..228ee96510 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking/chunking.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking/chunking.go @@ -201,6 +201,45 @@ func (c *ChunkHandler) saveChunk(path string, r io.ReadCloser) (bool, string, er return true, assembledFileName, nil } +// Assemble saves an intermediate chunk and, once all chunks have arrived, +// assembles them into a single temp file. Returns (reader, size, true, nil) +// when the transfer is complete. Returns (nil, 0, false, nil) when more +// chunks are still expected (partial transfer) +// the returned reader removes the temp file on Close +func (c *ChunkHandler) Assemble(chunkName string, body io.ReadCloser) (io.ReadCloser, int64, bool, error) { + origPath, assembledPath, err := c.WriteChunk(chunkName, body) + if err != nil { + return nil, 0, false, err + } + if origPath == "" { + return nil, 0, false, nil // partial + } + f, err := os.Open(assembledPath) + if err != nil { + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + fi, err := f.Stat() + if err != nil { + _ = f.Close() + _ = os.RemoveAll(assembledPath) + return nil, 0, false, err + } + return &assembledFile{File: f, path: assembledPath}, fi.Size(), true, nil +} + +// assembledFile wraps an *os.File and removes the temp file on Close. +type assembledFile struct { + *os.File + path string +} + +func (a *assembledFile) Close() error { + err := a.File.Close() + _ = os.RemoveAll(a.path) + return err +} + // WriteChunk saves an intermediate chunk temporarily and assembles all chunks // once the final one is received. func (c *ChunkHandler) WriteChunk(fn string, r io.ReadCloser) (string, string, error) { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects/aspects.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects/aspects.go deleted file mode 100644 index 7f6d91a227..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects/aspects.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2018-2024 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package aspects - -import ( - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper" -) - -// Aspects holds dependencies for handling aspects of the decomposedfs -type Aspects struct { - Lookup node.PathLookup - Tree node.Tree - Trashbin trashbin.Trashbin - Permissions permissions.Permissions - EventStream events.Stream - DisableVersioning bool - UserMapper usermapper.Mapper -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go deleted file mode 100644 index 08b9bf089d..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/decomposedfs.go +++ /dev/null @@ -1,1359 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "io" - "math" - "net/url" - "path" - "path/filepath" - "strconv" - "strings" - "time" - - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/jellydator/ttlcache/v2" - "github.com/pkg/errors" - "github.com/rs/zerolog" - tusd "github.com/tus/tusd/v2/pkg/handler" - microstore "go-micro.dev/v4/store" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/timemanager" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/store" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -type CtxKey int - -const ( - CtxKeySpaceGID CtxKey = iota -) - -var ( - tracer trace.Tracer - - _registeredEvents = []events.Unmarshaller{ - events.PostprocessingFinished{}, - events.PostprocessingStepFinished{}, - events.RestartPostprocessing{}, - events.CleanUpload{}, - events.DeleteRevision{}, - } -) - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs") -} - -// Session is the interface that OcisSession implements. By combining tus.Upload, -// storage.UploadSession and custom functions we can reuse the same struct throughout -// the whole upload lifecycle. -// -// Some functions that are only used by decomposedfs are not yet part of this interface. -// They might be added after more refactoring. -type Session interface { - tusd.Upload - storage.UploadSession - upload.Session - LockID() string -} - -type SessionStore interface { - New(ctx context.Context) *upload.OcisSession - List(ctx context.Context) ([]*upload.OcisSession, error) - Get(ctx context.Context, id string) (*upload.OcisSession, error) -} - -// Decomposedfs provides the base for decomposed filesystem implementations -type Decomposedfs struct { - lu node.PathLookup - tp node.Tree - trashbin trashbin.Trashbin - o *options.Options - p permissions.Permissions - um usermapper.Mapper - chunkHandler *chunking.ChunkHandler - stream events.Stream - sessionStore SessionStore - - UserCache *ttlcache.Cache - userSpaceIndex *spaceidindex.Index - groupSpaceIndex *spaceidindex.Index - spaceTypeIndex *spaceidindex.Index - - log *zerolog.Logger -} - -// NewDefault returns an instance with default components -func NewDefault(m map[string]interface{}, bs tree.Blobstore, es events.Stream, log *zerolog.Logger) (storage.FS, error) { - if log == nil { - log = &zerolog.Logger{} - } - - o, err := options.New(m) - if err != nil { - return nil, err - } - - var lu *lookup.Lookup - switch o.MetadataBackend { - case "xattrs": - lu = lookup.New(metadata.NewXattrsBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) - case "messagepack": - lu = lookup.New(metadata.NewMessagePackBackend(o.Root, o.FileMetadataCache), o, &timemanager.Manager{}) - default: - return nil, fmt.Errorf("unknown metadata backend %s, only 'messagepack' or 'xattrs' (default) supported", o.MetadataBackend) - } - - tp := tree.New(lu, bs, o, store.Create( - store.Store(o.IDCache.Store), - store.TTL(o.IDCache.TTL), - store.Size(o.IDCache.Size), - microstore.Nodes(o.IDCache.Nodes...), - microstore.Database(o.IDCache.Database), - microstore.Table(o.IDCache.Table), - store.DisablePersistence(o.IDCache.DisablePersistence), - store.Authentication(o.IDCache.AuthUsername, o.IDCache.AuthPassword), - store.TLSEnabled(o.IDCache.TLSEnabled), - store.TLSInsecure(o.IDCache.TLSInsecure), - store.TLSRootCA(o.IDCache.TLSRootCACertificate), - ), log) - - permissionsSelector, err := pool.PermissionsSelector(o.PermissionsSVC, pool.WithTLSMode(o.PermTLSMode)) - if err != nil { - return nil, err - } - - aspects := aspects.Aspects{ - Lookup: lu, - Tree: tp, - Permissions: permissions.NewPermissions(node.NewPermissions(lu), permissionsSelector), - EventStream: es, - DisableVersioning: o.DisableVersioning, - Trashbin: &DecomposedfsTrashbin{}, - } - - return New(o, aspects, log) -} - -// New returns an implementation of the storage.FS interface that talks to -// a local filesystem. -func New(o *options.Options, aspects aspects.Aspects, log *zerolog.Logger) (storage.FS, error) { - if log == nil { - log = &zerolog.Logger{} - } - - err := aspects.Tree.Setup() - if err != nil { - log.Error().Err(err).Msg("could not setup tree") - return nil, errors.Wrap(err, "could not setup tree") - } - - // Run migrations & return - m := migrator.New(aspects.Lookup, log) - err = m.RunMigrations() - if err != nil { - log.Error().Err(err).Msg("could not migrate tree") - return nil, errors.Wrap(err, "could not migrate tree") - } - - if o.MaxAcquireLockCycles != 0 { - filelocks.SetMaxLockCycles(o.MaxAcquireLockCycles) - } - - if o.LockCycleDurationFactor != 0 { - filelocks.SetLockCycleDurationFactor(o.LockCycleDurationFactor) - } - userSpaceIndex := spaceidindex.New(filepath.Join(o.Root, "indexes"), "by-user-id") - err = userSpaceIndex.Init() - if err != nil { - return nil, err - } - groupSpaceIndex := spaceidindex.New(filepath.Join(o.Root, "indexes"), "by-group-id") - err = groupSpaceIndex.Init() - if err != nil { - return nil, err - } - spaceTypeIndex := spaceidindex.New(filepath.Join(o.Root, "indexes"), "by-type") - err = spaceTypeIndex.Init() - if err != nil { - return nil, err - } - - if aspects.Trashbin == nil { - return nil, errors.New("need trashbin") - } - // set a null usermapper if we don't have one - if aspects.UserMapper == nil { - aspects.UserMapper = &usermapper.NullMapper{} - } - - fs := &Decomposedfs{ - tp: aspects.Tree, - lu: aspects.Lookup, - trashbin: aspects.Trashbin, - o: o, - p: aspects.Permissions, - um: aspects.UserMapper, - chunkHandler: chunking.NewChunkHandler(filepath.Join(o.Root, "uploads")), - stream: aspects.EventStream, - UserCache: ttlcache.NewCache(), - userSpaceIndex: userSpaceIndex, - groupSpaceIndex: groupSpaceIndex, - spaceTypeIndex: spaceTypeIndex, - log: log, - } - fs.sessionStore = upload.NewSessionStore(fs, aspects, o.Root, o.AsyncFileUploads, o.Tokens, log) - if err = fs.trashbin.Setup(fs); err != nil { - return nil, err - } - - if o.AsyncFileUploads { - if fs.stream == nil { - log.Error().Msg("need event stream for async file processing") - return nil, errors.New("need nats for async file processing") - } - - ch, err := events.Consume(fs.stream, "dcfs", _registeredEvents...) - if err != nil { - return nil, err - } - - if o.Events.NumConsumers <= 0 { - o.Events.NumConsumers = 1 - } - - for i := 0; i < o.Events.NumConsumers; i++ { - go fs.Postprocessing(ch) - } - } - - return fs, nil -} - -// Postprocessing starts the postprocessing result collector -func (fs *Decomposedfs) Postprocessing(ch <-chan events.Event) { - for event := range ch { - // derive a per-event context from the event's W3C trace context so the - // postprocessing work is linked to the originating request - ctx := event.GetTraceContext(context.Background()) - ctx = ctxpkg.ContextSetInitiator(ctx, event.InitiatorID) - fs.handlePostprocessingEvent(ctx, event) - } -} - -// handlePostprocessingEvent processes a single postprocessing event under a span -// scoped to that event's trace context. -func (fs *Decomposedfs) handlePostprocessingEvent(ctx context.Context, event events.Event) { - ctx, span := tracer.Start(ctx, "Postprocessing/"+event.Type) - defer span.End() - log := fs.log - switch ev := event.Event.(type) { - case events.PostprocessingFinished: - sublog := log.With().Str("event", "PostprocessingFinished").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - if !n.Exists { - sublog.Debug().Msg("node no longer exists") - session.Cleanup(false, true, true, false) - return - } - - var ( - failed bool - revertNodeMetadata bool - keepUpload bool - ) - unmarkPostprocessing := true - - switch ev.Outcome { - default: - sublog.Error().Str("outcome", string(ev.Outcome)).Msg("unknown postprocessing outcome - aborting") - fallthrough - case events.PPOutcomeAbort: - failed = true - revertNodeMetadata = true - keepUpload = true - metrics.UploadSessionsAborted.Inc() - case events.PPOutcomeContinue: - if err := session.Finalize(ctx); err != nil { - sublog.Error().Err(err).Msg("could not finalize upload") - failed = true - revertNodeMetadata = false - keepUpload = true - // keep postprocessing status so the upload is not deleted during housekeeping - unmarkPostprocessing = false - } else { - metrics.UploadSessionsFinalized.Inc() - } - case events.PPOutcomeDelete: - failed = true - revertNodeMetadata = true - metrics.UploadSessionsDeleted.Inc() - } - - getParent := func() *node.Node { - p, err := n.Parent(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read parent") - return nil - } - return p - } - - now := time.Now() - if failed { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - sublog.Error().Err(err).Msg("reading node for session failed") - } - if latestSession == session.ID() { - // propagate reverted sizeDiff after failed postprocessing - if err := fs.tp.Propagate(ctx, n, -session.SizeDiff()); err != nil { - sublog.Error().Err(err).Msg("could not propagate tree size change") - } - } - } else if p := getParent(); p != nil { - // update parent tmtime to propagate etag change after successful postprocessing - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - sublog.Error().Err(err).Msg("could not propagate etag change") - } - } - - session.Cleanup(revertNodeMetadata, !keepUpload, !keepUpload, unmarkPostprocessing) - - var isVersion bool - if session.NodeExists() { - info, err := session.GetInfo(ctx) - if err == nil && info.MetaData["versionsPath"] != "" { - isVersion = true - } - } - - if err := events.Publish( - ctx, - fs.stream, - events.UploadReady{ - UploadID: ev.UploadID, - Failed: failed, - ExecutingUser: ev.ExecutingUser, - Filename: ev.Filename, - FileRef: &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - }, - ResourceID: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - Timestamp: utils.TimeToTS(now), - SpaceOwner: n.SpaceOwnerOrManager(ctx), - IsVersion: isVersion, - ImpersonatingUser: ev.ImpersonatingUser, - }, - ); err != nil { - sublog.Error().Err(err).Msg("Failed to publish UploadReady event") - } - case events.RestartPostprocessing: - sublog := log.With().Str("event", "RestartPostprocessing").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - n, err := session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not read node") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - s, err := session.URL(ctx) - if err != nil { - sublog.Error().Err(err).Msg("could not create url") - return - } - - metrics.UploadSessionsRestarted.Inc() - - // restart postprocessing - if err := events.Publish(ctx, fs.stream, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(ctx), - ExecutingUser: &user.User{Id: &user.UserId{OpaqueId: "postprocessing-restart"}}, // send nil instead? - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - }); err != nil { - sublog.Error().Err(err).Msg("Failed to publish BytesReceived event") - } - case events.CleanUpload: - sublog := log.With().Str("event", "CleanUpload").Str("uploadid", ev.UploadID).Logger() - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return // NOTE: since we can't get the upload, we can't delete the blob - } - session.Cleanup(true, !ev.KeepUpload, !ev.KeepUpload, true) - case events.DeleteRevision: - sublog := log.With().Str("event", "DeleteRevision").Interface("nodeid", ev.ResourceID).Logger() - n, err := fs.lu.NodeFromID(ctx, ev.ResourceID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node") - return - } - - var deleteErr error - if ev.Timestamp == nil { - // the node's current revision is targeted - revert to the pre-upload - // state. Only stuck nodes are targeted: a reverted node has its - // processing flag removed, so a redelivered event is a no-op. - if !n.IsProcessing(ctx) { - sublog.Debug().Msg("node is not stuck, ignoring") - return - } - _, deleteErr = n.DeleteRevision(ctx, "") - } else { - versionID := time.Unix(int64(ev.Timestamp.Seconds), int64(ev.Timestamp.Nanos)).UTC().Format(time.RFC3339Nano) - deleteErr = fs.deleteRevisionFile(ctx, n, n.ID+node.RevisionIDDelimiter+versionID) - } - if deleteErr != nil { - sublog.Error().Err(deleteErr).Msg("Failed to delete revision") - } - case events.PostprocessingStepFinished: - sublog := log.With().Str("event", "PostprocessingStepFinished").Str("uploadid", ev.UploadID).Logger() - if ev.FinishedStep != events.PPStepAntivirus { - // atm we are only interested in antivirus results - return - } - - res := ev.Result.(events.VirusscanResult) - if res.ErrorMsg != "" { - // scan failed somehow - // Should we handle this here? - return - } - sublog = log.With().Str("scan_description", res.Description).Bool("infected", res.Infected).Logger() - - var n *node.Node - switch ev.UploadID { - case "": - // uploadid is empty -> this was an on-demand scan - /* ON DEMAND SCANNING NOT SUPPORTED ATM - ctx := ctxpkg.ContextSetUser(context.Background(), ev.ExecutingUser) - ref := &provider.Reference{ResourceId: ev.ResourceID} - - no, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to get node after scan") - continue - - } - n = no - if ev.Outcome == events.PPOutcomeDelete { - // antivir wants us to delete the file. We must obey and need to - - // check if there a previous versions existing - revs, err := fs.ListRevisions(ctx, ref) - if len(revs) == 0 { - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to list revisions. Fallback to delete file") - } - - // no versions -> trash file - err := fs.Delete(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to delete infected resource") - continue - } - - // now purge it from the recycle bin - if err := fs.PurgeRecycleItem(ctx, &provider.Reference{ResourceId: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.SpaceID}}, n.ID, "/"); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Failed to purge infected resource from trash") - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - - // we have versions - find the newest - versions := make(map[uint64]string) // remember all versions - we need them later - var nv uint64 - for _, v := range revs { - versions[v.Mtime] = v.Key - if v.Mtime > nv { - nv = v.Mtime - } - } - - // restore newest version - if err := fs.RestoreRevision(ctx, ref, versions[nv]); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", versions[nv]).Msg("Failed to restore revision") - continue - } - - // now find infected version - revs, err = fs.ListRevisions(ctx, ref) - if err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Msg("Error listing revisions after restore") - } - - for _, v := range revs { - // we looking for a version that was previously not there - if _, ok := versions[v.Mtime]; ok { - continue - } - - if err := fs.DeleteRevision(ctx, ref, v.Key); err != nil { - log.Error().Err(err).Interface("resourceID", ev.ResourceID).Str("revision", v.Key).Msg("Failed to delete revision") - } - } - - // remove cache entry in gateway - fs.cache.RemoveStatContext(ctx, ev.ExecutingUser.GetId(), &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}) - continue - } - */ - default: - // uploadid is not empty -> this is an async upload - session, err := fs.sessionStore.Get(ctx, ev.UploadID) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get upload") - return - } - - n, err = session.Node(ctx) - if err != nil { - sublog.Error().Err(err).Msg("Failed to get node after scan") - return - } - sublog = log.With().Str("spaceid", session.SpaceID()).Str("nodeid", session.NodeID()).Logger() - - session.SetScanData(res.Description, res.Scandate) - if err := session.Persist(ctx); err != nil { - sublog.Error().Err(err).Msg("Failed to persist scan results") - } - } - - if err := n.SetScanData(ctx, res.Description, res.Scandate); err != nil { - sublog.Error().Err(err).Msg("Failed to set scan results") - return - } - - metrics.UploadSessionsScanned.Inc() - default: - log.Error().Interface("event", ev).Msg("Unknown event") - } -} - -// Shutdown shuts down the storage -func (fs *Decomposedfs) Shutdown(ctx context.Context) error { - return nil -} - -// GetQuota returns the quota available -// TODO Document in the cs3 should we return quota or free space? -func (fs *Decomposedfs) GetQuota(ctx context.Context, ref *provider.Reference) (total uint64, inUse uint64, remaining uint64, err error) { - ctx, span := tracer.Start(ctx, "GetQuota") - defer span.End() - var n *node.Node - if ref == nil { - err = errtypes.BadRequest("no space given") - return 0, 0, 0, err - } - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return 0, 0, 0, err - } - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return 0, 0, 0, err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return 0, 0, 0, err - case !rp.GetQuota && !fs.p.ListAllSpaces(ctx): - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return 0, 0, 0, errtypes.PermissionDenied(f) - } - return 0, 0, 0, errtypes.NotFound(f) - } - - // FIXME move treesize & quota to fieldmask - ri, err := n.AsResourceInfo(ctx, rp, []string{"treesize", "quota"}, []string{}, true) - if err != nil { - return 0, 0, 0, err - } - - quotaStr := node.QuotaUnknown - if ri.Opaque != nil && ri.Opaque.Map != nil && ri.Opaque.Map["quota"] != nil && ri.Opaque.Map["quota"].Decoder == "plain" { - quotaStr = string(ri.Opaque.Map["quota"].Value) - } - - return fs.calculateTotalUsedRemaining(quotaStr, ri.Size) -} - -func (fs *Decomposedfs) calculateTotalUsedRemaining(quotaStr string, inUse uint64) (uint64, uint64, uint64, error) { - var err error - var total uint64 - - remaining := uint64(math.MaxUint64) - switch quotaStr { - case node.QuotaUncalculated, node.QuotaUnknown: - // best we can do is return current total - // TODO indicate unlimited total? -> in opaque data? - case node.QuotaUnlimited: - total = 0 - default: - total, err = strconv.ParseUint(quotaStr, 10, 64) - if err != nil { - return 0, 0, 0, err - } - - switch { - case total > inUse: - remaining = total - inUse - case total <= inUse: - remaining = 0 - } - - } - return total, inUse, remaining, nil -} - -// CreateHome creates a new home node for the given user -func (fs *Decomposedfs) CreateHome(ctx context.Context) (err error) { - ctx, span := tracer.Start(ctx, "CreateHome") - defer span.End() - if fs.o.UserLayout == "" { - return errtypes.NotSupported("Decomposedfs: CreateHome() home supported disabled") - } - - u := ctxpkg.ContextMustGetUser(ctx) - res, err := fs.CreateStorageSpace(ctx, &provider.CreateStorageSpaceRequest{ - Type: _spaceTypePersonal, - Owner: u, - }) - if err != nil { - return err - } - if res.Status.Code != rpcv1beta1.Code_CODE_OK { - return errtypes.NewErrtypeFromStatus(res.Status) - } - return nil -} - -// GetHome is called to look up the home path for a user -// It is NOT supposed to return the internal path but the external path -func (fs *Decomposedfs) GetHome(ctx context.Context) (string, error) { - ctx, span := tracer.Start(ctx, "GetHome") - defer span.End() - if fs.o.UserLayout == "" { - return "", errtypes.NotSupported("Decomposedfs: GetHome() home supported disabled") - } - u := ctxpkg.ContextMustGetUser(ctx) - layout := templates.WithUser(u, fs.o.UserLayout) - return filepath.Join(fs.o.Root, layout), nil // TODO use a namespace? -} - -// GetPathByID returns the fn pointed by the file id, without the internal namespace -func (fs *Decomposedfs) GetPathByID(ctx context.Context, id *provider.ResourceId) (string, error) { - ctx, span := tracer.Start(ctx, "GetPathByID") - defer span.End() - n, err := fs.lu.NodeFromID(ctx, id) - if err != nil { - return "", err - } - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return "", err - case !rp.GetPath: - f := storagespace.FormatResourceID(id) - if rp.Stat { - return "", errtypes.PermissionDenied(f) - } - return "", errtypes.NotFound(f) - } - - hp := func(n *node.Node) bool { - perms, err := fs.p.AssemblePermissions(ctx, n) - if err != nil { - return false - } - return perms.GetPath - } - return fs.lu.Path(ctx, n, hp) -} - -// CreateDir creates the specified directory -func (fs *Decomposedfs) CreateDir(ctx context.Context, ref *provider.Reference) (err error) { - ctx, span := tracer.Start(ctx, "CreateDir") - defer span.End() - - name := path.Base(ref.Path) - if name == "" || name == "." || name == "/" { - return errtypes.BadRequest("Invalid path: " + ref.Path) - } - - parentRef := &provider.Reference{ - ResourceId: ref.ResourceId, - Path: path.Dir(ref.Path), - } - - // verify parent exists - var n *node.Node - if n, err = fs.lu.NodeFromResource(ctx, parentRef); err != nil { - if e, ok := err.(errtypes.NotFound); ok { - return errtypes.PreconditionFailed(e.Error()) - } - return - } - // TODO check if user has access to root / space - if !n.Exists { - return errtypes.PreconditionFailed(parentRef.Path) - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return err - case !rp.CreateContainer: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } - - // verify child does not exist, yet - if n, err = n.Child(ctx, name); err != nil { - return - } - if n.Exists { - return errtypes.AlreadyExists(ref.Path) - } - - if err = fs.tp.CreateDir(ctx, n); err != nil { - return - } - - return -} - -// TouchFile as defined in the storage.FS interface -func (fs *Decomposedfs) TouchFile(ctx context.Context, ref *provider.Reference, markprocessing bool, mtime string) error { - ctx, span := tracer.Start(ctx, "TouchFile") - defer span.End() - parentRef := &provider.Reference{ - ResourceId: ref.ResourceId, - Path: path.Dir(ref.Path), - } - - // verify parent exists - parent, err := fs.lu.NodeFromResource(ctx, parentRef) - if err != nil { - return errtypes.InternalError(err.Error()) - } - if !parent.Exists { - return errtypes.NotFound(parentRef.Path) - } - - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errtypes.InternalError(err.Error()) - } - - // new file: check write permission on parent, not the not-yet-existing node - permNode := n - if !n.Exists { - permNode = parent - } - rp, err := fs.p.AssemblePermissions(ctx, permNode) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } - return fs.tp.TouchFile(ctx, n, markprocessing, mtime) -} - -// CreateReference creates a reference as a node folder with the target stored in extended attributes -// There is no difference between the /Shares folder and normal nodes because the storage is not supposed to be accessible -// without the storage provider. In effect everything is a shadow namespace. -// To mimic the eos and owncloud driver we only allow references as children of the "/Shares" folder -// FIXME: This comment should explain briefly what a reference is in this context. -func (fs *Decomposedfs) CreateReference(ctx context.Context, p string, targetURI *url.URL) (err error) { - return errtypes.NotSupported("not implemented") -} - -// Move moves a resource from one reference to another -func (fs *Decomposedfs) Move(ctx context.Context, oldRef, newRef *provider.Reference) (*storage.MoveResult, error) { - ctx, span := tracer.Start(ctx, "Move") - defer span.End() - var oldNode, newNode *node.Node - var err error - if oldNode, err = fs.lu.NodeFromResource(ctx, oldRef); err != nil { - return nil, err - } - - if !oldNode.Exists { - return nil, errtypes.NotFound(filepath.Join(oldNode.ParentID, oldNode.Name)) - } - - orp, err := fs.p.AssemblePermissions(ctx, oldNode) - switch { - case err != nil: - return nil, err - case !orp.Move: - f, _ := storagespace.FormatReference(oldRef) - if orp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - } - - if newNode, err = fs.lu.NodeFromResource(ctx, newRef); err != nil { - return nil, err - } - if newNode.Exists { - return nil, errtypes.AlreadyExists(filepath.Join(newNode.ParentID, newNode.Name)) - } - - nrp, err := fs.p.AssemblePermissions(ctx, newNode) - switch { - case err != nil: - return nil, err - case oldNode.IsDir(ctx) && !nrp.CreateContainer: - f, _ := storagespace.FormatReference(newRef) - if nrp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - case !oldNode.IsDir(ctx) && !nrp.InitiateFileUpload: - f, _ := storagespace.FormatReference(newRef) - if nrp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - } - - // check lock on source - if err := oldNode.CheckLock(ctx); err != nil { - return nil, err - } - - // check processing on source - if oldNode.IsProcessing(ctx) { - return nil, errtypes.ResourceProcessing(oldRef.String()) - } - - if err := fs.tp.Move(ctx, oldNode, newNode); err != nil { - return nil, err - } - - log := appctx.GetLogger(ctx) - newResolved := newRef - if nref, err := fs.refFromNode(ctx, newNode, newRef.GetResourceId().GetStorageId(), nrp); err != nil { - log.Error().Err(err).Msg("move: failed to resolve new reference") - } else { - newResolved = nref - } - oldResolved := oldRef - if oref, err := fs.refFromNode(ctx, oldNode, oldRef.GetResourceId().GetStorageId(), orp); err != nil { - log.Error().Err(err).Msg("move: failed to resolve old reference") - } else { - oldResolved = oref - } - return &storage.MoveResult{ - SpaceOwner: newNode.SpaceOwnerOrManager(ctx), - OldReference: oldResolved, - NewReference: newResolved, - }, nil -} - -// GetMD returns the metadata for the specified resource -func (fs *Decomposedfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []string, fieldMask []string) (ri *provider.ResourceInfo, err error) { - ctx, span := tracer.Start(ctx, "GetMD") - defer span.End() - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return nil, err - case !rp.Stat: - f, _ := storagespace.FormatReference(ref) - return nil, errtypes.NotFound(f) - } - - md, err := node.AsResourceInfo(ctx, rp, mdKeys, fieldMask, utils.IsRelativeReference(ref)) - if err != nil { - return nil, err - } - - addSpace := len(fieldMask) == 0 - for _, p := range fieldMask { - if p == "space" || p == "*" { - addSpace = true - break - } - } - if addSpace { - if md.Space, err = fs.StorageSpaceFromNode(ctx, node, true); err != nil { - return nil, err - } - } - - return md, nil -} - -// ListFolder returns a list of resources in the specified folder -func (fs *Decomposedfs) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys []string, fieldMask []string) ([]*provider.ResourceInfo, error) { - ctx, span := tracer.Start(ctx, "ListFolder") - defer span.End() - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, err - } - - if !n.Exists { - return nil, errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, err - case !rp.ListContainer: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - } - - children, err := fs.tp.ListFolder(ctx, n) - if err != nil { - return nil, err - } - - numWorkers := fs.o.MaxConcurrency - if len(children) < numWorkers { - numWorkers = len(children) - } - work := make(chan *node.Node, len(children)) - results := make(chan *provider.ResourceInfo, len(children)) - - g, ctx := errgroup.WithContext(ctx) - - // Distribute work - g.Go(func() error { - defer close(work) - for _, child := range children { - select { - case work <- child: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - - // Spawn workers that'll concurrently work the queue - for i := 0; i < numWorkers; i++ { - g.Go(func() error { - for child := range work { - np := rp - // add this childs permissions - pset, _ := child.PermissionSet(ctx) - node.AddPermissions(np, pset) - ri, err := child.AsResourceInfo(ctx, np, mdKeys, fieldMask, utils.IsRelativeReference(ref)) - if err != nil { - return errtypes.InternalError(err.Error()) - } - select { - case results <- ri: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - } - - // Wait for things to settle down, then close results chan - go func() { - _ = g.Wait() // error is checked later - close(results) - }() - - finfos := make([]*provider.ResourceInfo, len(children)) - i := 0 - for fi := range results { - finfos[i] = fi - i++ - } - - if err := g.Wait(); err != nil { - return nil, err - } - - return finfos, nil -} - -// Delete deletes the specified resource -func (fs *Decomposedfs) Delete(ctx context.Context, ref *provider.Reference) (err error) { - ctx, span := tracer.Start(ctx, "Delete") - defer span.End() - var node *node.Node - if node, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !node.Exists { - return errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - } - - if node.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return err - case !rp.Delete: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, node.SpaceOwnerOrManager(ctx)) - - if err := node.CheckLock(ctx); err != nil { - return err - } - - return fs.tp.Delete(ctx, node) -} - -// Download returns a reader to the specified resource -func (fs *Decomposedfs) Download(ctx context.Context, ref *provider.Reference, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { - ctx, span := tracer.Start(ctx, "Download") - defer span.End() - // check if we are trying to download a revision - // TODO the CS3 api should allow initiating a revision download - if ref.ResourceId != nil && strings.Contains(ref.ResourceId.OpaqueId, node.RevisionIDDelimiter) { - return fs.DownloadRevision(ctx, ref, ref.ResourceId.OpaqueId, openReaderFunc) - } - - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, nil, err - } - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return nil, nil, err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, nil, err - case !rp.InitiateFileDownload: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return nil, nil, errtypes.PermissionDenied(f) - } - return nil, nil, errtypes.NotFound(f) - } - - // check processing - if n.IsProcessing(ctx) { - return nil, nil, errtypes.ResourceProcessing(ref.String()) - } - ri, err := n.AsResourceInfo(ctx, rp, nil, []string{"size", "mimetype", "etag"}, true) - if err != nil { - return nil, nil, err - } - var reader io.ReadCloser - if openReaderFunc(ri) { - reader, err = fs.tp.ReadBlob(n) - if err != nil { - return nil, nil, errors.Wrap(err, "Decomposedfs: error download blob '"+n.ID+"'") - } - } - return ri, reader, nil -} - -// GetLock returns an existing lock on the given reference -func (fs *Decomposedfs) GetLock(ctx context.Context, ref *provider.Reference) (*provider.Lock, error) { - ctx, span := tracer.Start(ctx, "GetLock") - defer span.End() - node, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error resolving ref") - } - - if !node.Exists { - err = errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - return nil, err - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return nil, err - case !rp.InitiateFileDownload: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - } - - return node.ReadLock(ctx, false) -} - -// SetLock puts a lock on the given reference -func (fs *Decomposedfs) SetLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - ctx, span := tracer.Start(ctx, "SetLock") - defer span.End() - node, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error resolving ref") - } - - if !node.Exists { - return errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - return node.SetLock(ctx, lock) -} - -// RefreshLock refreshes an existing lock on the given reference -func (fs *Decomposedfs) RefreshLock(ctx context.Context, ref *provider.Reference, lock *provider.Lock, existingLockID string) error { - ctx, span := tracer.Start(ctx, "RefreshLock") - defer span.End() - if lock.LockId == "" { - return errtypes.BadRequest("missing lockid") - } - - node, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error resolving ref") - } - - if !node.Exists { - return errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - return node.RefreshLock(ctx, lock, existingLockID) -} - -// Unlock removes an existing lock from the given reference -func (fs *Decomposedfs) Unlock(ctx context.Context, ref *provider.Reference, lock *provider.Lock) error { - ctx, span := tracer.Start(ctx, "Unlock") - defer span.End() - if lock.LockId == "" { - return errtypes.BadRequest("missing lockid") - } - - node, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error resolving ref") - } - - if !node.Exists { - return errtypes.NotFound(filepath.Join(node.ParentID, node.Name)) - } - - rp, err := fs.p.AssemblePermissions(ctx, node) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: // TODO do we need a dedicated permission? - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - return node.Unlock(ctx, lock) -} - -func (fs *Decomposedfs) AddLabel(ctx context.Context, ref *provider.Reference, userID *user.UserId, label string) error { - return errtypes.NotSupported("AddLabel not implemented") -} - -func (fs *Decomposedfs) RemoveLabel(ctx context.Context, ref *provider.Reference, userID *user.UserId, label string) error { - return errtypes.NotSupported("RemoveLabel not implemented") -} - -func (fs *Decomposedfs) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { - return fs.trashbin.ListRecycle(ctx, ref, key, relativePath) -} -func (fs *Decomposedfs) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { - return fs.trashbin.RestoreRecycleItem(ctx, ref, key, relativePath, restoreRef) -} -func (fs *Decomposedfs) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { - return fs.trashbin.PurgeRecycleItem(ctx, ref, key, relativePath) -} -func (fs *Decomposedfs) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { - return fs.trashbin.EmptyRecycle(ctx, ref) -} - -func (fs *Decomposedfs) getNodePath(ctx context.Context, n *node.Node, perms *provider.ResourcePermissions) (string, error) { - hp := func(n *node.Node) bool { - return perms.GetGetPath() - } - return fs.lu.Path(ctx, n, hp) -} - -func (fs *Decomposedfs) refFromNode(ctx context.Context, n *node.Node, storageId string, perms *provider.ResourcePermissions) (*provider.Reference, error) { - var err error - if perms == nil { - perms, err = fs.p.AssemblePermissions(ctx, n) - if err != nil { - return nil, err - } - } - path, err := fs.getNodePath(ctx, n, perms) - if err != nil { - return nil, err - } - return &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: storageId, - OpaqueId: n.SpaceID, - SpaceId: n.SpaceID, - }, - Path: path, - }, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/grants.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/grants.go deleted file mode 100644 index b8f64331a4..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/grants.go +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "path/filepath" - "strings" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -// DenyGrant denies access to a resource. -func (fs *Decomposedfs) DenyGrant(ctx context.Context, ref *provider.Reference, grantee *provider.Grantee) error { - _, span := tracer.Start(ctx, "DenyGrant") - defer span.End() - log := appctx.GetLogger(ctx) - - log.Debug().Interface("ref", ref).Interface("grantee", grantee).Msg("DenyGrant()") - - grantNode, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return err - } - if !grantNode.Exists { - return errtypes.NotFound(filepath.Join(grantNode.ParentID, grantNode.Name)) - } - - // set all permissions to false - grant := &provider.Grant{ - Grantee: grantee, - Permissions: &provider.ResourcePermissions{}, - } - - // add acting user - u := ctxpkg.ContextMustGetUser(ctx) - grant.Creator = u.GetId() - - rp, err := fs.p.AssemblePermissions(ctx, grantNode) - - switch { - case err != nil: - return err - case !rp.DenyGrant: - return errtypes.PermissionDenied(filepath.Join(grantNode.ParentID, grantNode.Name)) - } - - return fs.storeGrant(ctx, grantNode, grant) -} - -// AddGrant adds a grant to a resource -func (fs *Decomposedfs) AddGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - _, span := tracer.Start(ctx, "AddGrant") - defer span.End() - log := appctx.GetLogger(ctx) - log.Debug().Interface("ref", ref).Interface("grant", g).Msg("AddGrant()") - grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g) - if err != nil { - return err - } - defer func() { - _ = unlockFunc() - }() - - if grant != nil { - return errtypes.AlreadyExists(filepath.Join(grantNode.ParentID, grantNode.Name)) - } - - owner := grantNode.Owner() - grants, err := grantNode.ListGrants(ctx) - if err != nil { - return err - } - - // If the owner is empty and there are no grantees then we are dealing with a just created project space. - // In this case we don't need to check for permissions and just add the grant since this will be the project - // manager. - // When the owner is empty but grants are set then we do want to check the grants. - // However, if we are trying to edit an existing grant we do not have to check for permission if the user owns the grant - // TODO: find a better to check this - if len(grants) != 0 || (owner != nil && owner.OpaqueId != "" && (owner.OpaqueId != grantNode.SpaceID || owner.Type != 8)) { - rp, err := fs.p.AssemblePermissions(ctx, grantNode) - switch { - case err != nil: - return err - case !rp.AddGrant: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - } - // check processing - if grantNode.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - return fs.storeGrant(ctx, grantNode, g) -} - -// ListGrants lists the grants on the specified resource -func (fs *Decomposedfs) ListGrants(ctx context.Context, ref *provider.Reference) (grants []*provider.Grant, err error) { - _, span := tracer.Start(ctx, "ListGrants") - defer span.End() - var grantNode *node.Node - if grantNode, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !grantNode.Exists { - err = errtypes.NotFound(filepath.Join(grantNode.ParentID, grantNode.Name)) - return - } - rp, err := fs.p.AssemblePermissions(ctx, grantNode) - switch { - case err != nil: - return nil, err - case !rp.ListGrants && !rp.Stat: - f, _ := storagespace.FormatReference(ref) - return nil, errtypes.NotFound(f) - } - log := appctx.GetLogger(ctx) - var attrs node.Attributes - if attrs, err = grantNode.Xattrs(ctx); err != nil { - log.Error().Err(err).Msg("error listing attributes") - return nil, err - } - - aces := []*ace.ACE{} - for k, v := range attrs { - if strings.HasPrefix(k, prefixes.GrantPrefix) { - var err error - var e *ace.ACE - principal := k[len(prefixes.GrantPrefix):] - if e, err = ace.Unmarshal(principal, v); err != nil { - log.Error().Err(err).Str("principal", principal).Str("attr", k).Msg("could not unmarshal ace") - continue - } - aces = append(aces, e) - } - } - - uid := ctxpkg.ContextMustGetUser(ctx).GetId() - grants = make([]*provider.Grant, 0, len(aces)) - for i := range aces { - g := aces[i].Grant() - - // you may list your own grants even without listgrants permission - if !rp.ListGrants && !utils.UserIDEqual(g.Creator, uid) && !utils.UserIDEqual(g.Grantee.GetUserId(), uid) { - continue - } - - grants = append(grants, g) - } - - return grants, nil -} - -// RemoveGrant removes a grant from resource -func (fs *Decomposedfs) RemoveGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (err error) { - _, span := tracer.Start(ctx, "RemoveGrant") - defer span.End() - grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g) - if err != nil { - return err - } - defer func() { - _ = unlockFunc() - }() - - if grant == nil { - return errtypes.NotFound("grant not found") - } - - // you are allowed to remove grants if you created them yourself or have the proper permission - if !utils.UserIDEqual(grant.Creator, ctxpkg.ContextMustGetUser(ctx).GetId()) { - rp, err := fs.p.AssemblePermissions(ctx, grantNode) - switch { - case err != nil: - return err - case !rp.RemoveGrant: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - } - // check processing - if grantNode.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - if err := grantNode.DeleteGrant(ctx, g, false); err != nil { - return err - } - - if isShareGrant(ctx) { - // do not invalidate by user or group indexes - // FIXME we should invalidate the by-type index, but that requires reference counting - } else { - // invalidate space grant - switch g.Grantee.Type { - case provider.GranteeType_GRANTEE_TYPE_USER: - // remove from user index - if err := fs.userSpaceIndex.Remove(g.Grantee.GetUserId().GetOpaqueId(), grantNode.SpaceID); err != nil { - return err - } - case provider.GranteeType_GRANTEE_TYPE_GROUP: - // remove from group index - if err := fs.groupSpaceIndex.Remove(g.Grantee.GetGroupId().GetOpaqueId(), grantNode.SpaceID); err != nil { - return err - } - } - } - - return fs.tp.Propagate(ctx, grantNode, 0) -} - -func isShareGrant(ctx context.Context) bool { - _, ok := storageprovider.SpaceTypeFromContext(ctx) - return !ok -} - -// UpdateGrant updates a grant on a resource -// TODO remove AddGrant or UpdateGrant grant from CS3 api, redundant? tracked in https://github.com/cs3org/cs3apis/issues/92 -func (fs *Decomposedfs) UpdateGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) error { - _, span := tracer.Start(ctx, "UpdateGrant") - defer span.End() - log := appctx.GetLogger(ctx) - log.Debug().Interface("ref", ref).Interface("grant", g).Msg("UpdateGrant()") - - grantNode, unlockFunc, grant, err := fs.loadGrant(ctx, ref, g) - if err != nil { - return err - } - defer func() { - _ = unlockFunc() - }() - - if grant == nil { - // grant not found - // TODO: fallback to AddGrant? - return errtypes.NotFound(g.Grantee.GetUserId().GetOpaqueId()) - } - - // You may update a grant when you have the UpdateGrant permission or created the grant (regardless what your permissions are now) - if !utils.UserIDEqual(grant.Creator, ctxpkg.ContextMustGetUser(ctx).GetId()) { - rp, err := fs.p.AssemblePermissions(ctx, grantNode) - switch { - case err != nil: - return err - case !rp.UpdateGrant: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - } - - // check processing - if grantNode.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - return fs.storeGrant(ctx, grantNode, g) -} - -// checks if the given grant exists and returns it. Nil grant means it doesn't exist -func (fs *Decomposedfs) loadGrant(ctx context.Context, ref *provider.Reference, g *provider.Grant) (*node.Node, metadata.UnlockFunc, *provider.Grant, error) { - _, span := tracer.Start(ctx, "loadGrant") - defer span.End() - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return nil, nil, nil, err - } - if !n.Exists { - return nil, nil, nil, errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - } - - // lock the metadata file - unlockFunc, err := fs.lu.MetadataBackend().Lock(n.InternalPath()) - if err != nil { - return nil, nil, nil, err - } - - grants, err := n.ListGrants(ctx) - if err != nil { - return nil, nil, nil, err - } - - for _, grant := range grants { - switch grant.Grantee.GetType() { - case provider.GranteeType_GRANTEE_TYPE_USER: - if g.Grantee.GetUserId().GetOpaqueId() == grant.Grantee.GetUserId().GetOpaqueId() { - return n, unlockFunc, grant, nil - } - case provider.GranteeType_GRANTEE_TYPE_GROUP: - if g.Grantee.GetGroupId().GetOpaqueId() == grant.Grantee.GetGroupId().GetOpaqueId() { - return n, unlockFunc, grant, nil - } - } - } - - return n, unlockFunc, nil, nil -} - -func (fs *Decomposedfs) storeGrant(ctx context.Context, n *node.Node, g *provider.Grant) error { - _, span := tracer.Start(ctx, "storeGrant") - defer span.End() - // if is a grant to a space root, the receiver needs the space type to update the indexes - spaceType, ok := storageprovider.SpaceTypeFromContext(ctx) - if !ok { - // this is not a grant on a space root we are just adding a share - spaceType = spaceTypeShare - } - - // set the grant - e := ace.FromGrant(g) - principal, value := e.Marshal() - attribs := node.Attributes{ - prefixes.GrantPrefix + principal: value, - } - if err := n.SetXattrsWithContext(ctx, attribs, false); err != nil { - appctx.GetLogger(ctx).Error().Err(err). - Str("principal", principal).Msg("Could not set grant for principal") - return err - } - - // update the indexes only after successfully setting the grant - err := fs.updateIndexes(ctx, g.GetGrantee(), spaceType, n.SpaceID, n.ID) - if err != nil { - return err - } - - return fs.tp.Propagate(ctx, n, 0) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go deleted file mode 100644 index 70d11d4848..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup/lookup.go +++ /dev/null @@ -1,412 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package lookup - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - - user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -var tracer trace.Tracer - -const ( - _spaceTypePersonal = "personal" -) - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/lookup") -} - -// Lookup implements transformations from filepath to node and back -type Lookup struct { - Options *options.Options - - metadataBackend metadata.Backend - tm node.TimeManager -} - -// New returns a new Lookup instance -func New(b metadata.Backend, o *options.Options, tm node.TimeManager) *Lookup { - return &Lookup{ - Options: o, - metadataBackend: b, - tm: tm, - } -} - -// MetadataBackend returns the metadata backend -func (lu *Lookup) MetadataBackend() metadata.Backend { - return lu.metadataBackend -} - -func (lu *Lookup) ReadBlobIDAndSizeAttr(ctx context.Context, path string, attrs node.Attributes) (string, int64, error) { - blobID := "" - blobSize := int64(0) - var err error - - if attrs != nil { - blobID = attrs.String(prefixes.BlobIDAttr) - if blobID != "" { - blobSize, err = attrs.Int64(prefixes.BlobsizeAttr) - if err != nil { - return "", 0, err - } - } - } else { - attrs, err := lu.metadataBackend.All(ctx, path) - if err != nil { - return "", 0, errors.Wrapf(err, "error reading blobid xattr") - } - nodeAttrs := node.Attributes(attrs) - blobID = nodeAttrs.String(prefixes.BlobIDAttr) - blobSize, err = nodeAttrs.Int64(prefixes.BlobsizeAttr) - if err != nil { - return "", 0, errors.Wrapf(err, "error reading blobsize xattr") - } - } - return blobID, blobSize, nil -} - -func readChildNodeFromLink(path string) (string, error) { - link, err := os.Readlink(path) - if err != nil { - return "", err - } - nodeID := strings.TrimLeft(link, "/.") - nodeID = strings.ReplaceAll(nodeID, "/", "") - return nodeID, nil -} - -func (lu *Lookup) NodeIDFromParentAndName(ctx context.Context, parent *node.Node, name string) (string, error) { - nodeID, err := readChildNodeFromLink(filepath.Join(parent.InternalPath(), name)) - if err != nil { - return "", errors.Wrap(err, "decomposedfs: Wrap: readlink error") - } - return nodeID, nil -} - -// TypeFromPath returns the type of the node at the given path -func (lu *Lookup) TypeFromPath(ctx context.Context, path string) provider.ResourceType { - // Try to read from xattrs - typeAttr, err := lu.metadataBackend.GetInt64(ctx, path, prefixes.TypeAttr) - if err == nil { - return provider.ResourceType(int32(typeAttr)) - } - - t := provider.ResourceType_RESOURCE_TYPE_INVALID - // Fall back to checking on disk - fi, err := os.Lstat(path) - if err != nil { - return t - } - - switch { - case fi.IsDir(): - if _, err = lu.metadataBackend.Get(ctx, path, prefixes.ReferenceAttr); err == nil { - t = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } else { - t = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - case fi.Mode().IsRegular(): - t = provider.ResourceType_RESOURCE_TYPE_FILE - case fi.Mode()&os.ModeSymlink != 0: - t = provider.ResourceType_RESOURCE_TYPE_SYMLINK - // TODO reference using ext attr on a symlink - // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } - return t -} - -// NodeFromResource takes in a request path or request id and converts it to a Node -func (lu *Lookup) NodeFromResource(ctx context.Context, ref *provider.Reference) (*node.Node, error) { - ctx, span := tracer.Start(ctx, "NodeFromResource") - defer span.End() - - if ref.ResourceId != nil { - // check if a storage space reference is used - // currently, the decomposed fs uses the root node id as the space id - n, err := lu.NodeFromID(ctx, ref.ResourceId) - if err != nil { - return nil, err - } - // is this a relative reference? - if ref.Path != "" { - p := filepath.Clean(ref.Path) - if p != "." && p != "/" { - // walk the relative path - n, err = lu.WalkPath(ctx, n, p, false, func(ctx context.Context, n *node.Node) error { return nil }) - if err != nil { - return nil, err - } - n.SpaceID = ref.ResourceId.SpaceId - } - } - return n, nil - } - - // reference is invalid - return nil, fmt.Errorf("invalid reference %+v. resource_id must be set", ref) -} - -// NodeFromID returns the internal path for the id -func (lu *Lookup) NodeFromID(ctx context.Context, id *provider.ResourceId) (n *node.Node, err error) { - ctx, span := tracer.Start(ctx, "NodeFromID") - defer span.End() - if id == nil { - return nil, fmt.Errorf("invalid resource id %+v", id) - } - if id.OpaqueId == "" { - // The Resource references the root of a space - return lu.NodeFromSpaceID(ctx, id.SpaceId) - } - return node.ReadNode(ctx, lu, id.SpaceId, id.OpaqueId, false, nil, false) -} - -// Pathify segments the beginning of a string into depth segments of width length -// Pathify("aabbccdd", 3, 1) will return "a/a/b/bccdd" -func Pathify(id string, depth, width int) string { - b := strings.Builder{} - i := 0 - for ; i < depth; i++ { - if len(id) <= i*width+width { - break - } - b.WriteString(id[i*width : i*width+width]) - b.WriteRune(filepath.Separator) - } - b.WriteString(id[i*width:]) - return b.String() -} - -// NodeFromSpaceID converts a resource id into a Node -func (lu *Lookup) NodeFromSpaceID(ctx context.Context, spaceID string) (n *node.Node, err error) { - node, err := node.ReadNode(ctx, lu, spaceID, spaceID, false, nil, false) - if err != nil { - return nil, err - } - - node.SpaceRoot = node - return node, nil -} - -// GenerateSpaceID generates a new space id and alias -func (lu *Lookup) GenerateSpaceID(spaceType string, owner *user.User) (string, error) { - switch spaceType { - case _spaceTypePersonal: - return owner.Id.OpaqueId, nil - default: - return uuid.New().String(), nil - } -} - -// Path returns the path for node -func (lu *Lookup) Path(ctx context.Context, n *node.Node, hasPermission node.PermissionFunc) (p string, err error) { - root := n.SpaceRoot - var child *node.Node - for n.ID != root.ID { - p = filepath.Join(n.Name, p) - child = n - if n, err = n.Parent(ctx); err != nil { - appctx.GetLogger(ctx). - Error().Err(err). - Str("path", p). - Str("spaceid", child.SpaceID). - Str("nodeid", child.ID). - Str("parentid", child.ParentID). - Msg("Path()") - return - } - - if !hasPermission(n) { - break - } - } - p = filepath.Join("/", p) - return -} - -// WalkPath calls n.Child(segment) on every path segment in p starting at the node r. -// If a function f is given it will be executed for every segment node, but not the root node r. -// If followReferences is given the current visited reference node is replaced by the referenced node. -func (lu *Lookup) WalkPath(ctx context.Context, r *node.Node, p string, followReferences bool, f func(ctx context.Context, n *node.Node) error) (*node.Node, error) { - segments := strings.Split(strings.Trim(p, "/"), "/") - var err error - for i := range segments { - if r, err = r.Child(ctx, segments[i]); err != nil { - return r, err - } - - if followReferences { - if attrBytes, err := r.Xattr(ctx, prefixes.ReferenceAttr); err == nil { - realNodeID := attrBytes - ref, err := refFromCS3(realNodeID) - if err != nil { - return nil, err - } - - r, err = lu.NodeFromID(ctx, ref.ResourceId) - if err != nil { - return nil, err - } - } - } - if r.IsSpaceRoot(ctx) { - r.SpaceRoot = r - } - - if !r.Exists && i < len(segments)-1 { - return r, errtypes.NotFound(segments[i]) - } - if f != nil { - if err = f(ctx, r); err != nil { - return r, err - } - } - } - return r, nil -} - -// InternalRoot returns the internal storage root directory -func (lu *Lookup) InternalRoot() string { - return lu.Options.Root -} - -// InternalPath returns the internal path for a given ID -func (lu *Lookup) InternalPath(spaceID, nodeID string) string { - return filepath.Join(lu.Options.Root, "spaces", Pathify(spaceID, 1, 2), "nodes", Pathify(nodeID, 4, 2)) -} - -// // ReferenceFromAttr returns a CS3 reference from xattr of a node. -// // Supported formats are: "cs3:storageid/nodeid" -// func ReferenceFromAttr(b []byte) (*provider.Reference, error) { -// return refFromCS3(b) -// } - -// refFromCS3 creates a CS3 reference from a set of bytes. This method should remain private -// and only be called after validation because it can potentially panic. -func refFromCS3(b []byte) (*provider.Reference, error) { - parts := string(b[4:]) - return &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: strings.Split(parts, "/")[0], - OpaqueId: strings.Split(parts, "/")[1], - }, - }, nil -} - -// CopyMetadata copies all extended attributes from source to target. -// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix -// For the source file, a shared lock is acquired. -// NOTE: target resource will be write locked! -func (lu *Lookup) CopyMetadata(ctx context.Context, src, target string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) { - // Acquire a read log on the source node - // write lock existing node before reading treesize or tree time - lock, err := lockedfile.OpenFile(lu.MetadataBackend().LockfilePath(src), os.O_RDONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - - if err != nil { - return errors.Wrap(err, "xattrs: Unable to lock source to read") - } - defer func() { - rerr := lock.Close() - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - return lu.CopyMetadataWithSourceLock(ctx, src, target, filter, lock, acquireTargetLock) -} - -// CopyMetadataWithSourceLock copies all extended attributes from source to target. -// The optional filter function can be used to filter by attribute name, e.g. by checking a prefix -// For the source file, a matching lockedfile is required. -// NOTE: target resource will be write locked! -func (lu *Lookup) CopyMetadataWithSourceLock(ctx context.Context, sourcePath, targetPath string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) { - switch { - case lockedSource == nil: - return errors.New("no lock provided") - case lockedSource.Name() != lu.MetadataBackend().LockfilePath(sourcePath): - return errors.New("lockpath does not match filepath") - } - - attrs, err := lu.metadataBackend.All(ctx, sourcePath) - if err != nil { - return err - } - - newAttrs := make(map[string][]byte, 0) - for attrName, val := range attrs { - if filter != nil { - var ok bool - if val, ok = filter(attrName, val); !ok { - continue - } - } - newAttrs[attrName] = val - } - - return lu.MetadataBackend().SetMultiple(ctx, targetPath, newAttrs, acquireTargetLock) -} - -// TimeManager returns the time manager -func (lu *Lookup) TimeManager() node.TimeManager { - return lu.tm -} - -// DetectBackendOnDisk returns the name of the metadata backend being used on disk -func DetectBackendOnDisk(root string) string { - matches, _ := filepath.Glob(filepath.Join(root, "spaces", "*", "*")) - if len(matches) > 0 { - base := matches[len(matches)-1] - spaceid := strings.ReplaceAll( - strings.TrimPrefix(base, filepath.Join(root, "spaces")), - "/", "") - spaceRoot := Pathify(spaceid, 4, 2) - _, err := os.Stat(filepath.Join(base, "nodes", spaceRoot+".mpk")) - if err == nil { - return "mpk" - } - _, err = os.Stat(filepath.Join(base, "nodes", spaceRoot+".ini")) - if err == nil { - return "ini" - } - } - return "xattrs" -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata.go deleted file mode 100644 index becd43b87e..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "path/filepath" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" -) - -// SetArbitraryMetadata sets the metadata on a resource -func (fs *Decomposedfs) SetArbitraryMetadata(ctx context.Context, ref *provider.Reference, md *provider.ArbitraryMetadata) (err error) { - _, span := tracer.Start(ctx, "SetArbitraryMetadata") - defer span.End() - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error resolving ref") - } - sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: // TODO add explicit SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } - - errs := []error{} - // TODO should we really continue updating when an error occurs? - if md.Metadata != nil { - if val, ok := md.Metadata["mtime"]; ok { - delete(md.Metadata, "mtime") - if err := n.SetMtimeString(ctx, val); err != nil { - errs = append(errs, errors.Wrap(err, "could not set mtime")) - } - } - // TODO(jfd) special handling for atime? - // TODO(jfd) allow setting birth time (btime)? - // TODO(jfd) any other metadata that is interesting? fileid? - // TODO unset when file is updated - // TODO unset when folder is updated or add timestamp to etag? - if val, ok := md.Metadata["etag"]; ok { - delete(md.Metadata, "etag") - if err := n.SetEtag(ctx, val); err != nil { - errs = append(errs, errors.Wrap(err, "could not set etag")) - } - } - if val, ok := md.Metadata[node.FavoriteKey]; ok { - delete(md.Metadata, node.FavoriteKey) - if u, ok := ctxpkg.ContextGetUser(ctx); ok { - if uid := u.GetId(); uid != nil { - if err := n.SetFavorite(ctx, uid, val); err != nil { - sublog.Error().Err(err). - Interface("user", u). - Msg("could not set favorite flag") - errs = append(errs, errors.Wrap(err, "could not set favorite flag")) - } - } else { - sublog.Error().Interface("user", u).Msg("user has no id") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) - } - } else { - sublog.Error().Interface("user", u).Msg("error getting user from ctx") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) - } - } - } - for k, v := range md.Metadata { - attrName := prefixes.MetadataPrefix + k - if err = n.SetXattrString(ctx, attrName, v); err != nil { - errs = append(errs, errors.Wrap(err, "Decomposedfs: could not set metadata attribute "+attrName+" to "+k)) - } - } - - switch len(errs) { - case 0: - return fs.tp.Propagate(ctx, n, 0) - case 1: - // TODO Propagate if anything changed - return errs[0] - default: - // TODO Propagate if anything changed - // TODO how to return multiple errors? - return errors.New("multiple errors occurred, see log for details") - } -} - -// UnsetArbitraryMetadata unsets the metadata on the given resource -func (fs *Decomposedfs) UnsetArbitraryMetadata(ctx context.Context, ref *provider.Reference, keys []string) (err error) { - _, span := tracer.Start(ctx, "UnsetArbitraryMetadata") - defer span.End() - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error resolving ref") - } - sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() - - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return err - case !rp.InitiateFileUpload: // TODO use SetArbitraryMetadata grant to CS3 api, tracked in https://github.com/cs3org/cs3apis/issues/91 - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } - - errs := []error{} - for _, k := range keys { - switch k { - case node.FavoriteKey: - // the favorite flag is specific to the user, so we need to incorporate the userid - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - sublog.Error(). - Interface("user", u). - Msg("error getting user from ctx") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "error getting user from ctx")) - continue - } - var uid *userpb.UserId - if uid = u.GetId(); uid == nil || uid.OpaqueId == "" { - sublog.Error(). - Interface("user", u). - Msg("user has no id") - errs = append(errs, errors.Wrap(errtypes.UserRequired("userrequired"), "user has no id")) - continue - } - fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if err := n.RemoveXattr(ctx, fa, true); err != nil { - if metadata.IsAttrUnset(err) { - continue // already gone, ignore - } - sublog.Error().Err(err). - Interface("user", u). - Str("key", fa). - Msg("could not unset favorite flag") - errs = append(errs, errors.Wrap(err, "could not unset favorite flag")) - } - default: - if err = n.RemoveXattr(ctx, prefixes.MetadataPrefix+k, true); err != nil { - if metadata.IsAttrUnset(err) { - continue // already gone, ignore - } - sublog.Error().Err(err). - Str("key", k). - Msg("could not unset metadata") - errs = append(errs, errors.Wrap(err, "could not unset metadata")) - } - } - } - switch len(errs) { - case 0: - return fs.tp.Propagate(ctx, n, 0) - case 1: - // TODO Propagate if anything changed - return errs[0] - default: - // TODO Propagate if anything changed - // TODO how to return multiple errors? - return errors.New("multiple errors occurred, see log for details") - } -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/errors.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/errors.go deleted file mode 100644 index 6d2f4f6d9c..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/errors.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package metadata - -import ( - "io/fs" - "os" - "syscall" - - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/pkg/errors" - "github.com/pkg/xattr" -) - -// IsNotExist checks if there is a os not exists error buried inside the xattr error, -// as we cannot just use os.IsNotExist(). -func IsNotExist(err error) bool { - if _, ok := err.(errtypes.IsNotFound); ok { - return true - } - if os.IsNotExist(errors.Cause(err)) { - return true - } - if xerr, ok := errors.Cause(err).(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOENT - } - } - return false -} - -// IsErrRange checks for a syscall.ERANGE buried inside an xattr error. listxattr -// returns ERANGE when the set of attribute names grows between the size-probe call -// and the read call (a concurrent setxattr landed in between). This is transient and -// self-clearing once the competing writer finishes its batch. -func IsErrRange(err error) bool { - if xerr, ok := errors.Cause(err).(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ERANGE - } - } - return false -} - -// IsErrTooBig checks for a syscall.E2BIG buried inside an xattr error. listxattr -// returns E2BIG when the attribute name list is larger than the maximum size the -// kernel will hand out (the 64KiB VFS limit, e.g. on XFS). Unlike ERANGE this is -// not a transient race: the list cannot be retrieved at all, so callers should -// fail fast instead of retrying. -func IsErrTooBig(err error) bool { - if xerr, ok := errors.Cause(err).(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.E2BIG - } - } - return false -} - -// IsAttrUnset checks the xattr.ENOATTR from the xattr package which redifines it as ENODATA on platforms that do not natively support it (eg. linux) -// see https://github.com/pkg/xattr/blob/8725d4ccc0fcef59c8d9f0eaf606b3c6f962467a/xattr_linux.go#L19-L22 -func IsAttrUnset(err error) bool { - if xerr, ok := errors.Cause(err).(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == xattr.ENOATTR - } - } - return false -} - -// The os error is buried inside the fs.PathError error -func IsNotDir(err error) bool { - if perr, ok := errors.Cause(err).(*fs.PathError); ok { - if serr, ok2 := perr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOTDIR - } - } - if xerr, ok := errors.Cause(err).(*xattr.Error); ok { - if serr, ok2 := xerr.Err.(syscall.Errno); ok2 { - return serr == syscall.ENOTDIR - } - } - return false -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/list.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/list.go deleted file mode 100644 index 77be5d08d0..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/list.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2026 OpenCloud GmbH -// SPDX-License-Identifier: Apache-2.0 - -package metadata - -import ( - "context" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/pkg/xattr" -) - -// maxListXattrRetries bounds the number of attempts made by listXattr when it -// keeps racing with a concurrent writer. -const maxListXattrRetries = 10 - -// listXattr lists the extended attribute names of the file at path, retrying on -// transient ERANGE errors. -// -// xattr.List is not atomic: internally it issues a listxattr syscall to learn -// the size of the extended-attribute name list, allocates a buffer of that -// size and issues a second listxattr to read the names into it. If another -// goroutine or process adds an extended attribute to the same inode between the -// two syscalls, the name list grows beyond the buffer and the kernel returns -// ERANGE ("numerical result out of range"). -// -// This is a transient condition: a fresh size query on the next attempt -// reflects the new size, so we simply retry a bounded number of times. See the -// posix fs watcher, whose assimilation can read a node's xattrs while the node -// is still being created, for a concrete source of such concurrent writes. -// -// E2BIG is deliberately not retried: it means the name list is larger than the -// maximum size the kernel will hand out (the 64KiB VFS limit, e.g. on XFS), -// which is not a race and cannot be retried away. It is returned as-is. -func listXattr(ctx context.Context, path string) ([]string, error) { - var ( - attrs []string - err error - ) - for i := 0; i < maxListXattrRetries; i++ { - attrs, err = xattr.List(path) - if err == nil { - if i > 0 { - appctx.GetLogger(ctx).Debug().Str("path", path).Int("attempt", i+1).Msg("xattr.List ERANGE recovered after retry") - } - return attrs, nil - } - if !IsErrRange(err) { - return attrs, err - } - appctx.GetLogger(ctx).Debug().Err(err).Str("path", path).Int("attempt", i+1).Msg("xattr.List ERANGE, retrying") - } - appctx.GetLogger(ctx).Error().Err(err).Str("path", path).Msg("xattr.List still failing after ERANGE retries") - return attrs, err -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go deleted file mode 100644 index 4a38300965..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/messagepack_backend.go +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package metadata - -import ( - "context" - "errors" - "io" - "io/fs" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/google/renameio/v2" - "github.com/opencloud-eu/reva/v2/pkg/storage/cache" - "github.com/pkg/xattr" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/shamaton/msgpack/v2" - "go.opentelemetry.io/otel/codes" -) - -// MessagePackBackend persists the attributes in messagepack format inside the file -type MessagePackBackend struct { - rootPath string - metaCache cache.FileMetadataCache -} - -type readWriteCloseSeekTruncater interface { - io.ReadWriteCloser - io.Seeker - Truncate(int64) error -} - -// NewMessagePackBackend returns a new MessagePackBackend instance -func NewMessagePackBackend(rootPath string, o cache.Config) MessagePackBackend { - return MessagePackBackend{ - rootPath: filepath.Clean(rootPath), - metaCache: cache.GetFileMetadataCache(o), - } -} - -// Name returns the name of the backend -func (MessagePackBackend) Name() string { return "messagepack" } - -// All reads all extended attributes for a node -func (b MessagePackBackend) All(ctx context.Context, path string) (map[string][]byte, error) { - return b.loadAttributes(ctx, path, nil) -} - -// Get an extended attribute value for the given key -func (b MessagePackBackend) Get(ctx context.Context, path, key string) ([]byte, error) { - attribs, err := b.loadAttributes(ctx, path, nil) - if err != nil { - return []byte{}, err - } - val, ok := attribs[key] - if !ok { - return []byte{}, &xattr.Error{Op: "mpk.get", Path: path, Name: key, Err: xattr.ENOATTR} - } - return val, nil -} - -// GetInt64 reads a string as int64 from the xattrs -func (b MessagePackBackend) GetInt64(ctx context.Context, path, key string) (int64, error) { - attribs, err := b.loadAttributes(ctx, path, nil) - if err != nil { - return 0, err - } - val, ok := attribs[key] - if !ok { - return 0, &xattr.Error{Op: "mpk.get", Path: path, Name: key, Err: xattr.ENOATTR} - } - i, err := strconv.ParseInt(string(val), 10, 64) - if err != nil { - return 0, err - } - return i, nil -} - -// List retrieves a list of names of extended attributes associated with the -// given path in the file system. -func (b MessagePackBackend) List(ctx context.Context, path string) ([]string, error) { - attribs, err := b.loadAttributes(ctx, path, nil) - if err != nil { - return nil, err - } - keys := []string{} - for k := range attribs { - keys = append(keys, k) - } - return keys, nil -} - -// Set sets one attribute for the given path -func (b MessagePackBackend) Set(ctx context.Context, path, key string, val []byte) error { - return b.SetMultiple(ctx, path, map[string][]byte{key: val}, true) -} - -// SetMultiple sets a set of attribute for the given path -func (b MessagePackBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error { - return b.saveAttributes(ctx, path, attribs, nil, acquireLock) -} - -// Remove an extended attribute key -func (b MessagePackBackend) Remove(ctx context.Context, path, key string, acquireLock bool) error { - return b.saveAttributes(ctx, path, nil, []string{key}, acquireLock) -} - -// AllWithLockedSource reads all extended attributes from the given reader (if possible). -// The path argument is used for storing the data in the cache -func (b MessagePackBackend) AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) { - return b.loadAttributes(ctx, path, source) -} - -func (b MessagePackBackend) saveAttributes(ctx context.Context, path string, setAttribs map[string][]byte, deleteAttribs []string, acquireLock bool) error { - var ( - err error - f readWriteCloseSeekTruncater - ) - ctx, span := tracer.Start(ctx, "saveAttributes") - defer func() { - if err != nil { - span.SetStatus(codes.Error, err.Error()) - } else { - span.SetStatus(codes.Ok, "") - } - span.End() - }() - - lockPath := b.LockfilePath(path) - metaPath := b.MetadataPath(path) - if acquireLock { - _, subspan := tracer.Start(ctx, "lockedfile.OpenFile") - f, err = lockedfile.OpenFile(lockPath, os.O_RDWR|os.O_CREATE, 0600) - subspan.End() - if err != nil { - return err - } - defer f.Close() - } - // Read current state - _, subspan := tracer.Start(ctx, "os.ReadFile") - var msgBytes []byte - msgBytes, err = os.ReadFile(metaPath) - subspan.End() - attribs := map[string][]byte{} - switch { - case err != nil: - if !errors.Is(err, fs.ErrNotExist) { - return err - } - case len(msgBytes) == 0: - // ugh. an empty file? bail out - return errors.New("encountered empty metadata file") - default: - // only unmarshal if we read data - err = msgpack.Unmarshal(msgBytes, &attribs) - if err != nil { - return err - } - } - - // prepare metadata - for key, val := range setAttribs { - attribs[key] = val - } - for _, key := range deleteAttribs { - delete(attribs, key) - } - var d []byte - d, err = msgpack.Marshal(attribs) - if err != nil { - return err - } - - // overwrite file atomically - _, subspan = tracer.Start(ctx, "renameio.Writefile") - err = renameio.WriteFile(metaPath, d, 0600) - if err != nil { - return err - } - subspan.End() - - _, subspan = tracer.Start(ctx, "metaCache.PushToCache") - err = b.metaCache.PushToCache(b.cacheKey(path), attribs) - subspan.End() - return err -} - -func (b MessagePackBackend) loadAttributes(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) { - ctx, span := tracer.Start(ctx, "loadAttributes") - defer span.End() - attribs := map[string][]byte{} - err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) - if err == nil { - return attribs, err - } - - metaPath := b.MetadataPath(path) - var msgBytes []byte - - if source == nil { - // // No cached entry found. Read from storage and store in cache - _, subspan := tracer.Start(ctx, "os.OpenFile") - // source, err = lockedfile.Open(metaPath) - source, err = os.Open(metaPath) - subspan.End() - // // No cached entry found. Read from storage and store in cache - if err != nil { - if os.IsNotExist(err) { - // some of the caller rely on ENOTEXISTS to be returned when the - // actual file (not the metafile) does not exist in order to - // determine whether a node exists or not -> stat the actual node - _, subspan := tracer.Start(ctx, "os.Stat") - _, err := os.Stat(path) - subspan.End() - if err != nil { - return nil, err - } - return attribs, nil // no attributes set yet - } - } - _, subspan = tracer.Start(ctx, "io.ReadAll") - msgBytes, err = io.ReadAll(source) - source.(*os.File).Close() - subspan.End() - } else { - _, subspan := tracer.Start(ctx, "io.ReadAll") - msgBytes, err = io.ReadAll(source) - subspan.End() - } - - if err != nil { - return nil, err - } - if len(msgBytes) == 0 { - return nil, errors.New("encountered empty metadata file") - } - - err = msgpack.Unmarshal(msgBytes, &attribs) - if err != nil { - return nil, err - } - - _, subspan := tracer.Start(ctx, "metaCache.PushToCache") - err = b.metaCache.PushToCache(b.cacheKey(path), attribs) - subspan.End() - if err != nil { - return nil, err - } - - return attribs, nil -} - -// IsMetaFile returns whether the given path represents a meta file -func (MessagePackBackend) IsMetaFile(path string) bool { - return strings.HasSuffix(path, ".mpk") || strings.HasSuffix(path, ".mlock") -} - -// Purge purges the data of a given path -func (b MessagePackBackend) Purge(_ context.Context, path string) error { - if err := b.metaCache.RemoveMetadata(b.cacheKey(path)); err != nil { - return err - } - return os.Remove(b.MetadataPath(path)) -} - -// Rename moves the data for a given path to a new path -func (b MessagePackBackend) Rename(oldPath, newPath string) error { - data := map[string][]byte{} - err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data) - if err == nil { - err = b.metaCache.PushToCache(b.cacheKey(newPath), data) - if err != nil { - return err - } - } - err = b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) - if err != nil { - return err - } - - return os.Rename(b.MetadataPath(oldPath), b.MetadataPath(newPath)) -} - -// MetadataPath returns the path of the file holding the metadata for the given path -func (MessagePackBackend) MetadataPath(path string) string { return path + ".mpk" } - -// LockfilePath returns the path of the lock file -func (MessagePackBackend) LockfilePath(path string) string { return path + ".mlock" } - -// Lock locks the metadata for the given path -func (b MessagePackBackend) Lock(path string) (UnlockFunc, error) { - metaLockPath := b.LockfilePath(path) - mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return nil, err - } - return func() error { - err := mlock.Close() - if err != nil { - return err - } - return os.Remove(metaLockPath) - }, nil -} - -func (b MessagePackBackend) cacheKey(path string) string { - // rootPath is guaranteed to have no trailing slash - // the cache key shouldn't begin with a slash as some stores drop it which can cause - // confusion - return strings.TrimPrefix(path, b.rootPath+"/") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/metadata.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/metadata.go deleted file mode 100644 index 3c95586b22..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/metadata.go +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package metadata - -import ( - "context" - "errors" - "io" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -var tracer trace.Tracer - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/metadata") -} - -var errUnconfiguredError = errors.New("no metadata backend configured. Bailing out") - -type UnlockFunc func() error - -// Backend defines the interface for file attribute backends -type Backend interface { - Name() string - - All(ctx context.Context, path string) (map[string][]byte, error) - Get(ctx context.Context, path, key string) ([]byte, error) - - GetInt64(ctx context.Context, path, key string) (int64, error) - List(ctx context.Context, path string) (attribs []string, err error) - Set(ctx context.Context, path, key string, val []byte) error - SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error - Remove(ctx context.Context, path, key string, acquireLock bool) error - - Lock(path string) (UnlockFunc, error) - Purge(ctx context.Context, path string) error - Rename(oldPath, newPath string) error - IsMetaFile(path string) bool - MetadataPath(path string) string - LockfilePath(path string) string - - AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) -} - -// NullBackend is the default stub backend, used to enforce the configuration of a proper backend -type NullBackend struct{} - -// Name returns the name of the backend -func (NullBackend) Name() string { return "null" } - -// All reads all extended attributes for a node -func (NullBackend) All(ctx context.Context, path string) (map[string][]byte, error) { - return nil, errUnconfiguredError -} - -// Get an extended attribute value for the given key -func (NullBackend) Get(ctx context.Context, path, key string) ([]byte, error) { - return []byte{}, errUnconfiguredError -} - -// GetInt64 reads a string as int64 from the xattrs -func (NullBackend) GetInt64(ctx context.Context, path, key string) (int64, error) { - return 0, errUnconfiguredError -} - -// List retrieves a list of names of extended attributes associated with the -// given path in the file system. -func (NullBackend) List(ctx context.Context, path string) ([]string, error) { - return nil, errUnconfiguredError -} - -// Set sets one attribute for the given path -func (NullBackend) Set(ctx context.Context, path string, key string, val []byte) error { - return errUnconfiguredError -} - -// SetMultiple sets a set of attribute for the given path -func (NullBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) error { - return errUnconfiguredError -} - -// Remove removes an extended attribute key -func (NullBackend) Remove(ctx context.Context, path string, key string, acquireLock bool) error { - return errUnconfiguredError -} - -// Lock locks the metadata for the given path -func (NullBackend) Lock(path string) (UnlockFunc, error) { - return nil, nil -} - -// IsMetaFile returns whether the given path represents a meta file -func (NullBackend) IsMetaFile(path string) bool { return false } - -// Purge purges the data of a given path from any cache that might hold it -func (NullBackend) Purge(_ context.Context, purges string) error { return errUnconfiguredError } - -// Rename moves the data for a given path to a new path -func (NullBackend) Rename(oldPath, newPath string) error { return errUnconfiguredError } - -// MetadataPath returns the path of the file holding the metadata for the given path -func (NullBackend) MetadataPath(path string) string { return "" } - -// LockfilePath returns the path of the lock file -func (NullBackend) LockfilePath(path string) string { return "" } - -// AllWithLockedSource reads all extended attributes from the given reader -// The path argument is used for storing the data in the cache -func (NullBackend) AllWithLockedSource(ctx context.Context, path string, source io.Reader) (map[string][]byte, error) { - return nil, errUnconfiguredError -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix.go deleted file mode 100644 index 18e94cf46c..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build !freebsd - -package prefixes - -// The default namespace for ocis. As non root users can only manipulate -// the user. namespace, which is what is used to store ownCloud specific -// metadata. To prevent name collisions with other apps, we are going to -// introduce a sub namespace "user.ocis." -const ( - OcisPrefix string = "user.ocis." -) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix_freebsd.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix_freebsd.go deleted file mode 100644 index c601a0839d..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/ocis_prefix_freebsd.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build freebsd - -package prefixes - -// On FreeBSD the `user` namespace is implied through a separate syscall argument -// and will fail with invalid argument when you try to start an xattr name with user. or system. -// For that reason we drop the superfluous user. prefix for FreeBSD specifically. -const ( - OcisPrefix string = "ocis." -) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go deleted file mode 100644 index 855b3998ef..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes/prefixes.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package prefixes - -import "github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace" - -// Declare a list of xattr keys -// TODO the below comment is currently copied from the owncloud driver, revisit -// Currently,extended file attributes have four separated -// namespaces (user, trusted, security and system) followed by a dot. -// A non root user can only manipulate the user. namespace, which is what -// we will use to store ownCloud specific metadata. To prevent name -// collisions with other apps We are going to introduce a sub namespace -// "user.ocis." in the xattrs_prefix*.go files. -const ( - TypeAttr string = OcisPrefix + "type" - IDAttr string = OcisPrefix + "id" - ParentidAttr string = OcisPrefix + "parentid" - OwnerIDAttr string = OcisPrefix + "owner.id" - OwnerIDPAttr string = OcisPrefix + "owner.idp" - OwnerTypeAttr string = OcisPrefix + "owner.type" - // the base name of the node - // updated when the file is renamed or moved - NameAttr string = OcisPrefix + "name" - - BlobIDAttr string = OcisPrefix + "blobid" - BlobsizeAttr string = OcisPrefix + "blobsize" - - // statusPrefix is the prefix for the node status - StatusPrefix string = OcisPrefix + "nodestatus" - - // scanPrefix is the prefix for the virus scan status and date - ScanStatusPrefix string = OcisPrefix + "scanstatus" - ScanDatePrefix string = OcisPrefix + "scandate" - - // grantPrefix is the prefix for sharing related extended attributes - GrantPrefix string = OcisPrefix + "grant." - GrantUserAcePrefix string = OcisPrefix + "grant." + ace.UserAcePrefix - GrantGroupAcePrefix string = OcisPrefix + "grant." + ace.GroupAcePrefix - MetadataPrefix string = OcisPrefix + "md." - - // favorite flag, per user - FavPrefix string = OcisPrefix + "fav." - - // a temporary etag for a folder that is removed when the mtime propagation happens - TmpEtagAttr string = OcisPrefix + "tmp.etag" - ReferenceAttr string = OcisPrefix + "cs3.ref" // arbitrary metadata - ChecksumPrefix string = OcisPrefix + "cs." // followed by the algorithm, eg. ocis.cs.sha1 - TrashOriginAttr string = OcisPrefix + "trash.origin" // trash origin - - // we use a single attribute to enable or disable propagation of both: synctime and treesize - // The propagation attribute is set to '1' at the top of the (sub)tree. Propagation will stop at - // that node. - PropagationAttr string = OcisPrefix + "propagation" - - // we need mtime to keep mtime in sync with the metadata - MTimeAttr string = OcisPrefix + "mtime" - // the tree modification time of the tree below this node, - // propagated when synctime_accounting is true and - // user.ocis.propagation=1 is set - // stored as a readable time.RFC3339Nano - TreeMTimeAttr string = OcisPrefix + "tmtime" - - // the deletion/disabled time of a space or node - // used to mark space roots as disabled - // stored as a readable time.RFC3339Nano - DTimeAttr string = OcisPrefix + "dtime" - - // the size of the tree below this node, - // propagated when treesize_accounting is true and - // user.ocis.propagation=1 is set - // stored as uint64, little endian - TreesizeAttr string = OcisPrefix + "treesize" - - // the quota for the storage space / tree, regardless who accesses it - QuotaAttr string = OcisPrefix + "quota" - - // the name given to a storage space. It should not contain any semantics as its only purpose is to be read. - SpaceIDAttr string = OcisPrefix + "space.id" - SpaceNameAttr string = OcisPrefix + "space.name" - SpaceTypeAttr string = OcisPrefix + "space.type" - SpaceDescriptionAttr string = OcisPrefix + "space.description" - SpaceReadmeAttr string = OcisPrefix + "space.readme" - SpaceImageAttr string = OcisPrefix + "space.image" - SpaceAliasAttr string = OcisPrefix + "space.alias" -) diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go deleted file mode 100644 index 619c34b3fe..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/xattrs_backend.go +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package metadata - -import ( - "context" - "io" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/opencloud-eu/reva/v2/pkg/storage/cache" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks" - "github.com/pkg/errors" - "github.com/pkg/xattr" - "github.com/rogpeppe/go-internal/lockedfile" -) - -// XattrsBackend stores the file attributes in extended attributes -type XattrsBackend struct { - rootPath string - metaCache cache.FileMetadataCache -} - -// NewMessageBackend returns a new XattrsBackend instance -func NewXattrsBackend(rootPath string, o cache.Config) XattrsBackend { - return XattrsBackend{ - metaCache: cache.GetFileMetadataCache(o), - } -} - -// Name returns the name of the backend -func (XattrsBackend) Name() string { return "xattrs" } - -// Get an extended attribute value for the given key -// No file locking is involved here as reading a single xattr is -// considered to be atomic. -func (b XattrsBackend) Get(ctx context.Context, path, key string) ([]byte, error) { - attribs := map[string][]byte{} - err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) - if err == nil && len(attribs[key]) > 0 { - return attribs[key], err - } - - return xattr.Get(path, key) -} - -// GetInt64 reads a string as int64 from the xattrs -func (b XattrsBackend) GetInt64(ctx context.Context, filePath, key string) (int64, error) { - attr, err := b.Get(ctx, filePath, key) - if err != nil { - return 0, err - } - v, err := strconv.ParseInt(string(attr), 10, 64) - if err != nil { - return 0, err - } - return v, nil -} - -// List retrieves a list of names of extended attributes associated with the -// given path in the file system. -func (b XattrsBackend) List(ctx context.Context, filePath string) (attribs []string, err error) { - return b.list(ctx, filePath, true) -} - -func (b XattrsBackend) list(ctx context.Context, filePath string, acquireLock bool) (attribs []string, err error) { - attrs, err := listXattr(ctx, filePath) - if err == nil { - return attrs, nil - } - - // E2BIG means the attribute name list exceeds the maximum size the kernel - // returns (64KiB VFS limit, e.g. on XFS). It is not transient: fail fast - // instead of paying for the locked retry below. - if IsErrTooBig(err) { - return nil, err - } - - // listing xattrs failed, try again, either with lock or without - if acquireLock { - f, err := lockedfile.OpenFile(filePath+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return nil, err - } - defer cleanupLockfile(ctx, f) - - } - return listXattr(ctx, filePath) -} - -// All reads all extended attributes for a node, protected by a -// shared file lock -func (b XattrsBackend) All(ctx context.Context, path string) (map[string][]byte, error) { - return b.getAll(ctx, path, false, true) -} - -func (b XattrsBackend) getAll(ctx context.Context, path string, skipCache, acquireLock bool) (map[string][]byte, error) { - attribs := map[string][]byte{} - - if !skipCache { - err := b.metaCache.PullFromCache(b.cacheKey(path), &attribs) - if err == nil { - return attribs, err - } - } - - attrNames, err := b.list(ctx, path, acquireLock) - if err != nil { - return nil, err - } - - if len(attrNames) == 0 { - return attribs, nil - } - - var ( - xerrs = 0 - xerr error - ) - // error handling: Count if there are errors while reading all attribs. - // if there were any, return an error. - attribs = make(map[string][]byte, len(attrNames)) - for _, name := range attrNames { - var val []byte - if val, xerr = xattr.Get(path, name); xerr != nil && !IsAttrUnset(xerr) { - xerrs++ - } else { - attribs[name] = val - } - } - - if xerrs > 0 { - return nil, errors.Wrap(xerr, "Failed to read all xattrs") - } - - err = b.metaCache.PushToCache(b.cacheKey(path), attribs) - if err != nil { - return nil, err - } - - return attribs, nil -} - -// Set sets one attribute for the given path -func (b XattrsBackend) Set(ctx context.Context, path string, key string, val []byte) (err error) { - return b.SetMultiple(ctx, path, map[string][]byte{key: val}, true) -} - -// SetMultiple sets a set of attribute for the given path -func (b XattrsBackend) SetMultiple(ctx context.Context, path string, attribs map[string][]byte, acquireLock bool) (err error) { - if acquireLock { - err := os.MkdirAll(filepath.Dir(path), 0600) - if err != nil { - return err - } - lockedFile, err := lockedfile.OpenFile(b.LockfilePath(path), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer cleanupLockfile(ctx, lockedFile) - } - - // error handling: Count if there are errors while setting the attribs. - // if there were any, return an error. - var ( - xerrs = 0 - xerr error - ) - for key, val := range attribs { - if xerr = xattr.Set(path, key, val); xerr != nil { - // log - xerrs++ - } - } - if xerrs > 0 { - return errors.Wrap(xerr, "Failed to set all xattrs") - } - - attribs, err = b.getAll(ctx, path, true, false) - if err != nil { - return err - } - return b.metaCache.PushToCache(b.cacheKey(path), attribs) -} - -// Remove an extended attribute key -func (b XattrsBackend) Remove(ctx context.Context, path string, key string, acquireLock bool) error { - if acquireLock { - lockedFile, err := lockedfile.OpenFile(path+filelocks.LockFileSuffix, os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer cleanupLockfile(ctx, lockedFile) - } - - err := xattr.Remove(path, key) - if err != nil { - return err - } - attribs, err := b.getAll(ctx, path, true, false) - if err != nil { - return err - } - return b.metaCache.PushToCache(b.cacheKey(path), attribs) -} - -// IsMetaFile returns whether the given path represents a meta file -func (XattrsBackend) IsMetaFile(path string) bool { return strings.HasSuffix(path, ".meta.lock") } - -// Purge purges the data of a given path -func (b XattrsBackend) Purge(ctx context.Context, path string) error { - _, err := os.Stat(path) - if err == nil { - attribs, err := b.getAll(ctx, path, true, true) - if err != nil { - return err - } - - for attr := range attribs { - if strings.HasPrefix(attr, prefixes.OcisPrefix) { - err := xattr.Remove(path, attr) - if err != nil { - return err - } - } - } - } - - return b.metaCache.RemoveMetadata(b.cacheKey(path)) -} - -// Rename moves the data for a given path to a new path -func (b XattrsBackend) Rename(oldPath, newPath string) error { - data := map[string][]byte{} - err := b.metaCache.PullFromCache(b.cacheKey(oldPath), &data) - if err == nil { - err = b.metaCache.PushToCache(b.cacheKey(newPath), data) - if err != nil { - return err - } - } - return b.metaCache.RemoveMetadata(b.cacheKey(oldPath)) -} - -// MetadataPath returns the path of the file holding the metadata for the given path -func (XattrsBackend) MetadataPath(path string) string { return path } - -// LockfilePath returns the path of the lock file -func (XattrsBackend) LockfilePath(path string) string { return path + ".mlock" } - -// Lock locks the metadata for the given path -func (b XattrsBackend) Lock(path string) (UnlockFunc, error) { - metaLockPath := b.LockfilePath(path) - mlock, err := lockedfile.OpenFile(metaLockPath, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return nil, err - } - return func() error { - err := mlock.Close() - if err != nil { - return err - } - return os.Remove(metaLockPath) - }, nil -} - -func cleanupLockfile(ctx context.Context, f *lockedfile.File) { - _ = f.Close() - _ = os.Remove(f.Name()) -} - -// AllWithLockedSource reads all extended attributes from the given reader. -// The path argument is used for storing the data in the cache -func (b XattrsBackend) AllWithLockedSource(ctx context.Context, path string, _ io.Reader) (map[string][]byte, error) { - return b.All(ctx, path) -} - -func (b XattrsBackend) cacheKey(path string) string { - // rootPath is guaranteed to have no trailing slash - // the cache key shouldn't begin with a slash as some stores drop it which can cause - // confusion - return strings.TrimPrefix(path, b.rootPath+"/") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go deleted file mode 100644 index 398a8f1c32..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0001_create_spaces_directory_structure.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "context" - "errors" - "os" - "path/filepath" - - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" -) - -func init() { - registerMigration("0001", Migration0001{}) -} - -type Migration0001 struct{} - -// Migrate creates the spaces directory structure -func (m Migration0001) Migrate(migrator *Migrator) (Result, error) { - migrator.log.Info().Msg("Migrating spaces directory structure...") - - // create spaces folder and iterate over existing nodes to populate it - nodesPath := filepath.Join(migrator.lu.InternalRoot(), "nodes") - fi, err := os.Stat(nodesPath) - if err == nil && fi.IsDir() { - f, err := os.Open(nodesPath) - if err != nil { - return stateFailed, err - } - nodes, err := f.Readdir(0) - if err != nil { - return stateFailed, err - } - - for _, n := range nodes { - nodePath := filepath.Join(nodesPath, n.Name()) - - attr, err := migrator.lu.MetadataBackend().Get(context.Background(), nodePath, prefixes.ParentidAttr) - if err == nil && string(attr) == node.RootID { - if err := m.moveNode(migrator, n.Name(), n.Name()); err != nil { - migrator.log.Error().Err(err). - Str("space", n.Name()). - Msg("could not move space") - continue - } - m.linkSpaceNode(migrator, "personal", n.Name()) - } - } - // TODO delete nodesPath if empty - } - return stateSucceeded, nil -} - -// Rollback is not implemented -func (Migration0001) Rollback(_ *Migrator) (Result, error) { - return stateFailed, errors.New("rollback not implemented") -} - -func (m Migration0001) moveNode(migrator *Migrator, spaceID, nodeID string) error { - dirPath := filepath.Join(migrator.lu.InternalRoot(), "nodes", nodeID) - f, err := os.Open(dirPath) - if err != nil { - return err - } - children, err := f.Readdir(0) - if err != nil { - return err - } - for _, child := range children { - old := filepath.Join(migrator.lu.InternalRoot(), "nodes", child.Name()) - new := filepath.Join(migrator.lu.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(child.Name(), 4, 2)) - if err := os.Rename(old, new); err != nil { - migrator.log.Error().Err(err). - Str("space", spaceID). - Str("nodes", child.Name()). - Str("oldpath", old). - Str("newpath", new). - Msg("could not rename node") - } - if child.IsDir() { - if err := m.moveNode(migrator, spaceID, child.Name()); err != nil { - return err - } - } - } - return nil -} - -// linkSpace creates a new symbolic link for a space with the given type st, and node id -func (m Migration0001) linkSpaceNode(migrator *Migrator, spaceType, spaceID string) { - spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, spaceID) - expectedTarget := "../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) - linkTarget, err := os.Readlink(spaceTypesPath) - if errors.Is(err, os.ErrNotExist) { - err = os.Symlink(expectedTarget, spaceTypesPath) - if err != nil { - migrator.log.Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Msg("could not create symlink") - } - } else { - if err != nil { - migrator.log.Error().Err(err). - Str("space_type", spaceType). - Str("space", spaceID). - Msg("could not read symlink") - } - if linkTarget != expectedTarget { - migrator.log.Warn(). - Str("space_type", spaceType). - Str("space", spaceID). - Str("expected", expectedTarget). - Str("actual", linkTarget). - Msg("expected a different link target") - } - } -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go deleted file mode 100644 index 5a81f36aa5..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0002_move_spacetypes_to_indexes.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "errors" - "io" - "os" - "path/filepath" - - "github.com/opencloud-eu/reva/v2/pkg/logger" -) - -func init() { - registerMigration("0002", Migration0002{}) -} - -type Migration0002 struct{} - -// Migrate migrates spacetypes to indexes -func (m Migration0002) Migrate(migrator *Migrator) (Result, error) { - migrator.log.Info().Msg("Migrating space types indexes...") - - spaceTypesPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes") - fi, err := os.Stat(spaceTypesPath) - if err == nil && fi.IsDir() { - - f, err := os.Open(spaceTypesPath) - if err != nil { - return stateFailed, err - } - spaceTypes, err := f.Readdir(0) - if err != nil { - return stateFailed, err - } - - for _, st := range spaceTypes { - err := m.moveSpaceType(migrator, st.Name()) - if err != nil { - logger.New().Error().Err(err). - Str("space", st.Name()). - Msg("could not move space") - continue - } - } - - // delete spacetypespath - d, err := os.Open(spaceTypesPath) - if err != nil { - logger.New().Error().Err(err). - Str("spacetypesdir", spaceTypesPath). - Msg("could not open spacetypesdir") - return stateFailed, nil - } - defer d.Close() - _, err = d.Readdirnames(1) // Or f.Readdir(1) - if err == io.EOF { - // directory is empty we can delete - err := os.Remove(spaceTypesPath) - if err != nil { - logger.New().Error().Err(err). - Str("spacetypesdir", d.Name()). - Msg("could not delete") - } - } else { - logger.New().Error().Err(err). - Str("spacetypesdir", d.Name()). - Msg("could not delete, not empty") - } - } - return stateSucceeded, nil -} - -// Rollback is not implemented -func (Migration0002) Rollback(_ *Migrator) (Result, error) { - return stateFailed, errors.New("rollback not implemented") -} - -func (m Migration0002) moveSpaceType(migrator *Migrator, spaceType string) error { - dirPath := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType) - f, err := os.Open(dirPath) - if err != nil { - return err - } - children, err := f.Readdir(0) - if err != nil { - return err - } - for _, child := range children { - old := filepath.Join(migrator.lu.InternalRoot(), "spacetypes", spaceType, child.Name()) - target, err := os.Readlink(old) - if err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldLink", old). - Msg("could not read old symlink") - continue - } - newDir := filepath.Join(migrator.lu.InternalRoot(), "indexes", "by-type", spaceType) - if err := os.MkdirAll(newDir, 0700); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("targetDir", newDir). - Msg("could not read old symlink") - } - newLink := filepath.Join(newDir, child.Name()) - if err := os.Symlink(filepath.Join("..", target), newLink); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldpath", old). - Str("newpath", newLink). - Msg("could not rename node") - continue - } - if err := os.Remove(old); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("nodes", child.Name()). - Str("oldLink", old). - Msg("could not remove old symlink") - continue - } - } - if err := os.Remove(dirPath); err != nil { - logger.New().Error().Err(err). - Str("space", spaceType). - Str("dir", dirPath). - Msg("could not remove spaces folder, folder probably not empty") - } - return nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go deleted file mode 100644 index 97463b2959..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0003_switch_to_messagepack_metadata.go +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "context" - "errors" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/opencloud-eu/reva/v2/pkg/storage/cache" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" -) - -func init() { - registerMigration("0003", Migration0003{}) -} - -type Migration0003 struct{} - -// Migrate migrates the file metadata to the current backend. -// Only the xattrs -> messagepack path is supported. -func (m Migration0003) Migrate(migrator *Migrator) (Result, error) { - bod := lookup.DetectBackendOnDisk(migrator.lu.InternalRoot()) - if bod == "" { - return stateFailed, errors.New("could not detect metadata backend on disk") - } - - if bod != "xattrs" || migrator.lu.MetadataBackend().Name() != "messagepack" { - return stateSucceededRunAgain, nil - } - - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating to messagepack metadata backend...") - xattrs := metadata.NewXattrsBackend(migrator.lu.InternalRoot(), cache.Config{}) - mpk := metadata.NewMessagePackBackend(migrator.lu.InternalRoot(), cache.Config{}) - - spaces, _ := filepath.Glob(filepath.Join(migrator.lu.InternalRoot(), "spaces", "*", "*")) - for _, space := range spaces { - err := filepath.WalkDir(filepath.Join(space, "nodes"), func(path string, _ fs.DirEntry, err error) error { - // Do not continue on error - if err != nil { - return err - } - - if strings.HasSuffix(path, ".mpk") || strings.HasSuffix(path, ".flock") { - // None of our business - return nil - } - - fi, err := os.Lstat(path) - if err != nil { - return err - } - - if !fi.IsDir() && !fi.Mode().IsRegular() { - return nil - } - - mpkPath := mpk.MetadataPath(path) - _, err = os.Stat(mpkPath) - if err == nil { - return nil - } - - attribs, err := xattrs.All(context.Background(), path) - if err != nil { - migrator.log.Error().Err(err).Str("path", path).Msg("error converting file") - return err - } - if len(attribs) == 0 { - return nil - } - - err = mpk.SetMultiple(context.Background(), path, attribs, false) - if err != nil { - migrator.log.Error().Err(err).Str("path", path).Msg("error setting attributes") - return err - } - - for k := range attribs { - err = xattrs.Remove(context.Background(), path, k, false) - if err != nil { - migrator.log.Debug().Err(err).Str("path", path).Msg("error removing xattr") - } - } - - return nil - }) - if err != nil { - migrator.log.Error().Err(err).Msg("error migrating nodes to messagepack metadata backend") - } - } - - migrator.log.Info().Msg("done.") - return stateSucceeded, nil -} - -// Rollback is not implemented -func (Migration0003) Rollback(_ *Migrator) (Result, error) { - return stateFailed, errors.New("rollback not implemented") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go deleted file mode 100644 index 87dd226c0c..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0004_switch_to_messagepack_space_index.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "os" - "path/filepath" - "strings" - - "github.com/shamaton/msgpack/v2" -) - -func init() { - registerMigration("0004", Migration0004{}) -} - -type Migration0004 struct{} - -// Migrate migrates the directory tree based space indexes to messagepack -func (Migration0004) Migrate(migrator *Migrator) (Result, error) { - root := migrator.lu.InternalRoot() - - // migrate user indexes - users, err := os.ReadDir(filepath.Join(root, "indexes", "by-user-id")) - if err != nil { - migrator.log.Warn().Err(err).Msg("error listing user indexes") - } - for _, user := range users { - if !user.IsDir() { - continue - } - id := user.Name() - indexPath := filepath.Join(root, "indexes", "by-user-id", id+".mpk") - dirIndexPath := filepath.Join(root, "indexes", "by-user-id", id) - - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") - err := migrateSpaceIndex(indexPath, dirIndexPath) - if err != nil { - migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - - // migrate group indexes - groups, err := os.ReadDir(filepath.Join(root, "indexes", "by-group-id")) - if err != nil { - migrator.log.Warn().Err(err).Msg("error listing group indexes") - } - for _, group := range groups { - if !group.IsDir() { - continue - } - id := group.Name() - indexPath := filepath.Join(root, "indexes", "by-group-id", id+".mpk") - dirIndexPath := filepath.Join(root, "indexes", "by-group-id", id) - - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") - err := migrateSpaceIndex(indexPath, dirIndexPath) - if err != nil { - migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - - // migrate project indexes - for _, spaceType := range []string{"personal", "project", "share"} { - indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk") - dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType) - - _, err := os.Stat(dirIndexPath) - if err != nil { - continue - } - - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Migrating " + indexPath + " to messagepack index format...") - err = migrateSpaceIndex(indexPath, dirIndexPath) - if err != nil { - migrator.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - - migrator.log.Info().Msg("done.") - return stateSucceeded, nil -} - -func migrateSpaceIndex(indexPath, dirIndexPath string) error { - links := map[string][]byte{} - m, err := filepath.Glob(dirIndexPath + "/*") - if err != nil { - return err - } - for _, match := range m { - link, err := os.Readlink(match) - if err != nil { - continue - } - links[filepath.Base(match)] = []byte(link) - } - - // rewrite index as file - d, err := msgpack.Marshal(links) - if err != nil { - return err - } - err = os.WriteFile(indexPath, d, 0600) - if err != nil { - return err - } - return os.RemoveAll(dirIndexPath) -} - -// Rollback migrates the directory messagepack indexes to symlinks -func (Migration0004) Rollback(m *Migrator) (Result, error) { - root := m.lu.InternalRoot() - - // migrate user indexes - users, err := filepath.Glob(filepath.Join(root, "indexes", "by-user-id", "*.mpk")) - if err != nil { - m.log.Warn().Err(err).Msg("error listing user indexes") - } - for _, indexPath := range users { - dirIndexPath := strings.TrimSuffix(indexPath, ".mpk") - - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...") - err := downSpaceIndex(indexPath, dirIndexPath) - if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - - // migrate group indexes - groups, err := filepath.Glob(filepath.Join(root, "indexes", "by-group-id", "*.mpk")) - if err != nil { - m.log.Warn().Err(err).Msg("error listing group indexes") - } - for _, indexPath := range groups { - dirIndexPath := strings.TrimSuffix(indexPath, ".mpk") - - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...") - err := downSpaceIndex(indexPath, dirIndexPath) - if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - - // migrate project indexes - for _, spaceType := range []string{"personal", "project", "share"} { - indexPath := filepath.Join(root, "indexes", "by-type", spaceType+".mpk") - dirIndexPath := filepath.Join(root, "indexes", "by-type", spaceType) - - _, err := os.Stat(indexPath) - if err != nil || os.IsNotExist(err) { - continue - } - - m.log.Info().Str("root", m.lu.InternalRoot()).Msg("Migrating " + indexPath + " to symlinks index format...") - err = downSpaceIndex(indexPath, dirIndexPath) - if err != nil { - m.log.Error().Err(err).Str("path", dirIndexPath).Msg("error migrating index") - } - } - return stateDown, nil -} - -func downSpaceIndex(indexPath, dirIndexPath string) error { - d, err := os.ReadFile(indexPath) - if err != nil { - return err - } - - links := map[string][]byte{} - err = msgpack.Unmarshal(d, &links) - if err != nil { - return err - } - - err = os.MkdirAll(dirIndexPath, 0700) - if err != nil { - return err - } - for link, target := range links { - err = os.Symlink(string(target), filepath.Join(dirIndexPath, link)) - if err != nil { - return err - } - } - - return os.Remove(indexPath) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go deleted file mode 100644 index d3a7bded47..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/0005_fix_messagepack_space_index_format.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "os" - "path/filepath" - - "github.com/shamaton/msgpack/v2" -) - -func init() { - registerMigration("0005", Migration0005{}) -} - -type Migration0005 struct{} - -// Migrate fixes the messagepack space index data structure -func (Migration0005) Migrate(migrator *Migrator) (Result, error) { - root := migrator.lu.InternalRoot() - - indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk")) - if err != nil { - return stateFailed, err - } - for _, i := range indexes { - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i) - - // Read old-format index - oldData, err := os.ReadFile(i) - if err != nil { - return stateFailed, err - } - oldIndex := map[string][]byte{} - err = msgpack.Unmarshal(oldData, &oldIndex) - if err != nil { - // likely already migrated -> skip - migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i) - continue - } - - // Write new-format index - newIndex := map[string]string{} - for k, v := range oldIndex { - newIndex[k] = string(v) - } - newData, err := msgpack.Marshal(newIndex) - if err != nil { - return stateFailed, err - } - err = os.WriteFile(i, newData, 0600) - if err != nil { - return stateFailed, err - } - } - migrator.log.Info().Msg("done.") - return stateSucceeded, nil -} - -// Rollback rolls back the migration -func (Migration0005) Rollback(migrator *Migrator) (Result, error) { - root := migrator.lu.InternalRoot() - - indexes, err := filepath.Glob(filepath.Join(root, "indexes", "**", "*.mpk")) - if err != nil { - return stateFailed, err - } - for _, i := range indexes { - migrator.log.Info().Str("root", migrator.lu.InternalRoot()).Msg("Fixing index format of " + i) - - oldData, err := os.ReadFile(i) - if err != nil { - return stateFailed, err - } - oldIndex := map[string]string{} - err = msgpack.Unmarshal(oldData, &oldIndex) - if err != nil { - migrator.log.Warn().Str("root", migrator.lu.InternalRoot()).Msg("Invalid index format found in " + i) - continue - } - - // Write new-format index - newIndex := map[string][]byte{} - for k, v := range oldIndex { - newIndex[k] = []byte(v) - } - newData, err := msgpack.Marshal(newIndex) - if err != nil { - return stateFailed, err - } - err = os.WriteFile(i, newData, 0600) - if err != nil { - return stateFailed, err - } - } - migrator.log.Info().Msg("done.") - return stateDown, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/migrator.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/migrator.go deleted file mode 100644 index 482fcd084c..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator/migrator.go +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package migrator - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "sort" - - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/rs/zerolog" -) - -const ( - statePending = "pending" - stateFailed = "failed" - stateSucceeded = "succeeded" - stateDown = "down" - stateSucceededRunAgain = "runagain" -) - -type migration interface { - Migrate(*Migrator) (Result, error) - Rollback(*Migrator) (Result, error) -} - -var migrations = map[string]migration{} - -type migrationStates map[string]MigrationState - -func registerMigration(name string, migration migration) { - migrations[name] = migration -} - -func allMigrations() []string { - ms := make([]string, 0, len(migrations)) - - for k := range migrations { - ms = append(ms, k) - } - - sort.Strings(ms) - return ms -} - -// MigrationState holds the state of a migration -type MigrationState struct { - State string - Message string -} - -// Result represents the result of a migration run -type Result string - -// Migrator runs migrations on an existing decomposedfs -type Migrator struct { - lu node.PathLookup - states migrationStates - log *zerolog.Logger -} - -// New returns a new Migrator instance -func New(lu node.PathLookup, log *zerolog.Logger) Migrator { - return Migrator{ - lu: lu, - log: log, - } -} - -// Migrations returns the list of migrations and their states -func (m *Migrator) Migrations() (map[string]MigrationState, error) { - err := m.readStates() - if err != nil { - return nil, err - } - - states := map[string]MigrationState{} - for _, migration := range allMigrations() { - if s, ok := m.states[migration]; ok { - states[migration] = s - } else { - states[migration] = MigrationState{ - State: statePending, - } - } - } - - return states, nil -} - -// RunMigration runs or rolls back a migration -func (m *Migrator) RunMigration(id string, rollback bool) error { - if _, ok := migrations[id]; !ok { - return fmt.Errorf("invalid migration '%s'", id) - } - - lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - defer lock.Close() - - err = m.readStates() - if err != nil { - return err - } - - var res Result - if !rollback { - m.log.Info().Msg("Running migration " + id + "...") - res, err = migrations[id].Migrate(m) - } else { - m.log.Info().Msg("Rolling back migration " + id + "...") - res, err = migrations[id].Rollback(m) - } - - // write back state - s := m.states[id] - s.State = string(res) - - if err != nil { - m.log.Error().Err(err).Msg("migration " + id + " failed") - s.Message = err.Error() - } - - m.states[id] = s - err = m.writeStates() - if err != nil { - return err - } - m.log.Info().Msg("done") - return nil -} - -// RunMigrations runs all migrations in sequence. Note this sequence must not be changed or it might -// damage existing decomposed fs. -func (m *Migrator) RunMigrations() error { - lock, err := lockedfile.OpenFile(filepath.Join(m.lu.InternalRoot(), ".migrations.lock"), os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - defer lock.Close() - - err = m.readStates() - if err != nil { - return err - } - - for _, migration := range allMigrations() { - s := m.states[migration] - if s.State == stateSucceeded || s.State == stateDown { - continue - } - - res, err := migrations[migration].Migrate(m) - s.State = string(res) - if err != nil { - m.log.Error().Err(err).Msg("migration " + migration + " failed") - s.Message = err.Error() - } - - m.states[migration] = s - err = m.writeStates() - if err != nil { - return err - } - } - return nil -} - -func (m *Migrator) readStates() error { - m.states = migrationStates{} - - d, err := os.ReadFile(filepath.Join(m.lu.InternalRoot(), ".migrations")) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } - - if len(d) > 0 { - err = json.Unmarshal(d, &m.states) - if err != nil { - return err - } - } - - return nil -} - -func (m *Migrator) writeStates() error { - d, err := json.Marshal(m.states) - if err != nil { - m.log.Error().Err(err).Msg("could not marshal migration states") - return nil - } - return os.WriteFile(filepath.Join(m.lu.InternalRoot(), ".migrations"), d, 0600) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/map.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/map.go deleted file mode 100644 index aa1d0cdd24..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/map.go +++ /dev/null @@ -1,45 +0,0 @@ -package mtimesyncedcache - -import "sync" - -type Map[K comparable, V any] struct { - m sync.Map -} - -func (m *Map[K, V]) Delete(key K) { m.m.Delete(key) } - -func (m *Map[K, V]) Load(key K) (value V, ok bool) { - v, ok := m.m.Load(key) - if !ok { - return value, ok - } - return v.(V), ok -} - -func (m *Map[K, V]) LoadAndDelete(key K) (value V, loaded bool) { - v, loaded := m.m.LoadAndDelete(key) - if !loaded { - return value, loaded - } - return v.(V), loaded -} - -func (m *Map[K, V]) LoadOrStore(key K, value V) (actual V, loaded bool) { - a, loaded := m.m.LoadOrStore(key, value) - return a.(V), loaded -} - -func (m *Map[K, V]) Range(f func(key K, value V) bool) { - m.m.Range(func(key, value any) bool { return f(key.(K), value.(V)) }) -} - -func (m *Map[K, V]) Store(key K, value V) { m.m.Store(key, value) } - -func (m *Map[K, V]) Count() int { - l := 0 - m.Range(func(_ K, _ V) bool { - l++ - return true - }) - return l -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/mtimesyncedcache.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/mtimesyncedcache.go deleted file mode 100644 index 65f7bd3f8d..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache/mtimesyncedcache.go +++ /dev/null @@ -1,59 +0,0 @@ -package mtimesyncedcache - -import ( - "sync" - "time" -) - -type Cache[K comparable, T any] struct { - entries Map[K, *entry[T]] -} - -type entry[T any] struct { - mtime time.Time - value T - - mu sync.Mutex -} - -func New[K comparable, T any]() Cache[K, T] { - return Cache[K, T]{ - entries: Map[K, *entry[T]]{}, - } -} - -func (c *Cache[K, T]) Store(key K, mtime time.Time, value T) error { - c.entries.Store(key, &entry[T]{ - mtime: mtime, - value: value, - }) - return nil -} - -func (c *Cache[K, T]) Load(key K) (T, bool) { - entry, ok := c.entries.Load(key) - if !ok { - var t T - return t, false - } - return entry.value, true -} - -func (c *Cache[K, T]) LoadOrStore(key K, mtime time.Time, f func() (T, error)) (T, error) { - e, _ := c.entries.LoadOrStore(key, &entry[T]{}) - - e.mu.Lock() - defer e.mu.Unlock() - if mtime.After(e.mtime) { - e.mtime = mtime - v, err := f() - if err != nil { - var t T - return t, err - } - e.value = v - c.entries.Store(key, e) - } - - return e.value, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/locks.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/locks.go deleted file mode 100644 index 3dd9ad57cf..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/locks.go +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "encoding/json" - "io/fs" - "os" - "path/filepath" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" -) - -// SetLock sets a lock on the node -func (n *Node) SetLock(ctx context.Context, lock *provider.Lock) error { - ctx, span := tracer.Start(ctx, "SetLock") - defer span.End() - lockFilePath := n.LockFilePath() - - // ensure parent path exists - if err := os.MkdirAll(filepath.Dir(lockFilePath), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock") - } - - // get file lock, so that nobody can create the lock in the meantime - fileLock, err := filelocks.AcquireWriteLock(n.InternalPath()) - if err != nil { - return err - } - - defer func() { - rerr := filelocks.ReleaseLock(fileLock) - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - // check if already locked - l, err := n.ReadLock(ctx, true) // we already have a write file lock, so ReadLock() would fail to acquire a read file lock -> skip it - switch err.(type) { - case errtypes.NotFound: - // file not locked, continue - case nil: - if l != nil { - return errtypes.PreconditionFailed("already locked") - } - default: - return errors.Wrap(err, "Decomposedfs: could check if file already is locked") - } - - // O_EXCL to make open fail when the file already exists - f, err := os.OpenFile(lockFilePath, os.O_EXCL|os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return errors.Wrap(err, "Decomposedfs: could not create lock file") - } - defer f.Close() - - if err := json.NewEncoder(f).Encode(lock); err != nil { - return errors.Wrap(err, "Decomposedfs: could not write lock file") - } - - return err -} - -// ReadLock reads the lock id for a node -func (n Node) ReadLock(ctx context.Context, skipFileLock bool) (*provider.Lock, error) { - ctx, span := tracer.Start(ctx, "ReadLock") - defer span.End() - - // ensure parent path exists - _, subspan := tracer.Start(ctx, "os.MkdirAll") - err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700) - subspan.End() - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: error creating parent folder for lock") - } - - // the caller of ReadLock already may hold a file lock - if !skipFileLock { - _, subspan := tracer.Start(ctx, "filelocks.AcquireReadLock") - fileLock, err := filelocks.AcquireReadLock(n.InternalPath()) - subspan.End() - - if err != nil { - return nil, err - } - - defer func() { - _, subspan := tracer.Start(ctx, "filelocks.ReleaseLock") - rerr := filelocks.ReleaseLock(fileLock) - subspan.End() - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - } - - _, subspan = tracer.Start(ctx, "os.Open") - f, err := os.Open(n.LockFilePath()) - subspan.End() - - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, errtypes.NotFound("no lock found") - } - return nil, errors.Wrap(err, "Decomposedfs: could not open lock file") - } - defer f.Close() - - lock := &provider.Lock{} - if err := json.NewDecoder(f).Decode(lock); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not decode lock file, ignoring") - return nil, errors.Wrap(err, "Decomposedfs: could not read lock file") - } - - // lock already expired - if lock.Expiration != nil && time.Now().After(time.Unix(int64(lock.Expiration.Seconds), int64(lock.Expiration.Nanos))) { - - _, subspan = tracer.Start(ctx, "os.Remove") - err = os.Remove(f.Name()) - subspan.End() - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: could not remove expired lock file") - } - // we successfully deleted the expired lock - return nil, errtypes.NotFound("no lock found") - } - - return lock, nil -} - -// RefreshLock refreshes the node's lock -func (n *Node) RefreshLock(ctx context.Context, lock *provider.Lock, existingLockID string) error { - ctx, span := tracer.Start(ctx, "RefreshLock") - defer span.End() - - // ensure parent path exists - if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock") - } - fileLock, err := filelocks.AcquireWriteLock(n.InternalPath()) - - if err != nil { - return err - } - - defer func() { - rerr := filelocks.ReleaseLock(fileLock) - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - f, err := os.OpenFile(n.LockFilePath(), os.O_RDWR, os.ModeExclusive) - switch { - case errors.Is(err, fs.ErrNotExist): - return errtypes.PreconditionFailed("lock does not exist") - case err != nil: - return errors.Wrap(err, "Decomposedfs: could not open lock file") - } - defer f.Close() - - readLock := &provider.Lock{} - if err := json.NewDecoder(f).Decode(readLock); err != nil { - return errors.Wrap(err, "Decomposedfs: could not read lock") - } - - // check refresh lockID match - if existingLockID == "" && readLock.LockId != lock.LockId { - return errtypes.Aborted("mismatching lock ID") - } - - // check if UnlockAndRelock sends the correct lockID - if existingLockID != "" && readLock.LockId != existingLockID { - return errtypes.Aborted("mismatching existing lock ID") - } - - if ok, err := isLockModificationAllowed(ctx, readLock, lock); !ok { - return err - } - - // Rewind to the beginning of the file before writing a refreshed lock - _, err = f.Seek(0, 0) - if err != nil { - return errors.Wrap(err, "could not seek to the beginning of the lock file") - } - if err := json.NewEncoder(f).Encode(lock); err != nil { - return errors.Wrap(err, "Decomposedfs: could not write lock file") - } - - return err -} - -// Unlock unlocks the node -func (n *Node) Unlock(ctx context.Context, lock *provider.Lock) error { - ctx, span := tracer.Start(ctx, "Unlock") - defer span.End() - - // ensure parent path exists - if err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: error creating parent folder for lock") - } - fileLock, err := filelocks.AcquireWriteLock(n.InternalPath()) - - if err != nil { - return err - } - - defer func() { - rerr := filelocks.ReleaseLock(fileLock) - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - f, err := os.OpenFile(n.LockFilePath(), os.O_RDONLY, os.ModeExclusive) - switch { - case errors.Is(err, fs.ErrNotExist): - return errtypes.Aborted("lock does not exist") - case err != nil: - return errors.Wrap(err, "Decomposedfs: could not open lock file") - } - defer f.Close() - - oldLock := &provider.Lock{} - if err := json.NewDecoder(f).Decode(oldLock); err != nil { - return errors.Wrap(err, "Decomposedfs: could not read lock") - } - - // check lock - if lock == nil || (oldLock.LockId != lock.LockId) { - return errtypes.Locked(oldLock.LockId) - } - - if ok, err := isLockModificationAllowed(ctx, oldLock, lock); !ok { - return err - } - - if err = os.Remove(f.Name()); err != nil { - return errors.Wrap(err, "Decomposedfs: could not remove lock file") - } - return err -} - -// CheckLock compares the context lock with the node lock -func (n *Node) CheckLock(ctx context.Context) error { - ctx, span := tracer.Start(ctx, "CheckLock") - defer span.End() - contextLock, _ := ctxpkg.ContextGetLockID(ctx) - diskLock, _ := n.ReadLock(ctx, false) - if diskLock != nil { - switch contextLock { - case "": - return errtypes.Locked(diskLock.LockId) // no lockid in request - case diskLock.LockId: - return nil // ok - default: - return errtypes.Aborted("mismatching lock") - } - } - if contextLock != "" { - return errtypes.Aborted("not locked") // no lock on disk. why is there a lockid in the context - } - return nil // ok -} - -func readLocksIntoOpaque(ctx context.Context, n *Node, ri *provider.ResourceInfo) error { - lock, err := n.ReadLock(ctx, false) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not read lock") - return err - } - - // reencode to ensure valid json - var b []byte - if b, err = json.Marshal(lock); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("Decomposedfs: could not marshal locks") - } - if ri.Opaque == nil { - ri.Opaque = &types.Opaque{ - Map: map[string]*types.OpaqueEntry{}, - } - } - ri.Opaque.Map["lock"] = &types.OpaqueEntry{ - Decoder: "json", - Value: b, - } - ri.Lock = lock - return err -} - -func (n *Node) hasLocks(ctx context.Context) bool { - _, err := os.Stat(n.LockFilePath()) // FIXME better error checking - return err == nil -} - -func isLockModificationAllowed(ctx context.Context, oldLock *provider.Lock, newLock *provider.Lock) (bool, error) { - if oldLock.Type == provider.LockType_LOCK_TYPE_SHARED { - return true, nil - } - - appNameEquals := oldLock.AppName == newLock.AppName - if !appNameEquals { - return false, errtypes.PermissionDenied("app names of the locks are mismatching") - } - - var lockUserEquals, contextUserEquals bool - if oldLock.User == nil && newLock.GetUser() == nil { - // no user lock set - lockUserEquals = true - contextUserEquals = true - } else { - lockUserEquals = utils.UserIDEqual(oldLock.User, newLock.GetUser()) - if !lockUserEquals { - return false, errtypes.PermissionDenied("users of the locks are mismatching") - } - - u := ctxpkg.ContextMustGetUser(ctx) - contextUserEquals = utils.UserIDEqual(oldLock.User, u.Id) - if !contextUserEquals { - return false, errtypes.PermissionDenied("lock holder and current user are mismatching") - } - } - - return appNameEquals && lockUserEquals && contextUserEquals, nil - -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go deleted file mode 100644 index 27b27d78d2..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node.go +++ /dev/null @@ -1,1561 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "crypto/md5" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "fmt" - "hash" - "hash/adler32" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/mime" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/grants" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -var tracer trace.Tracer - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/node") -} - -// Define keys and values used in the node metadata -const ( - LockdiscoveryKey = "lockdiscovery" - FavoriteKey = "http://owncloud.org/ns/favorite" - ShareTypesKey = "http://owncloud.org/ns/share-types" - ChecksumsKey = "http://owncloud.org/ns/checksums" - UserShareType = "0" - QuotaKey = "quota" - - QuotaUnlimited = "0" - QuotaUncalculated = "-1" - QuotaUnknown = "-2" - - // TrashIDDelimiter represents the characters used to separate the nodeid and the deletion time. - TrashIDDelimiter = ".T." - RevisionIDDelimiter = ".REV." - - // RootID defines the root node's ID - RootID = "root" - - // ProcessingStatus is the name of the status when processing a file - ProcessingStatus = "processing:" -) - -type TimeManager interface { - // OverrideMTime overrides the mtime of the node, either on the node itself or in the given attributes, depending on the implementation - OverrideMtime(ctx context.Context, n *Node, attrs *Attributes, mtime time.Time) error - - // MTime returns the mtime of the node - MTime(ctx context.Context, n *Node) (time.Time, error) - // SetMTime sets the mtime of the node - SetMTime(ctx context.Context, n *Node, mtime *time.Time) error - - // TMTime returns the tmtime of the node - TMTime(ctx context.Context, n *Node) (time.Time, error) - // SetTMTime sets the tmtime of the node - SetTMTime(ctx context.Context, n *Node, tmtime *time.Time) error - - // CTime returns the ctime of the node - CTime(ctx context.Context, n *Node) (time.Time, error) - - // DTime returns the deletion time of the node - DTime(ctx context.Context, n *Node) (time.Time, error) - // SetDTime sets the deletion time of the node - SetDTime(ctx context.Context, n *Node, mtime *time.Time) error -} - -// Tree is used to manage a tree hierarchy -type Tree interface { - Setup() error - - GetMD(ctx context.Context, node *Node) (os.FileInfo, error) - ListFolder(ctx context.Context, node *Node) ([]*Node, error) - // CreateHome(owner *userpb.UserId) (n *Node, err error) - CreateDir(ctx context.Context, node *Node) (err error) - TouchFile(ctx context.Context, node *Node, markprocessing bool, mtime string) error - // CreateReference(ctx context.Context, node *Node, targetURI *url.URL) error - Move(ctx context.Context, oldNode *Node, newNode *Node) (err error) - Delete(ctx context.Context, node *Node) (err error) - RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, target *Node) (*Node, *Node, func() error, error) - PurgeRecycleItemFunc(ctx context.Context, spaceid, key, purgePath string) (*Node, func() error, error) - - InitNewNode(ctx context.Context, n *Node, fsize uint64) (metadata.UnlockFunc, error) - - // TODO(OCISDEV-901): remove WriteBlob once FinishUploadDecomposed is replaced by the upload coordinator (OCISDEV-900). - WriteBlob(node *Node, source string) error - WriteBlobFromReader(node *Node, r io.Reader, size int64) error - ReadBlob(node *Node) (io.ReadCloser, error) - DeleteBlob(node *Node) error - - BuildSpaceIDIndexEntry(spaceID, nodeID string) string - ResolveSpaceIDIndexEntry(spaceID, entry string) (string, string, error) - - Propagate(ctx context.Context, node *Node, sizeDiff int64) (err error) -} - -// PathLookup defines the interface for the lookup component -type PathLookup interface { - NodeFromSpaceID(ctx context.Context, spaceID string) (n *Node, err error) - NodeFromResource(ctx context.Context, ref *provider.Reference) (*Node, error) - NodeFromID(ctx context.Context, id *provider.ResourceId) (n *Node, err error) - - NodeIDFromParentAndName(ctx context.Context, n *Node, name string) (string, error) - - GenerateSpaceID(spaceType string, owner *userpb.User) (string, error) - - InternalRoot() string - InternalPath(spaceID, nodeID string) string - Path(ctx context.Context, n *Node, hasPermission PermissionFunc) (path string, err error) - MetadataBackend() metadata.Backend - TimeManager() TimeManager - ReadBlobIDAndSizeAttr(ctx context.Context, path string, attrs Attributes) (string, int64, error) - TypeFromPath(ctx context.Context, path string) provider.ResourceType - CopyMetadataWithSourceLock(ctx context.Context, sourcePath, targetPath string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), lockedSource *lockedfile.File, acquireTargetLock bool) (err error) - CopyMetadata(ctx context.Context, src, target string, filter func(attributeName string, value []byte) (newValue []byte, copy bool), acquireTargetLock bool) (err error) -} - -type IDCacher interface { - CacheID(ctx context.Context, spaceID, nodeID, val string) error - GetCachedID(ctx context.Context, spaceID, nodeID string) (string, bool) -} - -// Node represents a node in the tree and provides methods to get a Parent or Child instance -type Node struct { - SpaceID string - ParentID string - ID string - Name string - Blobsize int64 - BlobID string - owner *userpb.UserId - Exists bool - SpaceRoot *Node - - lu PathLookup - xattrsCache map[string][]byte - nodeType *provider.ResourceType -} - -// New returns a new instance of Node -func New(spaceID, id, parentID, name string, blobsize int64, blobID string, t provider.ResourceType, owner *userpb.UserId, lu PathLookup) *Node { - if blobID == "" { - blobID = uuid.New().String() - } - return &Node{ - SpaceID: spaceID, - ID: id, - ParentID: parentID, - Name: name, - Blobsize: blobsize, - owner: owner, - lu: lu, - BlobID: blobID, - nodeType: &t, - } -} - -func (n *Node) MarshalJSON() ([]byte, error) { - return json.Marshal(&struct { - Name string `json:"name"` - ID string `json:"id"` - SpaceID string `json:"spaceID"` - ParentID string `json:"parentID"` - BlobID string `json:"blobID"` - BlobSize int64 `json:"blobSize"` - Exists bool `json:"exists"` - }{ - Name: n.Name, - ID: n.ID, - SpaceID: n.SpaceID, - ParentID: n.ParentID, - BlobID: n.BlobID, - BlobSize: n.Blobsize, - Exists: n.Exists, - }) -} - -// Type returns the node's resource type -func (n *Node) Type(ctx context.Context) provider.ResourceType { - _, span := tracer.Start(ctx, "Type") - defer span.End() - if n.nodeType != nil { - return *n.nodeType - } - - t := provider.ResourceType_RESOURCE_TYPE_INVALID - - // Try to read from xattrs - typeAttr, err := n.XattrInt32(ctx, prefixes.TypeAttr) - if err == nil { - t = provider.ResourceType(typeAttr) - n.nodeType = &t - return t - } - - // Fall back to checking on disk - fi, err := os.Lstat(n.InternalPath()) - if err != nil { - return t - } - - switch { - case fi.IsDir(): - if _, err = n.Xattr(ctx, prefixes.ReferenceAttr); err == nil { - t = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } else { - t = provider.ResourceType_RESOURCE_TYPE_CONTAINER - } - case fi.Mode().IsRegular(): - t = provider.ResourceType_RESOURCE_TYPE_FILE - case fi.Mode()&os.ModeSymlink != 0: - t = provider.ResourceType_RESOURCE_TYPE_SYMLINK - // TODO reference using ext attr on a symlink - // nodeType = provider.ResourceType_RESOURCE_TYPE_REFERENCE - } - n.nodeType = &t - return t -} - -// SetType sets the type of the node. -func (n *Node) SetType(t provider.ResourceType) { - n.nodeType = &t -} - -// NodeMetadata writes the Node metadata to disk and allows passing additional attributes -func (n *Node) NodeMetadata(ctx context.Context) Attributes { - attribs := Attributes{} - attribs.SetInt64(prefixes.TypeAttr, int64(n.Type(ctx))) - attribs.SetString(prefixes.ParentidAttr, n.ParentID) - attribs.SetString(prefixes.NameAttr, n.Name) - if n.Type(ctx) == provider.ResourceType_RESOURCE_TYPE_FILE { - attribs.SetString(prefixes.BlobIDAttr, n.BlobID) - attribs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) - } - return attribs -} - -// SetOwner sets the space owner on the node -func (n *Node) SetOwner(owner *userpb.UserId) { - n.SpaceRoot.owner = owner -} - -// SpaceOwnerOrManager returns the space owner of the space. If no owner is set -// one of the space managers is returned instead. -func (n *Node) SpaceOwnerOrManager(ctx context.Context) *userpb.UserId { - owner := n.Owner() - if owner != nil && owner.Type != userpb.UserType_USER_TYPE_SPACE_OWNER { - return owner - } - - // We don't have an owner set. Find a manager instead. - grants, err := n.SpaceRoot.ListGrants(ctx) - if err != nil { - return nil - } - for _, grant := range grants { - if grant.Permissions.Stat && grant.Permissions.ListContainer && grant.Permissions.InitiateFileDownload { - return grant.GetGrantee().GetUserId() - } - } - - return nil -} - -// ReadNode creates a new instance from an id and checks if it exists -func ReadNode(ctx context.Context, lu PathLookup, spaceID, nodeID string, canListDisabledSpace bool, spaceRoot *Node, skipParentCheck bool) (*Node, error) { - ctx, span := tracer.Start(ctx, "ReadNode") - defer span.End() - var err error - - if spaceRoot == nil { - // read space root - spaceRoot = &Node{ - SpaceID: spaceID, - lu: lu, - ID: spaceID, - } - spaceRoot.SpaceRoot = spaceRoot - spaceRoot.owner, err = spaceRoot.readOwner(ctx) - switch { - case metadata.IsNotExist(err): - return spaceRoot, nil // swallow not found, the node defaults to exists = false - case err != nil: - return nil, err - } - spaceRoot.Exists = true - - // lookup name in extended attributes - spaceRoot.Name, err = spaceRoot.XattrString(ctx, prefixes.NameAttr) - if err != nil { - return nil, err - } - } - - // TODO ReadNode should not check permissions - if !canListDisabledSpace && spaceRoot.IsDisabled(ctx) { - // no permission = not found - return nil, errtypes.NotFound(spaceID) - } - - // if current user cannot stat the root return not found? - // no for shares the root might be a different resource - - // check if this is a space root - if spaceID == nodeID { - return spaceRoot, nil - } - - // are we reading a revision? - revisionSuffix := "" - if strings.Contains(nodeID, RevisionIDDelimiter) { - // verify revision key format - kp := strings.SplitN(nodeID, RevisionIDDelimiter, 2) - if len(kp) == 2 { - // use the actual node for the metadata lookup - nodeID = kp[0] - // remember revision for blob metadata - revisionSuffix = RevisionIDDelimiter + kp[1] - } - } - - // read node - n := &Node{ - SpaceID: spaceID, - lu: lu, - ID: nodeID, - SpaceRoot: spaceRoot, - } - nodePath := n.InternalPath() - - // append back revision to nodeid, even when returning a not existing node - defer func() { - // when returning errors n is nil - if n != nil { - n.ID += revisionSuffix - } - }() - - attrs, err := n.Xattrs(ctx) - switch { - case metadata.IsNotExist(err): - return n, nil // swallow not found, the node defaults to exists = false - case err != nil: - return nil, err - } - n.Exists = true - - n.Name = attrs.String(prefixes.NameAttr) - n.ParentID = attrs.String(prefixes.ParentidAttr) - if n.ParentID == "" { - d, _ := os.ReadFile(lu.MetadataBackend().MetadataPath(n.InternalPath())) - if _, ok := lu.MetadataBackend().(metadata.MessagePackBackend); ok { - appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Str("nodeid", n.ID).Interface("attrs", attrs).Bytes("messagepack", d).Msg("missing parent id") - } - return nil, errtypes.InternalError("Missing parent ID on node") - } - - if revisionSuffix == "" { - n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, nodePath, attrs) - if err != nil { - return nil, err - } - } else { - n.BlobID, n.Blobsize, err = lu.ReadBlobIDAndSizeAttr(ctx, nodePath+revisionSuffix, nil) - if err != nil { - return nil, err - } - } - - return n, nil -} - -// Child returns the child node with the given name -func (n *Node) Child(ctx context.Context, name string) (*Node, error) { - ctx, span := tracer.Start(ctx, "Child") - defer span.End() - - spaceID := n.SpaceID - if spaceID == "" && n.ParentID == "root" { - spaceID = n.ID - } else if n.SpaceRoot != nil { - spaceID = n.SpaceRoot.ID - } - c := &Node{ - SpaceID: spaceID, - lu: n.lu, - ParentID: n.ID, - Name: name, - SpaceRoot: n.SpaceRoot, - } - - nodeID, err := n.lu.NodeIDFromParentAndName(ctx, n, name) - switch { - case metadata.IsNotExist(err) || metadata.IsNotDir(err): - return c, nil // if the file does not exist we return a node that has Exists = false - case err != nil: - return nil, err - } - - c, err = ReadNode(ctx, n.lu, spaceID, nodeID, false, n.SpaceRoot, true) - if err != nil { - return nil, errors.Wrap(err, "could not read child node") - } - - return c, nil -} - -// ParentWithReader returns the parent node -func (n *Node) ParentWithReader(ctx context.Context, r io.Reader) (*Node, error) { - _, span := tracer.Start(ctx, "ParentWithReader") - defer span.End() - if n.ParentID == "" { - return nil, fmt.Errorf("decomposedfs: root has no parent") - } - p := &Node{ - SpaceID: n.SpaceID, - lu: n.lu, - ID: n.ParentID, - SpaceRoot: n.SpaceRoot, - } - - // fill metadata cache using the reader - attrs, err := p.XattrsWithReader(ctx, r) - switch { - case metadata.IsNotExist(err): - return p, nil // swallow not found, the node defaults to exists = false - case err != nil: - return nil, err - } - p.Exists = true - - p.Name = attrs.String(prefixes.NameAttr) - p.ParentID = attrs.String(prefixes.ParentidAttr) - - return p, err -} - -// Parent returns the parent node -func (n *Node) Parent(ctx context.Context) (p *Node, err error) { - return n.ParentWithReader(ctx, nil) -} - -// Owner returns the space owner -func (n *Node) Owner() *userpb.UserId { - return n.SpaceRoot.owner -} - -// readOwner reads the owner from the extended attributes of the space root -// in case either owner id or owner idp are unset we return an error and an empty owner object -func (n *Node) readOwner(ctx context.Context) (*userpb.UserId, error) { - owner := &userpb.UserId{} - - // lookup parent id in extended attributes - var attr string - var err error - // lookup ID in extended attributes - attr, err = n.SpaceRoot.XattrString(ctx, prefixes.OwnerIDAttr) - switch { - case err == nil: - owner.OpaqueId = attr - case metadata.IsAttrUnset(err): - // ignore - default: - return nil, err - } - - // lookup IDP in extended attributes - attr, err = n.SpaceRoot.XattrString(ctx, prefixes.OwnerIDPAttr) - switch { - case err == nil: - owner.Idp = attr - case metadata.IsAttrUnset(err): - // ignore - default: - return nil, err - } - - // lookup type in extended attributes - attr, err = n.SpaceRoot.XattrString(ctx, prefixes.OwnerTypeAttr) - switch { - case err == nil: - owner.Type = utils.UserTypeMap(attr) - case metadata.IsAttrUnset(err): - // ignore - default: - return nil, err - } - - // owner is an optional property - if owner.Idp == "" && owner.OpaqueId == "" { - return nil, nil - } - return owner, nil -} - -// PermissionSet returns the permission set and an accessDenied flag -// for the current user -// the parent nodes are not taken into account -// accessDenied is separate from the resource permissions -// because we only support full denials -func (n *Node) PermissionSet(ctx context.Context) (*provider.ResourcePermissions, bool) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - appctx.GetLogger(ctx).Debug().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Msg("no user in context, returning default permissions") - return NoPermissions(), false - } - if utils.UserEqual(u.Id, n.SpaceRoot.Owner()) { - return OwnerPermissions(), false - } - // read the permissions for the current user from the acls of the current node - if np, accessDenied, err := n.ReadUserPermissions(ctx, u); err == nil { - return np, accessDenied - } - // be defensive, we could have access via another grant - return NoPermissions(), true -} - -// InternalPath returns the internal path of the Node -func (n *Node) InternalPath() string { - return n.lu.InternalPath(n.SpaceID, n.ID) -} - -// ParentPath returns the internal path of the parent of the current node -func (n *Node) ParentPath() string { - return n.lu.InternalPath(n.SpaceID, n.ParentID) -} - -// LockFilePath returns the internal path of the lock file of the node -func (n *Node) LockFilePath() string { - return n.InternalPath() + ".lock" -} - -// CalculateEtag returns a hash of fileid + tmtime (or mtime) -func CalculateEtag(id string, tmTime time.Time) (string, error) { - h := md5.New() - if _, err := io.WriteString(h, id); err != nil { - return "", err - } - /* TODO we could strengthen the etag by adding the blobid, but then all etags would change. we would need a legacy etag check as well - if _, err := io.WriteString(h, n.BlobID); err != nil { - return "", err - } - */ - if tb, err := tmTime.UTC().MarshalBinary(); err == nil { - if _, err := h.Write(tb); err != nil { - return "", err - } - } else { - return "", err - } - return fmt.Sprintf(`"%x"`, h.Sum(nil)), nil -} - -// SetMtimeString sets the mtime and atime of a node to the unixtime parsed from the given string -func (n *Node) SetMtimeString(ctx context.Context, mtime string) error { - mt, err := utils.MTimeToTime(mtime) - if err != nil { - return err - } - return n.SetMtime(ctx, &mt) -} - -// SetMTime writes the UTC mtime to the extended attributes or removes the attribute if nil is passed -func (n *Node) SetMtime(ctx context.Context, t *time.Time) (err error) { - if t == nil { - return n.RemoveXattr(ctx, prefixes.MTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.MTimeAttr, t.UTC().Format(time.RFC3339Nano)) -} - -// SetEtag sets the temporary etag of a node if it differs from the current etag -func (n *Node) SetEtag(ctx context.Context, val string) (err error) { - sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() - var tmTime time.Time - if tmTime, err = n.GetTMTime(ctx); err != nil { - return - } - var etag string - if etag, err = CalculateEtag(n.ID, tmTime); err != nil { - return - } - - // sanitize etag - val = fmt.Sprintf("\"%s\"", strings.Trim(val, "\"")) - if etag == val { - sublog.Debug(). - Str("etag", val). - Msg("ignoring request to update identical etag") - return nil - } - // etag is only valid until the calculated etag changes, is part of propagation - return n.SetXattrString(ctx, prefixes.TmpEtagAttr, val) -} - -// SetFavorite sets the favorite for the current user -// TODO we should not mess with the user here ... the favorites is now a user specific property for a file -// that cannot be mapped to extended attributes without leaking who has marked a file as a favorite -// it is a specific case of a tag, which is user individual as well -// TODO there are different types of tags -// 1. public that are managed by everyone -// 2. private tags that are only visible to the user -// 3. system tags that are only visible to the system -// 4. group tags that are only visible to a group ... -// urgh ... well this can be solved using different namespaces -// 1. public = p: -// 2. private = u:: for user specific -// 3. system = s: for system -// 4. group = g:: -// 5. app? = a:: for apps? -// obviously this only is secure when the u/s/g/a namespaces are not accessible by users in the filesystem -// public tags can be mapped to extended attributes -func (n *Node) SetFavorite(ctx context.Context, uid *userpb.UserId, val string) error { - // the favorite flag is specific to the user, so we need to incorporate the userid - fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - return n.SetXattrString(ctx, fa, val) -} - -// IsDir returns true if the node is a directory -func (n *Node) IsDir(ctx context.Context) bool { - attr, _ := n.XattrInt32(ctx, prefixes.TypeAttr) - return attr == int32(provider.ResourceType_RESOURCE_TYPE_CONTAINER) -} - -// AsResourceInfo return the node as CS3 ResourceInfo -func (n *Node) AsResourceInfo(ctx context.Context, rp *provider.ResourcePermissions, mdKeys, fieldMask []string, returnBasename bool) (ri *provider.ResourceInfo, err error) { - sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() - - var fn string - nodeType := n.Type(ctx) - - var target string - if nodeType == provider.ResourceType_RESOURCE_TYPE_REFERENCE { - target, _ = n.XattrString(ctx, prefixes.ReferenceAttr) - } - - id := &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID} - - switch { - case n.IsSpaceRoot(ctx): - fn = "." // space roots do not have a path as they are referencing themselves - case returnBasename: - fn = n.Name - default: - fn, err = n.lu.Path(ctx, n, NoCheck) - if err != nil { - return nil, err - } - } - - ri = &provider.ResourceInfo{ - Id: id, - Path: fn, - Type: nodeType, - MimeType: mime.Detect(nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER, fn), - Size: uint64(n.Blobsize), - Target: target, - PermissionSet: rp, - Owner: n.Owner(), - ParentId: &provider.ResourceId{ - SpaceId: n.SpaceID, - OpaqueId: n.ParentID, - }, - Name: n.Name, - } - - if n.IsProcessing(ctx) { - ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "status", "processing") - } - - if nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER { - ts, err := n.GetTreeSize(ctx) - if err == nil { - ri.Size = ts - } else { - ri.Size = 0 // make dirs always return 0 if it is unknown - sublog.Debug().Err(err).Msg("could not read treesize") - } - } - - // TODO make etag of files use fileid and checksum - - var tmTime time.Time - if tmTime, err = n.GetTMTime(ctx); err != nil { - sublog.Debug().Err(err).Msg("could not get tmtime") - } - - // use temporary etag if it is set - if b, err := n.XattrString(ctx, prefixes.TmpEtagAttr); err == nil && b != "" { - ri.Etag = fmt.Sprintf(`"%x"`, b) - } else if ri.Etag, err = CalculateEtag(n.ID, tmTime); err != nil { - sublog.Debug().Err(err).Msg("could not calculate etag") - } - - // mtime uses tmtime if present - // TODO expose mtime and tmtime separately? - un := tmTime.UnixNano() - ri.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - - mdKeysMap := make(map[string]struct{}) - for _, k := range mdKeys { - mdKeysMap[k] = struct{}{} - } - - var returnAllMetadata bool - if _, ok := mdKeysMap["*"]; len(mdKeys) == 0 || ok { - returnAllMetadata = true - } - - metadata := map[string]string{} - - fieldMaskKeysMap := make(map[string]struct{}) - for _, k := range fieldMask { - fieldMaskKeysMap[k] = struct{}{} - } - - var returnAllFields bool - if _, ok := fieldMaskKeysMap["*"]; len(fieldMask) == 0 || ok { - returnAllFields = true - } - - // read favorite flag for the current user - if _, ok := mdKeysMap[FavoriteKey]; returnAllMetadata || ok { - favorite := "" - if u, ok := ctxpkg.ContextGetUser(ctx); ok { - // the favorite flag is specific to the user, so we need to incorporate the userid - if uid := u.GetId(); uid != nil { - fa := fmt.Sprintf("%s:%s:%s@%s", prefixes.FavPrefix, utils.UserTypeToString(uid.GetType()), uid.GetOpaqueId(), uid.GetIdp()) - if val, err := n.XattrString(ctx, fa); err == nil { - sublog.Debug(). - Str("favorite", fa). - Msg("found favorite flag") - favorite = val - } - } else { - sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("user has no id") - } - } else { - sublog.Error().Err(errtypes.UserRequired("userrequired")).Msg("error getting user from ctx") - } - metadata[FavoriteKey] = favorite - } - // read locks - // FIXME move to fieldmask - if _, ok := mdKeysMap[LockdiscoveryKey]; returnAllMetadata || ok { - if n.hasLocks(ctx) { - err = readLocksIntoOpaque(ctx, n, ri) - if err != nil { - sublog.Debug().Err(errtypes.InternalError("lockfail")) - } - } - } - - // share indicator - if _, ok := fieldMaskKeysMap["share-types"]; returnAllFields || ok { - granteeTypes := n.getGranteeTypes(ctx) - if len(granteeTypes) > 0 { - // TODO add optional property to CS3 ResourceInfo to transport grants? - var s strings.Builder - first := true - for _, t := range granteeTypes { - if !first { - s.WriteString(",") - } else { - first = false - } - s.WriteString(strconv.Itoa(int(t))) - } - ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "share-types", s.String()) - } - } - - // checksums - // FIXME move to fieldmask - if _, ok := mdKeysMap[ChecksumsKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_FILE) && (returnAllMetadata || ok) { - // TODO which checksum was requested? sha1 adler32 or md5? for now hardcode sha1? - // TODO make ResourceInfo carry multiple checksums - n.readChecksumIntoResourceChecksum(ctx, storageprovider.XSSHA1, ri) - n.readChecksumIntoOpaque(ctx, storageprovider.XSMD5, ri) - n.readChecksumIntoOpaque(ctx, storageprovider.XSAdler32, ri) - } - // quota - // FIXME move to fieldmask - if _, ok := mdKeysMap[QuotaKey]; (nodeType == provider.ResourceType_RESOURCE_TYPE_CONTAINER) && returnAllMetadata || ok { - if n.SpaceRoot != nil && n.SpaceRoot.InternalPath() != "" { - n.SpaceRoot.readQuotaIntoOpaque(ctx, ri) - } - } - - // only read the requested metadata attributes - attrs, err := n.Xattrs(ctx) - if err != nil { - sublog.Error().Err(err).Msg("error getting list of extended attributes") - } else { - for key, value := range attrs { - // filter out non-custom properties - if !strings.HasPrefix(key, prefixes.MetadataPrefix) { - continue - } - // only read when key was requested - k := key[len(prefixes.MetadataPrefix):] - if _, ok := mdKeysMap[k]; returnAllMetadata || ok { - metadata[k] = string(value) - } - - } - } - ri.ArbitraryMetadata = &provider.ArbitraryMetadata{ - Metadata: metadata, - } - - // add virusscan information - if scanned, _, date := n.ScanData(ctx); scanned { - ri.Opaque = utils.AppendPlainToOpaque(ri.Opaque, "scantime", date.Format(time.RFC3339Nano)) - } - - sublog.Debug(). - Interface("ri", ri). - Msg("AsResourceInfo") - - return ri, nil -} - -func (n *Node) readChecksumIntoResourceChecksum(ctx context.Context, algo string, ri *provider.ResourceInfo) { - v, err := n.Xattr(ctx, prefixes.ChecksumPrefix+algo) - switch { - case err == nil: - ri.Checksum = &provider.ResourceChecksum{ - Type: storageprovider.PKG2GRPCXS(algo), - Sum: hex.EncodeToString(v), - } - case metadata.IsAttrUnset(err): - appctx.GetLogger(ctx).Debug().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Str("algorithm", algo).Msg("checksum not set") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Str("algorithm", algo).Msg("could not read checksum") - } -} - -func (n *Node) readChecksumIntoOpaque(ctx context.Context, algo string, ri *provider.ResourceInfo) { - v, err := n.Xattr(ctx, prefixes.ChecksumPrefix+algo) - switch { - case err == nil: - if ri.Opaque == nil { - ri.Opaque = &types.Opaque{ - Map: map[string]*types.OpaqueEntry{}, - } - } - ri.Opaque.Map[algo] = &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte(hex.EncodeToString(v)), - } - case metadata.IsAttrUnset(err): - appctx.GetLogger(ctx).Debug().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Str("algorithm", algo).Msg("checksum not set") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Str("algorithm", algo).Msg("could not read checksum") - } -} - -// quota is always stored on the root node -func (n *Node) readQuotaIntoOpaque(ctx context.Context, ri *provider.ResourceInfo) { - v, err := n.XattrString(ctx, prefixes.QuotaAttr) - switch { - case err == nil: - // make sure we have a proper signed int - // we use the same magic numbers to indicate: - // -1 = uncalculated - // -2 = unknown - // -3 = unlimited - if _, err := strconv.ParseInt(v, 10, 64); err == nil { - if ri.Opaque == nil { - ri.Opaque = &types.Opaque{ - Map: map[string]*types.OpaqueEntry{}, - } - } - ri.Opaque.Map[QuotaKey] = &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte(v), - } - } else { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Str("quota", v).Msg("malformed quota") - } - case metadata.IsAttrUnset(err): - appctx.GetLogger(ctx).Debug().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Msg("quota not set") - default: - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("nodepath", n.InternalPath()).Msg("could not read quota") - } -} - -// HasPropagation checks if the propagation attribute exists and is set to "1" -func (n *Node) HasPropagation(ctx context.Context) (propagation bool) { - if b, err := n.XattrString(ctx, prefixes.PropagationAttr); err == nil { - return b == "1" - } - return false -} - -// IsDisabled returns true when the node has a dmtime attribute set -// only used to check if a space is disabled -// FIXME confusing with the trash logic -func (n *Node) IsDisabled(ctx context.Context) bool { - if _, err := n.GetDTime(ctx); err == nil { - return true - } - return false -} - -// GetTreeSize reads the treesize from the extended attributes -func (n *Node) GetTreeSize(ctx context.Context) (treesize uint64, err error) { - ctx, span := tracer.Start(ctx, "GetTreeSize") - defer span.End() - s, err := n.XattrUint64(ctx, prefixes.TreesizeAttr) - if err != nil { - return 0, err - } - return s, nil -} - -// SetTreeSize writes the treesize to the extended attributes -func (n *Node) SetTreeSize(ctx context.Context, ts uint64) (err error) { - return n.SetXattrString(ctx, prefixes.TreesizeAttr, strconv.FormatUint(ts, 10)) -} - -// GetBlobSize reads the blobsize from the extended attributes -func (n *Node) GetBlobSize(ctx context.Context) (treesize uint64, err error) { - s, err := n.XattrInt64(ctx, prefixes.BlobsizeAttr) - if err != nil { - return 0, err - } - return uint64(s), nil -} - -// SetChecksum writes the checksum with the given checksum type to the extended attributes -func (n *Node) SetChecksum(ctx context.Context, csType string, h hash.Hash) (err error) { - return n.SetXattr(ctx, prefixes.ChecksumPrefix+csType, h.Sum(nil)) -} - -// UnsetTempEtag removes the temporary etag attribute -func (n *Node) UnsetTempEtag(ctx context.Context) (err error) { - return n.RemoveXattr(ctx, prefixes.TmpEtagAttr, true) -} - -func isGrantExpired(g *provider.Grant) bool { - if g.Expiration == nil { - return false - } - return time.Now().After(time.Unix(int64(g.Expiration.Seconds), int64(g.Expiration.Nanos))) -} - -// ReadUserPermissions will assemble the permissions for the current user on the given node without parent nodes -// we indicate if the access was denied by setting a grant with no permissions -func (n *Node) ReadUserPermissions(ctx context.Context, u *userpb.User) (ap *provider.ResourcePermissions, accessDenied bool, err error) { - // check if the current user is the owner - if utils.UserEqual(u.Id, n.Owner()) { - appctx.GetLogger(ctx).Debug().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Msg("user is owner, returning owner permissions") - return OwnerPermissions(), false, nil - } - - ap = &provider.ResourcePermissions{} - - // for an efficient group lookup convert the list of groups to a map - // groups are just strings ... groupnames ... or group ids ??? AAARGH !!! - groupsMap := make(map[string]bool, len(u.Groups)) - for i := range u.Groups { - groupsMap[u.Groups[i]] = true - } - - var g *provider.Grant - - // we read all grantees from the node - var grantees []string - if grantees, err = n.ListGrantees(ctx); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Msg("error listing grantees") - return NoPermissions(), true, err - } - - // instead of making n getxattr syscalls we are going to list the acls and filter them here - // we have two options here: - // 1. we can start iterating over the acls / grants on the node or - // 2. we can iterate over the number of groups - // The current implementation tries to be defensive for cases where users have hundreds or thousands of groups, so we iterate over the existing acls. - userace := prefixes.GrantPrefix + ace.UserAce(u.Id) - userFound := false - for i := range grantees { - switch { - // we only need to find the user once - case !userFound && grantees[i] == userace: - g, err = n.ReadGrant(ctx, grantees[i]) - case strings.HasPrefix(grantees[i], prefixes.GrantGroupAcePrefix): // only check group grantees - gr := strings.TrimPrefix(grantees[i], prefixes.GrantGroupAcePrefix) - if groupsMap[gr] { - g, err = n.ReadGrant(ctx, grantees[i]) - } else { - // no need to check attribute - continue - } - default: - // no need to check attribute - continue - } - - if isGrantExpired(g) { - continue - } - - switch { - case err == nil: - // If all permissions are set to false we have a deny grant - if grants.PermissionsEqual(g.Permissions, &provider.ResourcePermissions{}) { - return NoPermissions(), true, nil - } - AddPermissions(ap, g.GetPermissions()) - case metadata.IsAttrUnset(err): - appctx.GetLogger(ctx).Error().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("grant", grantees[i]).Interface("grantees", grantees).Msg("grant vanished from node after listing") - // continue with next segment - default: - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("grant", grantees[i]).Msg("error reading permissions") - // continue with next segment - } - } - - appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("user", u).Msg("returning aggregated permissions") - return ap, false, nil -} - -// IsDenied checks if the node was denied to that user -func (n *Node) IsDenied(ctx context.Context) bool { - gs, err := n.ListGrants(ctx) - if err != nil { - // be paranoid, resource is denied - return true - } - - u := ctxpkg.ContextMustGetUser(ctx) - isExecutant := func(g *provider.Grantee) bool { - switch g.GetType() { - case provider.GranteeType_GRANTEE_TYPE_USER: - return g.GetUserId().GetOpaqueId() == u.GetId().GetOpaqueId() - case provider.GranteeType_GRANTEE_TYPE_GROUP: - // check gid - gid := g.GetGroupId().GetOpaqueId() - for _, group := range u.Groups { - if gid == group { - return true - } - - } - return false - default: - return false - } - - } - - for _, g := range gs { - if !isExecutant(g.Grantee) { - continue - } - - if grants.PermissionsEqual(g.Permissions, &provider.ResourcePermissions{}) { - // resource is denied - return true - } - } - - // no deny grants - return false -} - -// ListGrantees lists the grantees of the current node -// We don't want to wast time and memory by creating grantee objects. -// The function will return a list of opaque strings that can be used to make a ReadGrant call -func (n *Node) ListGrantees(ctx context.Context) (grantees []string, err error) { - attrs, err := n.Xattrs(ctx) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Msg("error listing attributes") - return nil, err - } - for name := range attrs { - if strings.HasPrefix(name, prefixes.GrantPrefix) { - grantees = append(grantees, name) - } - } - return -} - -// ReadGrant reads a CS3 grant -func (n *Node) ReadGrant(ctx context.Context, grantee string) (g *provider.Grant, err error) { - xattr, err := n.Xattr(ctx, grantee) - if err != nil { - return nil, err - } - var e *ace.ACE - if e, err = ace.Unmarshal(strings.TrimPrefix(grantee, prefixes.GrantPrefix), xattr); err != nil { - return nil, err - } - return e.Grant(), nil -} - -// ReadGrant reads a CS3 grant -func (n *Node) DeleteGrant(ctx context.Context, g *provider.Grant, acquireLock bool) (err error) { - - var attr string - if g.Grantee.Type == provider.GranteeType_GRANTEE_TYPE_GROUP { - attr = prefixes.GrantGroupAcePrefix + g.Grantee.GetGroupId().OpaqueId - } else { - attr = prefixes.GrantUserAcePrefix + g.Grantee.GetUserId().OpaqueId - } - - if err = n.RemoveXattr(ctx, attr, acquireLock); err != nil { - return err - } - - return nil -} - -// Purge removes a node from disk. It does not move it to the trash -func (n *Node) Purge(ctx context.Context) error { - // remove node - if err := utils.RemoveItem(n.InternalPath()); err != nil { - return err - } - - // remove child entry in parent - src := filepath.Join(n.ParentPath(), n.Name) - if err := os.Remove(src); err != nil { - return err - } - - // remove .mpk and .mlock files - return n.lu.MetadataBackend().Purge(ctx, n.InternalPath()) -} - -// ListGrants lists all grants of the current node. -func (n *Node) ListGrants(ctx context.Context) ([]*provider.Grant, error) { - grantees, err := n.ListGrantees(ctx) - if err != nil { - return nil, err - } - - grants := make([]*provider.Grant, 0, len(grantees)) - for _, g := range grantees { - grant, err := n.ReadGrant(ctx, g) - if err != nil { - appctx.GetLogger(ctx). - Error(). - Err(err). - Str("spaceid", n.SpaceID). - Str("nodeid", n.ID). - Str("grantee", g). - Msg("error reading grant") - continue - } - grants = append(grants, grant) - } - return grants, nil -} - -func (n *Node) getGranteeTypes(ctx context.Context) []provider.GranteeType { - types := []provider.GranteeType{} - if g, err := n.ListGrantees(ctx); err == nil { - hasUserShares, hasGroupShares := false, false - for i := range g { - switch { - case !hasUserShares && strings.HasPrefix(g[i], prefixes.GrantUserAcePrefix): - hasUserShares = true - case !hasGroupShares && strings.HasPrefix(g[i], prefixes.GrantGroupAcePrefix): - hasGroupShares = true - case hasUserShares && hasGroupShares: - break - } - } - if hasUserShares { - types = append(types, provider.GranteeType_GRANTEE_TYPE_USER) - } - if hasGroupShares { - types = append(types, provider.GranteeType_GRANTEE_TYPE_GROUP) - } - } - return types -} - -// FindStorageSpaceRoot calls n.Parent() and climbs the tree -// until it finds the space root node and adds it to the node -func (n *Node) FindStorageSpaceRoot(ctx context.Context) error { - if n.SpaceRoot != nil { - return nil - } - var err error - // remember the node we ask for and use parent to climb the tree - parent := n - for { - if parent.IsSpaceRoot(ctx) { - n.SpaceRoot = parent - break - } - if parent, err = parent.Parent(ctx); err != nil { - return err - } - } - return nil -} - -// UnmarkProcessing removes the processing flag from the node -func (n *Node) UnmarkProcessing(ctx context.Context, uploadID string) error { - // we currently have to decrease the counter for every processing run to match the incrases - metrics.UploadProcessing.Sub(1) - - v, _ := n.XattrString(ctx, prefixes.StatusPrefix) - if v != ProcessingStatus+uploadID { - // file started another postprocessing later - do not remove - return nil - } - return n.RemoveXattr(ctx, prefixes.StatusPrefix, true) -} - -// IsProcessing returns true if the node is currently being processed -func (n *Node) IsProcessing(ctx context.Context) bool { - v, err := n.XattrString(ctx, prefixes.StatusPrefix) - return err == nil && strings.HasPrefix(v, ProcessingStatus) -} - -// ProcessingID returns the latest upload session id -func (n *Node) ProcessingID(ctx context.Context) (string, error) { - v, err := n.XattrString(ctx, prefixes.StatusPrefix) - return strings.TrimPrefix(v, ProcessingStatus), err -} - -// IsSpaceRoot checks if the node is a space root -func (n *Node) IsSpaceRoot(ctx context.Context) bool { - return n.ID == n.SpaceID -} - -// SetScanData sets the virus scan info to the node -func (n *Node) SetScanData(ctx context.Context, info string, date time.Time) error { - attribs := Attributes{} - attribs.SetString(prefixes.ScanStatusPrefix, info) - attribs.SetString(prefixes.ScanDatePrefix, date.Format(time.RFC3339Nano)) - return n.SetXattrsWithContext(ctx, attribs, true) -} - -// ScanData returns scanning information of the node -func (n *Node) ScanData(ctx context.Context) (scanned bool, virus string, scantime time.Time) { - ti, _ := n.XattrString(ctx, prefixes.ScanDatePrefix) - if ti == "" { - return // not scanned yet - } - - t, err := time.Parse(time.RFC3339Nano, ti) - if err != nil { - return - } - - i, err := n.XattrString(ctx, prefixes.ScanStatusPrefix) - if err != nil { - return - } - - return true, i, t -} - -// CheckQuota checks if both disk space and available quota are sufficient -// Overwrite must be set to true if the new file replaces the old file e.g. -// when creating a new file version. In such a case the function will -// reduce the used bytes by the old file size and then add the new size. -// If overwrite is false oldSize will be ignored. -var CheckQuota = func(ctx context.Context, spaceRoot *Node, overwrite bool, oldSize, newSize uint64) (quotaSufficient bool, err error) { - used, _ := spaceRoot.GetTreeSize(ctx) - if !enoughDiskSpace(spaceRoot.InternalPath(), newSize) { - return false, errtypes.InsufficientStorage("disk full") - } - quotaByteStr, _ := spaceRoot.XattrString(ctx, prefixes.QuotaAttr) - switch quotaByteStr { - case "": - // if quota is not set, it means unlimited - return true, nil - case QuotaUnlimited: - return true, nil - case QuotaUncalculated: - // treat it as unlimited - return true, nil - case QuotaUnknown: - // treat it as unlimited - return true, nil - } - quotaByte, _ := strconv.ParseUint(quotaByteStr, 10, 64) - if overwrite { - if quotaByte < used-oldSize+newSize { - return false, errtypes.InsufficientStorage("quota exceeded") - } - // if total is smaller than used, total-used could overflow and be bigger than fileSize - } else if newSize > quotaByte-used || quotaByte < used { - return false, errtypes.InsufficientStorage("quota exceeded") - } - return true, nil -} - -func enoughDiskSpace(path string, fileSize uint64) bool { - avalB, err := GetAvailableSize(path) - if err != nil { - return false - } - return avalB > fileSize -} - -// CalculateChecksums calculates the sha1, md5 and adler32 checksums of a file -func CalculateChecksums(ctx context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { - sha1h := sha1.New() - md5h := md5.New() - adler32h := adler32.New() - - _, subspan := tracer.Start(ctx, "os.Open") - f, err := os.Open(path) - subspan.End() - if err != nil { - return nil, nil, nil, err - } - defer f.Close() - - r1 := io.TeeReader(f, sha1h) - r2 := io.TeeReader(r1, md5h) - - _, subspan = tracer.Start(ctx, "io.Copy") - _, err = io.Copy(adler32h, r2) - subspan.End() - if err != nil { - return nil, nil, nil, err - } - - return sha1h, md5h, adler32h, nil -} - -// GetMTime reads the mtime from the extended attributes -func (n *Node) GetMTime(ctx context.Context) (time.Time, error) { - return n.lu.TimeManager().MTime(ctx, n) -} - -// GetTMTime reads the tmtime from the extended attributes -func (n *Node) GetTMTime(ctx context.Context) (time.Time, error) { - return n.lu.TimeManager().TMTime(ctx, n) -} - -// SetTMTime writes the UTC tmtime to the extended attributes or removes the attribute if nil is passed -func (n *Node) SetTMTime(ctx context.Context, t *time.Time) (err error) { - return n.lu.TimeManager().SetTMTime(ctx, n, t) -} - -// GetDTime reads the dmtime from the extended attributes -func (n *Node) GetDTime(ctx context.Context) (time.Time, error) { - return n.lu.TimeManager().DTime(ctx, n) -} - -// SetDTime writes the UTC dmtime to the extended attributes or removes the attribute if nil is passed -func (n *Node) SetDTime(ctx context.Context, t *time.Time) (err error) { - return n.lu.TimeManager().SetDTime(ctx, n, t) -} - -// RevertUpload reverts the upload that created the given revision: it -// restores the revision onto the node and removes the revision, including its -// metadata sidecars. The node's metadata lock is held for the duration of the -// operation. -func (n *Node) RevertUpload(ctx context.Context, versionID string) error { - if versionID == "" { - return errors.New("empty versionID") - } - - lock, err := lockedfile.OpenFile(n.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer lock.Close() - - revisionPath := n.InternalPath() + RevisionIDDelimiter + versionID - if _, err := os.Stat(revisionPath); err != nil { - appctx.GetLogger(ctx).Error().Str("versionpath", revisionPath).Err(err).Msg("revision does not exist") - return err - } - - if err := n.lu.CopyMetadata(ctx, revisionPath, n.InternalPath(), func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr - }, false); err != nil { - appctx.GetLogger(ctx).Info().Str("versionpath", revisionPath).Str("nodepath", n.InternalPath()).Err(err).Msg("restoring revision metadata failed") - return err - } - - if err := os.RemoveAll(revisionPath); err != nil { - appctx.GetLogger(ctx).Info().Str("versionpath", revisionPath).Str("nodepath", n.InternalPath()).Err(err).Msg("error removing version") - return err - } - - // remove the revision's metadata sidecars - if err := os.Remove(n.lu.MetadataBackend().MetadataPath(revisionPath)); err != nil { - appctx.GetLogger(ctx).Warn().Err(err).Str("versionpath", revisionPath).Msg("could not delete revision metadata, continuing") - } - if err := os.Remove(n.lu.MetadataBackend().LockfilePath(revisionPath)); err != nil { - appctx.GetLogger(ctx).Warn().Err(err).Str("versionpath", revisionPath).Msg("could not delete revision metadata lockfile, continuing") - } - if err := n.lu.MetadataBackend().Purge(ctx, revisionPath); err != nil { - appctx.GetLogger(ctx).Warn().Err(err).Str("versionpath", revisionPath).Msg("could not purge revision from cache, continuing") - } - return nil -} - -// DeleteRevision deletes a revision of the node and returns the blob id of the -// deleted revision, so that the caller can delete the blob. If versionID is -// empty the node's current revision is deleted: the latest stored revision is -// restored onto the node (or the node is purged when there is none) and the -// processing flag is removed, unstalling the node. No blob is returned in that -// case. -func (n *Node) DeleteRevision(ctx context.Context, versionID string) (string, error) { - if versionID != "" { - return n.deleteRevision(ctx, versionID) - } - - revisionPath, err := n.getLatestRevision(ctx) - if err != nil { - return "", err - } - - if revisionPath == "" { - // there is no revision - delete the node - lock, err := lockedfile.OpenFile(n.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return "", err - } - defer lock.Close() - if err := n.Purge(ctx); err != nil { - appctx.GetLogger(ctx).Info().Str("nodepath", n.InternalPath()).Err(err).Msg("error purging node") - return "", err - } - return "", nil - } - - latestID := strings.TrimPrefix(revisionPath, n.InternalPath()+RevisionIDDelimiter) - if err := n.RevertUpload(ctx, latestID); err != nil { - return "", err - } - - // we just deleted the current revision - remove processing flag if set - if uploadid, err := n.ProcessingID(ctx); err == nil { - return "", n.UnmarkProcessing(ctx, uploadid) - } - return "", nil -} - -// deleteRevision deletes the revision identified by versionID (the timestamp -// following the node's InternalPath + RevisionIDDelimiter) together with its -// metadata sidecars and returns its blob id. It is a no-op if the revision -// does not exist. -func (n *Node) deleteRevision(ctx context.Context, versionID string) (string, error) { - log := appctx.GetLogger(ctx) - - revisionPath := n.InternalPath() + RevisionIDDelimiter + versionID - if _, err := os.Stat(revisionPath); err != nil { - log.Warn().Str("nodeid", n.ID).Str("versionid", versionID).Msg("revision does not exist, nothing to delete") - return "", nil - } - - lock, err := lockedfile.OpenFile(n.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return "", err - } - defer lock.Close() - - blobID, _, err := n.lu.ReadBlobIDAndSizeAttr(ctx, revisionPath, nil) - if err != nil { - return "", err - } - - if err := os.RemoveAll(revisionPath); err != nil { - return "", err - } - - if err := os.Remove(n.lu.MetadataBackend().MetadataPath(revisionPath)); err != nil { - log.Warn().Err(err).Str("nodeid", n.ID).Str("versionid", versionID).Msg("could not delete revision metadata, continuing") - } - if err := os.Remove(n.lu.MetadataBackend().LockfilePath(revisionPath)); err != nil { - log.Warn().Err(err).Str("nodeid", n.ID).Str("versionid", versionID).Msg("could not delete revision metadata lockfile, continuing") - } - if err := n.lu.MetadataBackend().Purge(ctx, revisionPath); err != nil { - log.Warn().Err(err).Str("nodeid", n.ID).Str("versionid", versionID).Msg("could not purge revision from cache, continuing") - } - - return blobID, nil -} - -func (n *Node) getLatestRevision(ctx context.Context) (string, error) { - revPrefix := n.InternalPath() + RevisionIDDelimiter - revisions, err := filepath.Glob(revPrefix + "*") - if err != nil { - appctx.GetLogger(ctx).Error().Str("nodepath", n.InternalPath()).Err(err).Msg("error reading revisions") - return "", err - } - - revPath, latest := "", time.Time{} - for _, rev := range revisions { - if strings.HasSuffix(rev, ".mpk") || strings.HasSuffix(rev, ".mlock") { - continue - } - revDate, err := time.Parse(time.RFC3339Nano, strings.TrimPrefix(rev, revPrefix)) - if err != nil { - appctx.GetLogger(ctx).Error().Str("nodepath", n.InternalPath()).Err(err).Msg("error parsing revision date") - continue - } - - appctx.GetLogger(ctx).Error().Str("nodepath", n.InternalPath()).Str("revPath", revPath).Interface("time", revDate).Err(err).Msg("error parsing revision date") - if revDate.After(latest) { - latest = revDate - revPath = rev - } - } - return revPath, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_unix.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_unix.go deleted file mode 100644 index 0051557a7e..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_unix.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build !windows -// +build !windows - -package node - -import ( - "syscall" -) - -// GetAvailableSize stats the filesystem and return the available bytes -func GetAvailableSize(path string) (uint64, error) { - stat := syscall.Statfs_t{} - err := syscall.Statfs(path, &stat) - if err != nil { - return 0, err - } - - // convert stat.Bavail to uint64 because it returns an int64 on freebsd - return uint64(stat.Bavail) * uint64(stat.Bsize), nil //nolint:unconvert -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_windows.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_windows.go deleted file mode 100644 index 55febbbd88..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/node_windows.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -//go:build windows -// +build windows - -package node - -import "golang.org/x/sys/windows" - -// GetAvailableSize stats the filesystem and return the available bytes -func GetAvailableSize(path string) (uint64, error) { - var free, total, avail uint64 - pathPtr, err := windows.UTF16PtrFromString(path) - if err != nil { - return 0, err - } - err = windows.GetDiskFreeSpaceEx(pathPtr, &avail, &total, &free) - if err != nil { - return 0, err - } - return avail, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go deleted file mode 100644 index 6b1beb37d3..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/permissions.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "strings" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" -) - -// PermissionFunc should return true when the user has permission to access the node -type PermissionFunc func(*Node) bool - -var ( - // NoCheck doesn't check permissions, returns true always - NoCheck PermissionFunc = func(_ *Node) bool { - return true - } -) - -// NoPermissions represents an empty set of permissions -func NoPermissions() *provider.ResourcePermissions { - return &provider.ResourcePermissions{} -} - -// ShareFolderPermissions defines permissions for the shared jail -func ShareFolderPermissions() *provider.ResourcePermissions { - return &provider.ResourcePermissions{ - // read permissions - ListContainer: true, - Stat: true, - InitiateFileDownload: true, - GetPath: true, - GetQuota: true, - ListFileVersions: true, - } -} - -// OwnerPermissions defines permissions for nodes owned by the user -func OwnerPermissions() *provider.ResourcePermissions { - return &provider.ResourcePermissions{ - // all permissions - AddGrant: true, - CreateContainer: true, - Delete: true, - GetPath: true, - GetQuota: true, - InitiateFileDownload: true, - InitiateFileUpload: true, - ListContainer: true, - ListFileVersions: true, - ListGrants: true, - ListRecycle: true, - Move: true, - PurgeRecycle: true, - RemoveGrant: true, - RestoreFileVersion: true, - RestoreRecycleItem: true, - Stat: true, - UpdateGrant: true, - DenyGrant: true, - } -} - -// ServiceAccountPermissions defines the permissions for nodes when requested by a service account -func ServiceAccountPermissions() *provider.ResourcePermissions { - // TODO: Different permissions for different service accounts - return &provider.ResourcePermissions{ - Stat: true, - ListContainer: true, - GetPath: true, // for search index - InitiateFileUpload: true, // for personal data export - InitiateFileDownload: true, // for full-text-search - RemoveGrant: true, // for share expiry - ListRecycle: true, // for purge-trash-bin command - PurgeRecycle: true, // for purge-trash-bin command - RestoreRecycleItem: true, // for cli restore command - Delete: true, // for cli restore command with replace option - CreateContainer: true, // for space provisioning - AddGrant: true, // for initial project space member assignment - } -} - -// Permissions implements permission checks -type Permissions struct { - lu PathLookup -} - -// NewPermissions returns a new Permissions instance -func NewPermissions(lu PathLookup) *Permissions { - return &Permissions{ - lu: lu, - } -} - -// AssemblePermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes -func (p *Permissions) AssemblePermissions(ctx context.Context, n *Node) (ap *provider.ResourcePermissions, err error) { - return p.assemblePermissions(ctx, n, true) -} - -// AssembleTrashPermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes -func (p *Permissions) AssembleTrashPermissions(ctx context.Context, n *Node) (ap *provider.ResourcePermissions, err error) { - return p.assemblePermissions(ctx, n, false) -} - -// assemblePermissions will assemble the permissions for the current user on the given node, taking into account all parent nodes -func (p *Permissions) assemblePermissions(ctx context.Context, n *Node, failOnTrashedSubtree bool) (ap *provider.ResourcePermissions, err error) { - u, ok := ctxpkg.ContextGetUser(ctx) - if !ok { - return NoPermissions(), nil - } - - if u.GetId().GetType() == userpb.UserType_USER_TYPE_SERVICE { - return ServiceAccountPermissions(), nil - } - - // are we reading a revision? - if strings.Contains(n.ID, RevisionIDDelimiter) { - // verify revision key format - kp := strings.SplitN(n.ID, RevisionIDDelimiter, 2) - if len(kp) != 2 { - return NoPermissions(), errtypes.NotFound(n.ID) - } - // use the actual node for the permission assembly - n.ID = kp[0] - } - - // determine root - rn := n.SpaceRoot - cn := n - ap = &provider.ResourcePermissions{} - - // for an efficient group lookup convert the list of groups to a map - // groups are just strings ... groupnames ... or group ids ??? AAARGH !!! - groupsMap := make(map[string]bool, len(u.Groups)) - for i := range u.Groups { - groupsMap[u.Groups[i]] = true - } - - // for all segments, starting at the leaf - for cn.ID != rn.ID { - if np, accessDenied, err := cn.ReadUserPermissions(ctx, u); err == nil { - // check if we have a denial on this node - if accessDenied { - return np, nil - } - AddPermissions(ap, np) - } else { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", cn.SpaceID).Str("nodeid", cn.ID).Msg("error reading permissions") - // continue with next segment - } - - if cn, err = cn.Parent(ctx); err != nil { - // We get an error but get a parent, but can not read it from disk (eg. it has been deleted already) - if cn != nil { - return ap, errors.Wrap(err, "Decomposedfs: error getting parent for node "+cn.ID) - } - // We do not have a parent, so we assume the next valid parent is the spaceRoot (which must always exist) - cn = n.SpaceRoot - } - if failOnTrashedSubtree && !cn.Exists { - return NoPermissions(), errtypes.NotFound(n.ID) - } - - } - - // for the root node - if np, accessDenied, err := cn.ReadUserPermissions(ctx, u); err == nil { - // check if we have a denial on this node - if accessDenied { - return np, nil - } - AddPermissions(ap, np) - } else { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", cn.SpaceID).Str("nodeid", cn.ID).Msg("error reading root node permissions") - } - - // check if the current user is the owner - if utils.UserIDEqual(u.Id, n.Owner()) { - return OwnerPermissions(), nil - } - - appctx.GetLogger(ctx).Debug().Interface("permissions", ap).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("user", u).Msg("returning agregated permissions") - return ap, nil -} - -// AddPermissions merges a set of permissions into another -// TODO we should use a bitfield for this ... -func AddPermissions(l *provider.ResourcePermissions, r *provider.ResourcePermissions) { - l.AddGrant = l.AddGrant || r.AddGrant - l.CreateContainer = l.CreateContainer || r.CreateContainer - l.Delete = l.Delete || r.Delete - l.GetPath = l.GetPath || r.GetPath - l.GetQuota = l.GetQuota || r.GetQuota - l.InitiateFileDownload = l.InitiateFileDownload || r.InitiateFileDownload - l.InitiateFileUpload = l.InitiateFileUpload || r.InitiateFileUpload - l.ListContainer = l.ListContainer || r.ListContainer - l.ListFileVersions = l.ListFileVersions || r.ListFileVersions - l.ListGrants = l.ListGrants || r.ListGrants - l.ListRecycle = l.ListRecycle || r.ListRecycle - l.Move = l.Move || r.Move - l.PurgeRecycle = l.PurgeRecycle || r.PurgeRecycle - l.RemoveGrant = l.RemoveGrant || r.RemoveGrant - l.RestoreFileVersion = l.RestoreFileVersion || r.RestoreFileVersion - l.RestoreRecycleItem = l.RestoreRecycleItem || r.RestoreRecycleItem - l.Stat = l.Stat || r.Stat - l.UpdateGrant = l.UpdateGrant || r.UpdateGrant - l.DenyGrant = l.DenyGrant || r.DenyGrant -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go deleted file mode 100644 index b34d6fb264..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node/xattrs.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package node - -import ( - "context" - "io" - "io/fs" - "strconv" - "time" - - "github.com/pkg/xattr" -) - -// Attributes is a map of string keys and byte array values -type Attributes map[string][]byte - -// String reads a String value -func (md Attributes) String(key string) string { - return string(md[key]) -} - -// SetString sets a string value -func (md Attributes) SetString(key, val string) { - md[key] = []byte(val) -} - -// Int64 reads an int64 value -func (md Attributes) Int64(key string) (int64, error) { - return strconv.ParseInt(string(md[key]), 10, 64) -} - -// SetInt64 sets an int64 value -func (md Attributes) SetInt64(key string, val int64) { - md[key] = []byte(strconv.FormatInt(val, 10)) -} - -// UInt64 reads an uint64 value -func (md Attributes) UInt64(key string) (uint64, error) { - return strconv.ParseUint(string(md[key]), 10, 64) -} - -// SetInt64 sets an uint64 value -func (md Attributes) SetUInt64(key string, val uint64) { - md[key] = []byte(strconv.FormatUint(val, 10)) -} - -// Time reads a time value -func (md Attributes) Time(key string) (time.Time, error) { - return time.Parse(time.RFC3339Nano, string(md[key])) -} - -// SetXattrs sets multiple extended attributes on the write-through cache/node -func (n *Node) SetXattrsWithContext(ctx context.Context, attribs map[string][]byte, acquireLock bool) (err error) { - _, span := tracer.Start(ctx, "SetXattrsWithContext") - defer span.End() - if n.xattrsCache != nil { - for k, v := range attribs { - n.xattrsCache[k] = v - } - } - - return n.lu.MetadataBackend().SetMultiple(ctx, n.InternalPath(), attribs, acquireLock) -} - -// SetXattrs sets multiple extended attributes on the write-through cache/node -func (n *Node) SetXattrs(attribs map[string][]byte, acquireLock bool) (err error) { - return n.SetXattrsWithContext(context.Background(), attribs, acquireLock) -} - -// SetXattr sets an extended attribute on the write-through cache/node -func (n *Node) SetXattr(ctx context.Context, key string, val []byte) (err error) { - if n.xattrsCache != nil { - n.xattrsCache[key] = val - } - - return n.lu.MetadataBackend().Set(ctx, n.InternalPath(), key, val) -} - -// SetXattrString sets a string extended attribute on the write-through cache/node -func (n *Node) SetXattrString(ctx context.Context, key, val string) (err error) { - if n.xattrsCache != nil { - n.xattrsCache[key] = []byte(val) - } - - return n.lu.MetadataBackend().Set(ctx, n.InternalPath(), key, []byte(val)) -} - -// ResetXattrsCache clears the in-process xattr cache so the next read fetches from disk. -func (n *Node) ResetXattrsCache() { - n.xattrsCache = nil -} - -// RemoveXattr removes an extended attribute from the write-through cache/node -func (n *Node) RemoveXattr(ctx context.Context, key string, acquireLock bool) error { - if n.xattrsCache != nil { - delete(n.xattrsCache, key) - } - return n.lu.MetadataBackend().Remove(ctx, n.InternalPath(), key, acquireLock) -} - -// XattrsWithReader returns the extended attributes of the node. If the attributes have already -// been cached they are not read from disk again. -func (n *Node) XattrsWithReader(ctx context.Context, r io.Reader) (Attributes, error) { - if n.ID == "" { - // Do not try to read the attribute of an empty node. The InternalPath points to the - // base nodes directory in this case. - return Attributes{}, &xattr.Error{Op: "node.XattrsWithReader", Path: n.InternalPath(), Err: xattr.ENOATTR} - } - - if n.xattrsCache != nil { - return n.xattrsCache, nil - } - - var attrs Attributes - var err error - if r != nil { - attrs, err = n.lu.MetadataBackend().AllWithLockedSource(ctx, n.InternalPath(), r) - } else { - attrs, err = n.lu.MetadataBackend().All(ctx, n.InternalPath()) - } - if err != nil { - return nil, err - } - - n.xattrsCache = attrs - return n.xattrsCache, nil -} - -// Xattrs returns the extended attributes of the node. If the attributes have already -// been cached they are not read from disk again. -func (n *Node) Xattrs(ctx context.Context) (Attributes, error) { - return n.XattrsWithReader(ctx, nil) -} - -// Xattr returns an extended attribute of the node. If the attributes have already -// been cached it is not read from disk again. -func (n *Node) Xattr(ctx context.Context, key string) ([]byte, error) { - path := n.InternalPath() - - if path == "" { - // Do not try to read the attribute of an non-existing node - return []byte{}, fs.ErrNotExist - } - - if n.ID == "" { - // Do not try to read the attribute of an empty node. The InternalPath points to the - // base nodes directory in this case. - return []byte{}, &xattr.Error{Op: "node.Xattr", Path: path, Name: key, Err: xattr.ENOATTR} - } - - if n.xattrsCache == nil { - attrs, err := n.lu.MetadataBackend().All(ctx, path) - if err != nil { - return []byte{}, err - } - n.xattrsCache = attrs - } - - if val, ok := n.xattrsCache[key]; ok { - return val, nil - } - // wrap the error as xattr does - return []byte{}, &xattr.Error{Op: "node.Xattr", Path: path, Name: key, Err: xattr.ENOATTR} -} - -// XattrString returns the string representation of an attribute -func (n *Node) XattrString(ctx context.Context, key string) (string, error) { - b, err := n.Xattr(ctx, key) - if err != nil { - return "", err - } - return string(b), nil -} - -// XattrInt32 returns the int32 representation of an attribute -func (n *Node) XattrInt32(ctx context.Context, key string) (int32, error) { - b, err := n.XattrString(ctx, key) - if err != nil { - return 0, err - } - - typeInt, err := strconv.ParseInt(b, 10, 32) - if err != nil { - return 0, err - } - return int32(typeInt), nil -} - -// XattrInt64 returns the int64 representation of an attribute -func (n *Node) XattrInt64(ctx context.Context, key string) (int64, error) { - b, err := n.XattrString(ctx, key) - if err != nil { - return 0, err - } - return strconv.ParseInt(b, 10, 64) -} - -// XattrUint64 returns the uint64 representation of an attribute -func (n *Node) XattrUint64(ctx context.Context, key string) (uint64, error) { - b, err := n.XattrString(ctx, key) - if err != nil { - return 0, err - } - return strconv.ParseUint(b, 10, 64) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options/options.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options/options.go deleted file mode 100644 index d32a929550..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options/options.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package options - -import ( - "path/filepath" - "strings" - "time" - - "github.com/go-viper/mapstructure/v2" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" - "github.com/opencloud-eu/reva/v2/pkg/sharedconf" - "github.com/opencloud-eu/reva/v2/pkg/storage/cache" - "github.com/pkg/errors" -) - -// Option defines a single option function. -type Option func(o *Options) - -// Options defines the available options for this package. -type Options struct { - - // the gateway address - GatewayAddr string `mapstructure:"gateway_addr"` - - // the metadata backend to use, currently supports `xattr` or `ini` - MetadataBackend string `mapstructure:"metadata_backend"` - - // the propagator to use for this fs. currently only `sync` is fully supported, `async` is available as an experimental feature - Propagator string `mapstructure:"propagator"` - // Options specific to the async propagator - AsyncPropagatorOptions AsyncPropagatorOptions `mapstructure:"async_propagator_options"` - - // ocis fs works on top of a dir of uuid nodes - Root string `mapstructure:"root"` - - // the upload directory where uploads in progress are stored - UploadDirectory string `mapstructure:"upload_directory"` - - // UserLayout describes the relative path from the storage's root node to the users home node. - UserLayout string `mapstructure:"user_layout"` - - // ProjectLayout describes the relative path from the storage's root node to the project spaces root directory. - ProjectLayout string `mapstructure:"project_layout"` - - // propagate mtime changes as tmtime (tree modification time) to the parent directory when user.ocis.propagation=1 is set on a node - TreeTimeAccounting bool `mapstructure:"treetime_accounting"` - - // propagate size changes as treesize - TreeSizeAccounting bool `mapstructure:"treesize_accounting"` - - // permissions service to use when checking permissions - PermissionsSVC string `mapstructure:"permissionssvc"` - PermissionsClientTLSMode string `mapstructure:"permissionssvc_tls_mode"` - PermTLSMode pool.TLSMode - - PersonalSpaceAliasTemplate string `mapstructure:"personalspacealias_template"` - PersonalSpacePathTemplate string `mapstructure:"personalspacepath_template"` - GeneralSpaceAliasTemplate string `mapstructure:"generalspacealias_template"` - GeneralSpacePathTemplate string `mapstructure:"generalspacepath_template"` - - AsyncFileUploads bool `mapstructure:"asyncfileuploads"` - - Events EventOptions `mapstructure:"events"` - - Tokens TokenOptions `mapstructure:"tokens"` - - StatCache cache.Config `mapstructure:"statcache"` - FileMetadataCache cache.Config `mapstructure:"filemetadatacache"` - IDCache cache.Config `mapstructure:"idcache"` - - MaxAcquireLockCycles int `mapstructure:"max_acquire_lock_cycles"` - LockCycleDurationFactor int `mapstructure:"lock_cycle_duration_factor"` - MaxConcurrency int `mapstructure:"max_concurrency"` - - MaxQuota uint64 `mapstructure:"max_quota"` - - DisableVersioning bool `mapstructure:"disable_versioning"` - - MountID string `mapstructure:"mount_id"` -} - -// AsyncPropagatorOptions holds the configuration for the async propagator -type AsyncPropagatorOptions struct { - PropagationDelay time.Duration `mapstructure:"propagation_delay"` -} - -// EventOptions are the configurable options for events -type EventOptions struct { - NumConsumers int `mapstructure:"numconsumers"` -} - -// TokenOptions are the configurable option for tokens -type TokenOptions struct { - DownloadEndpoint string `mapstructure:"download_endpoint"` - DataGatewayEndpoint string `mapstructure:"datagateway_endpoint"` - TransferSharedSecret string `mapstructure:"transfer_shared_secret"` - TransferExpires int64 `mapstructure:"transfer_expires"` -} - -// New returns a new Options instance for the given configuration -func New(m map[string]interface{}) (*Options, error) { - o := &Options{} - if err := mapstructure.Decode(m, o); err != nil { - err = errors.Wrap(err, "error decoding conf") - return nil, err - } - - o.GatewayAddr = sharedconf.GetGatewaySVC(o.GatewayAddr) - - if o.MetadataBackend == "" { - o.MetadataBackend = "xattrs" - } - - // ensure user layout has no starting or trailing / - o.UserLayout = strings.Trim(o.UserLayout, "/") - - // c.DataDirectory should never end in / unless it is the root - o.Root = filepath.Clean(o.Root) - - if o.PersonalSpaceAliasTemplate == "" { - o.PersonalSpaceAliasTemplate = "{{.SpaceType}}/{{.User.Username}}" - } - - if o.GeneralSpaceAliasTemplate == "" { - o.GeneralSpaceAliasTemplate = "{{.SpaceType}}/{{.SpaceName | replace \" \" \"-\" | lower}}" - } - - if o.PermissionsClientTLSMode != "" { - var err error - o.PermTLSMode, err = pool.StringToTLSMode(o.PermissionsClientTLSMode) - if err != nil { - return nil, err - } - } else { - sharedOpt := sharedconf.GRPCClientOptions() - var err error - - if o.PermTLSMode, err = pool.StringToTLSMode(sharedOpt.TLSMode); err != nil { - return nil, err - } - } - - if o.MaxConcurrency <= 0 { - o.MaxConcurrency = 5 - } - - if o.Propagator == "" { - o.Propagator = "sync" - } - if o.AsyncPropagatorOptions.PropagationDelay == 0 { - o.AsyncPropagatorOptions.PropagationDelay = 5 * time.Second - } - - if o.UploadDirectory == "" { - o.UploadDirectory = filepath.Join(o.Root, "uploads") - } - - return o, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions/spacepermissions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions/spacepermissions.go deleted file mode 100644 index bf44405337..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions/spacepermissions.go +++ /dev/null @@ -1,169 +0,0 @@ -package permissions - -import ( - "context" - - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - cs3permissions "github.com/cs3org/go-cs3apis/cs3/permissions/v1beta1" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" - "google.golang.org/grpc" -) - -var ( - tracer trace.Tracer -) - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/permissions") -} - -const ( - _spaceTypePersonal = "personal" - _spaceTypeProject = "project" -) - -// PermissionsChecker defines an interface for checking permissions on a Node -type PermissionsChecker interface { - AssemblePermissions(ctx context.Context, n *node.Node) (ap *provider.ResourcePermissions, err error) - AssembleTrashPermissions(ctx context.Context, n *node.Node) (ap *provider.ResourcePermissions, err error) -} - -// CS3PermissionsClient defines an interface for checking permissions against the CS3 permissions service -type CS3PermissionsClient interface { - CheckPermission(ctx context.Context, in *cs3permissions.CheckPermissionRequest, opts ...grpc.CallOption) (*cs3permissions.CheckPermissionResponse, error) -} - -// Permissions manages permissions -type Permissions struct { - item PermissionsChecker // handles item permissions - permissionsSelector pool.Selectable[cs3permissions.PermissionsAPIClient] // handlers space permissions -} - -// NewPermissions returns a new Permissions instance -func NewPermissions(item PermissionsChecker, permissionsSelector pool.Selectable[cs3permissions.PermissionsAPIClient]) Permissions { - return Permissions{item: item, permissionsSelector: permissionsSelector} -} - -// AssemblePermissions is used to assemble file permissions -func (p Permissions) AssemblePermissions(ctx context.Context, n *node.Node) (*provider.ResourcePermissions, error) { - ctx, span := tracer.Start(ctx, "AssemblePermissions") - defer span.End() - return p.item.AssemblePermissions(ctx, n) -} - -// AssembleTrashPermissions is used to assemble file permissions -func (p Permissions) AssembleTrashPermissions(ctx context.Context, n *node.Node) (*provider.ResourcePermissions, error) { - _, span := tracer.Start(ctx, "AssembleTrashPermissions") - defer span.End() - return p.item.AssembleTrashPermissions(ctx, n) -} - -// CreateSpace returns true when the user is allowed to create the space -func (p Permissions) CreateSpace(ctx context.Context, spaceid string) bool { - return p.checkPermission(ctx, "Drives.Create", spaceRef(spaceid)) -} - -// SetSpaceQuota returns true when the user is allowed to change the spaces quota -func (p Permissions) SetSpaceQuota(ctx context.Context, spaceid string, spaceType string) bool { - switch spaceType { - default: - return false // only quotas of personal and project space may be changed - case _spaceTypePersonal: - return p.checkPermission(ctx, "Drives.ReadWritePersonalQuota", spaceRef(spaceid)) - case _spaceTypeProject: - return p.checkPermission(ctx, "Drives.ReadWriteProjectQuota", spaceRef(spaceid)) - } -} - -// ManageSpaceProperties returns true when the user is allowed to change space properties (name/subtitle) -func (p Permissions) ManageSpaceProperties(ctx context.Context, spaceid string) bool { - return p.checkPermission(ctx, "Drives.ReadWrite", spaceRef(spaceid)) -} - -// SpaceAbility returns true when the user is allowed to enable/disable the space -func (p Permissions) SpaceAbility(ctx context.Context, spaceid string) bool { - return p.checkPermission(ctx, "Drives.ReadWriteEnabled", spaceRef(spaceid)) -} - -// ListAllSpaces returns true when the user is allowed to list all spaces -func (p Permissions) ListAllSpaces(ctx context.Context) bool { - return p.checkPermission(ctx, "Drives.List", nil) -} - -// ListSpacesOfUser returns true when the user is allowed to list the spaces of the given user -func (p Permissions) ListSpacesOfUser(ctx context.Context, userid *userv1beta1.UserId) bool { - switch { - case userid == nil: - // there is no filter - return true // TODO: is `true` actually correct here? Shouldn't we check for ListAllSpaces too? - case utils.UserIDEqual(ctxpkg.ContextMustGetUser(ctx).GetId(), userid): - return true - default: - return p.ListAllSpaces(ctx) - } -} - -// DeleteAllSpaces returns true when the user is allowed to delete all spaces -func (p Permissions) DeleteAllSpaces(ctx context.Context) bool { - return p.checkPermission(ctx, "Drives.DeleteProject", nil) -} - -// DeleteAllHomeSpaces returns true when the user is allowed to delete all home spaces -func (p Permissions) DeleteAllHomeSpaces(ctx context.Context) bool { - return p.checkPermission(ctx, "Drives.DeletePersonal", nil) -} - -// checkPermission is used to check a users space permissions -func (p Permissions) checkPermission(ctx context.Context, perm string, ref *provider.Reference) bool { - permissionsClient, err := p.permissionsSelector.Next() - if err != nil { - return false - } - - user := ctxpkg.ContextMustGetUser(ctx) - checkRes, err := permissionsClient.CheckPermission(ctx, &cs3permissions.CheckPermissionRequest{ - Permission: perm, - SubjectRef: &cs3permissions.SubjectReference{ - Spec: &cs3permissions.SubjectReference_UserId{ - UserId: user.Id, - }, - }, - Ref: ref, - }) - if err != nil { - return false - } - - return checkRes.Status.Code == v1beta11.Code_CODE_OK -} - -// IsManager returns true if the given resource permissions evaluate the user as "manager" -func IsManager(rp *provider.ResourcePermissions) bool { - return rp.RemoveGrant -} - -// IsEditor returns true if the given resource permissions evaluate the user as "editor" -func IsEditor(rp *provider.ResourcePermissions) bool { - return rp.InitiateFileUpload -} - -// IsViewer returns true if the given resource permissions evaluate the user as "viewer" -func IsViewer(rp *provider.ResourcePermissions) bool { - return rp.Stat -} - -func spaceRef(spaceid string) *provider.Reference { - return &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: spaceid, - // OpaqueId is the same, no need to transfer it - }, - } -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/recycle.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/recycle.go deleted file mode 100644 index 43a17474fe..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/recycle.go +++ /dev/null @@ -1,460 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - iofs "io/fs" - "os" - "path/filepath" - "strings" - "time" - - "github.com/pkg/errors" - "golang.org/x/sync/errgroup" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" -) - -type DecomposedfsTrashbin struct { - fs *Decomposedfs -} - -// Setup the trashbin -func (tb *DecomposedfsTrashbin) Setup(fs storage.FS) error { - if _, ok := fs.(*Decomposedfs); !ok { - return errors.New("invalid filesystem") - } - tb.fs = fs.(*Decomposedfs) - return nil -} - -// Recycle items are stored inside the node folder and start with the uuid of the deleted node. -// The `.T.` indicates it is a trash item and what follows is the timestamp of the deletion. -// The deleted file is kept in the same location/dir as the original node. This prevents deletes -// from triggering cross storage moves when the trash is accidentally stored on another partition, -// because the admin mounted a different partition there. -// For an efficient listing of deleted nodes the ocis storage driver maintains a 'trash' folder -// with symlinks to trash files for every storagespace. - -// ListRecycle returns the list of available recycle items -// ref -> the space (= resourceid), key -> deleted node id, relativePath = relative to key -func (tb *DecomposedfsTrashbin) ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) { - _, span := tracer.Start(ctx, "ListRecycle") - defer span.End() - if ref == nil || ref.ResourceId == nil || ref.ResourceId.OpaqueId == "" { - return nil, errtypes.BadRequest("spaceid required") - } - if key == "" && relativePath != "" { - return nil, errtypes.BadRequest("key is required when navigating with a path") - } - spaceID := ref.ResourceId.OpaqueId - - sublog := appctx.GetLogger(ctx).With().Str("spaceid", spaceID).Str("key", key).Str("relative_path", relativePath).Logger() - - // check permissions - trashnode, err := tb.fs.lu.NodeFromSpaceID(ctx, spaceID) - if err != nil { - return nil, err - } - rp, err := tb.fs.p.AssembleTrashPermissions(ctx, trashnode) - switch { - case err != nil: - return nil, err - case !rp.ListRecycle: - if rp.Stat { - return nil, errtypes.PermissionDenied(key) - } - return nil, errtypes.NotFound(key) - } - - if key == "" && relativePath == "" { - return tb.listTrashRoot(ctx, spaceID) - } - - // build a list of trash items relative to the given trash root and path - items := make([]*provider.RecycleItem, 0) - - trashRootPath := filepath.Join(tb.getRecycleRoot(spaceID), lookup.Pathify(key, 4, 2)) - originalPath, _, timeSuffix, err := readTrashLink(trashRootPath) - if err != nil { - sublog.Error().Err(err).Str("trashRoot", trashRootPath).Msg("error reading trash link") - return nil, err - } - - origin := "" - attrs, err := tb.fs.lu.MetadataBackend().All(ctx, originalPath) - if err != nil { - return items, err - } - // lookup origin path in extended attributes - if attrBytes, ok := attrs[prefixes.TrashOriginAttr]; ok { - origin = string(attrBytes) - } else { - sublog.Error().Err(err).Str("spaceid", spaceID).Msg("could not read origin path, skipping") - return nil, err - } - - // all deleted items have the same deletion time - var deletionTime *types.Timestamp - if parsed, err := time.Parse(time.RFC3339Nano, timeSuffix); err == nil { - deletionTime = &types.Timestamp{ - Seconds: uint64(parsed.Unix()), - // TODO nanos - } - } else { - sublog.Error().Err(err).Msg("could not parse time format, ignoring") - } - - var size int64 - if relativePath == "" { - // this is the case when we want to directly list a file in the trashbin - nodeType := tb.fs.lu.TypeFromPath(ctx, originalPath) - switch nodeType { - case provider.ResourceType_RESOURCE_TYPE_FILE: - _, size, err = tb.fs.lu.ReadBlobIDAndSizeAttr(ctx, originalPath, nil) - if err != nil { - return items, err - } - case provider.ResourceType_RESOURCE_TYPE_CONTAINER: - size, err = tb.fs.lu.MetadataBackend().GetInt64(ctx, originalPath, prefixes.TreesizeAttr) - if err != nil { - return items, err - } - } - item := &provider.RecycleItem{ - Type: tb.fs.lu.TypeFromPath(ctx, originalPath), - Size: uint64(size), - Key: filepath.Join(key, relativePath), - DeletionTime: deletionTime, - Ref: &provider.Reference{ - Path: filepath.Join(origin, relativePath), - }, - } - items = append(items, item) - return items, err - } - - // we have to read the names and stat the path to follow the symlinks - childrenPath := filepath.Join(originalPath, relativePath) - childrenDir, err := os.Open(childrenPath) - if err != nil { - return nil, err - } - - names, err := childrenDir.Readdirnames(0) - if err != nil { - return nil, err - } - for _, name := range names { - resolvedChildPath, err := filepath.EvalSymlinks(filepath.Join(childrenPath, name)) - if err != nil { - sublog.Error().Err(err).Str("name", name).Msg("could not resolve symlink, skipping") - continue - } - - // reset size - size = 0 - - nodeType := tb.fs.lu.TypeFromPath(ctx, resolvedChildPath) - switch nodeType { - case provider.ResourceType_RESOURCE_TYPE_FILE: - _, size, err = tb.fs.lu.ReadBlobIDAndSizeAttr(ctx, resolvedChildPath, nil) - if err != nil { - sublog.Error().Err(err).Str("name", name).Msg("invalid blob size, skipping") - continue - } - case provider.ResourceType_RESOURCE_TYPE_CONTAINER: - size, err = tb.fs.lu.MetadataBackend().GetInt64(ctx, resolvedChildPath, prefixes.TreesizeAttr) - if err != nil { - sublog.Error().Err(err).Str("name", name).Msg("invalid tree size, skipping") - continue - } - case provider.ResourceType_RESOURCE_TYPE_INVALID: - sublog.Error().Err(err).Str("name", name).Str("resolvedChildPath", resolvedChildPath).Msg("invalid node type, skipping") - continue - } - - item := &provider.RecycleItem{ - Type: nodeType, - Size: uint64(size), - Key: filepath.Join(key, relativePath, name), - DeletionTime: deletionTime, - Ref: &provider.Reference{ - Path: filepath.Join(origin, relativePath, name), - }, - } - items = append(items, item) - } - return items, nil -} - -// readTrashLink returns path, nodeID and timestamp -func readTrashLink(path string) (string, string, string, error) { - link, err := os.Readlink(path) - if err != nil { - return "", "", "", err - } - resolved, err := filepath.EvalSymlinks(path) - if err != nil { - return "", "", "", err - } - // ../../../../../nodes/e5/6c/75/a8/-d235-4cbb-8b4e-48b6fd0f2094.T.2022-02-16T14:38:11.769917408Z - // TODO use filepath.Separator to support windows - link = strings.ReplaceAll(link, "/", "") - // ..........nodese56c75a8-d235-4cbb-8b4e-48b6fd0f2094.T.2022-02-16T14:38:11.769917408Z - if link[0:15] != "..........nodes" || link[51:54] != node.TrashIDDelimiter { - return "", "", "", errtypes.InternalError("malformed trash link") - } - return resolved, link[15:51], link[54:], nil -} - -func (tb *DecomposedfsTrashbin) listTrashRoot(ctx context.Context, spaceID string) ([]*provider.RecycleItem, error) { - log := appctx.GetLogger(ctx) - trashRoot := tb.getRecycleRoot(spaceID) - items := []*provider.RecycleItem{} - subTrees, err := filepath.Glob(trashRoot + "/*") - if err != nil { - return nil, err - } - - numWorkers := tb.fs.o.MaxConcurrency - if len(subTrees) < numWorkers { - numWorkers = len(subTrees) - } - - work := make(chan string, len(subTrees)) - results := make(chan *provider.RecycleItem, len(subTrees)) - - g, ctx := errgroup.WithContext(ctx) - - // Distribute work - g.Go(func() error { - defer close(work) - for _, itemPath := range subTrees { - select { - case work <- itemPath: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - - // Spawn workers that'll concurrently work the queue - for i := 0; i < numWorkers; i++ { - g.Go(func() error { - for subTree := range work { - matches, err := filepath.Glob(subTree + "/*/*/*/*") - if err != nil { - return err - } - - for _, itemPath := range matches { - // TODO can we encode this in the path instead of reading the link? - nodePath, nodeID, timeSuffix, err := readTrashLink(itemPath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Msg("error reading trash link, skipping") - continue - } - - md, err := os.Stat(nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not stat trash item, skipping") - continue - } - - attrs, err := tb.fs.lu.MetadataBackend().All(ctx, nodePath) - if err != nil { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("could not get extended attributes, skipping") - continue - } - - nodeType := tb.fs.lu.TypeFromPath(ctx, nodePath) - if nodeType == provider.ResourceType_RESOURCE_TYPE_INVALID { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("node_path", nodePath).Msg("invalid node type, skipping") - continue - } - - item := &provider.RecycleItem{ - Type: nodeType, - Size: uint64(md.Size()), - Key: nodeID, - } - if deletionTime, err := time.Parse(time.RFC3339Nano, timeSuffix); err == nil { - item.DeletionTime = &types.Timestamp{ - Seconds: uint64(deletionTime.Unix()), - // TODO nanos - } - } else { - log.Error().Err(err).Str("trashRoot", trashRoot).Str("item", itemPath).Str("spaceid", spaceID).Str("nodeid", nodeID).Str("dtime", timeSuffix).Msg("could not parse time format, ignoring") - } - - // lookup origin path in extended attributes - if attr, ok := attrs[prefixes.TrashOriginAttr]; ok { - item.Ref = &provider.Reference{Path: string(attr)} - } else { - log.Error().Str("trashRoot", trashRoot).Str("item", itemPath).Str("spaceid", spaceID).Str("nodeid", nodeID).Str("dtime", timeSuffix).Msg("could not read origin path") - } - - select { - case results <- item: - case <-ctx.Done(): - return ctx.Err() - } - } - } - return nil - }) - } - - // Wait for things to settle down, then close results chan - go func() { - _ = g.Wait() // error is checked later - close(results) - }() - - // Collect results - for ri := range results { - items = append(items, ri) - } - return items, nil -} - -// RestoreRecycleItem restores the specified item -func (tb *DecomposedfsTrashbin) RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error { - _, span := tracer.Start(ctx, "RestoreRecycleItem") - defer span.End() - if ref == nil { - return errtypes.BadRequest("missing reference, needs a space id") - } - - var targetNode *node.Node - if restoreRef != nil { - tn, err := tb.fs.lu.NodeFromResource(ctx, restoreRef) - if err != nil { - return err - } - - targetNode = tn - } - - rn, parent, restoreFunc, err := tb.fs.tp.RestoreRecycleItemFunc(ctx, ref.ResourceId.SpaceId, key, relativePath, targetNode) - if err != nil { - return err - } - - // check permissions of deleted node - rp, err := tb.fs.p.AssembleTrashPermissions(ctx, rn) - switch { - case err != nil: - return err - case !rp.RestoreRecycleItem: - if rp.Stat { - return errtypes.PermissionDenied(key) - } - return errtypes.NotFound(key) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, rn.SpaceOwnerOrManager(ctx)) - - // check we can write to the parent of the restore reference - pp, err := tb.fs.p.AssemblePermissions(ctx, parent) - switch { - case err != nil: - return err - case !pp.InitiateFileUpload: - // share receiver cannot restore to a shared resource to which she does not have write permissions. - if rp.Stat { - return errtypes.PermissionDenied(key) - } - return errtypes.NotFound(key) - } - - // Run the restore func - return restoreFunc() -} - -// PurgeRecycleItem purges the specified item, all its children and all their revisions -func (tb *DecomposedfsTrashbin) PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error { - _, span := tracer.Start(ctx, "PurgeRecycleItem") - defer span.End() - if ref == nil { - return errtypes.BadRequest("missing reference, needs a space id") - } - - rn, purgeFunc, err := tb.fs.tp.PurgeRecycleItemFunc(ctx, ref.ResourceId.OpaqueId, key, relativePath) - if err != nil { - if errors.Is(err, iofs.ErrNotExist) { - return errtypes.NotFound(key) - } - return err - } - - // check permissions of deleted node - rp, err := tb.fs.p.AssembleTrashPermissions(ctx, rn) - switch { - case err != nil: - return err - case !rp.PurgeRecycle: - if rp.Stat { - return errtypes.PermissionDenied(key) - } - return errtypes.NotFound(key) - } - - // Run the purge func - return purgeFunc() -} - -// EmptyRecycle empties the trash -func (tb *DecomposedfsTrashbin) EmptyRecycle(ctx context.Context, ref *provider.Reference) error { - _, span := tracer.Start(ctx, "EmptyRecycle") - defer span.End() - if ref == nil || ref.ResourceId == nil || ref.ResourceId.OpaqueId == "" { - return errtypes.BadRequest("spaceid must be set") - } - - items, err := tb.ListRecycle(ctx, ref, "", "") - if err != nil { - return err - } - - for _, i := range items { - if err := tb.PurgeRecycleItem(ctx, ref, i.Key, ""); err != nil { - return err - } - } - // TODO what permission should we check? we could check the root node of the user? or the owner permissions on his home root node? - // The current impl will wipe your own trash. or when no user provided the trash of 'root' - return os.RemoveAll(tb.getRecycleRoot(ref.ResourceId.SpaceId)) -} - -func (tb *DecomposedfsTrashbin) getRecycleRoot(spaceID string) string { - return filepath.Join(tb.fs.getSpaceRoot(spaceID), "trash") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/revisions.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/revisions.go deleted file mode 100644 index 12425e0f12..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/revisions.go +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "io" - "os" - "path/filepath" - "strings" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -// Revision entries are stored inside the node folder and start with the same uuid as the current version. -// The `.REV.` indicates it is a revision and what follows is a timestamp, so multiple versions -// can be kept in the same location as the current file content. This prevents new fileuploads -// to trigger cross storage moves when revisions accidentally are stored on another partition, -// because the admin mounted a different partition there. -// We can add a background process to move old revisions to a slower storage -// and replace the revision file with a symbolic link in the future, if necessary. - -// ListRevisions lists the revisions of the given resource -func (fs *Decomposedfs) ListRevisions(ctx context.Context, ref *provider.Reference) (revisions []*provider.FileVersion, err error) { - _, span := tracer.Start(ctx, "ListRevisions") - defer span.End() - var n *node.Node - if n, err = fs.lu.NodeFromResource(ctx, ref); err != nil { - return - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, err - case !rp.ListFileVersions: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return nil, errtypes.PermissionDenied(f) - } - return nil, errtypes.NotFound(f) - } - - revisions = []*provider.FileVersion{} - np := n.InternalPath() - if items, err := filepath.Glob(np + node.RevisionIDDelimiter + "*"); err == nil { - for i := range items { - if fs.lu.MetadataBackend().IsMetaFile(items[i]) || strings.HasSuffix(items[i], ".mlock") { - continue - } - - if fi, err := os.Stat(items[i]); err == nil { - parts := strings.SplitN(fi.Name(), node.RevisionIDDelimiter, 2) - if len(parts) != 2 { - appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("invalid revision name, skipping") - continue - } - mtime := fi.ModTime() - rev := &provider.FileVersion{ - Key: n.ID + node.RevisionIDDelimiter + parts[1], - Mtime: uint64(mtime.Unix()), - } - _, blobSize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, items[i], nil) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("name", fi.Name()).Msg("error reading blobsize xattr, using 0") - } - rev.Size = uint64(blobSize) - etag, err := node.CalculateEtag(n.ID, mtime) - if err != nil { - return nil, errors.Wrapf(err, "error calculating etag") - } - rev.Etag = etag - revisions = append(revisions, rev) - } - } - } - // maybe we need to sort the list by key - /* - sort.Slice(revisions, func(i, j int) bool { - return revisions[i].Key > revisions[j].Key - }) - */ - - return -} - -// DownloadRevision returns a reader for the specified revision -// FIXME the CS3 api should explicitly allow initiating revision and trash download, a related issue is https://github.com/cs3org/reva/issues/1813 -func (fs *Decomposedfs) DownloadRevision(ctx context.Context, ref *provider.Reference, revisionKey string, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { - _, span := tracer.Start(ctx, "DownloadRevision") - defer span.End() - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") - return nil, nil, errtypes.NotFound(revisionKey) - } - log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") - - spaceID := ref.ResourceId.SpaceId - // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) - if err != nil { - return nil, nil, err - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return nil, nil, err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, nil, err - case !rp.ListFileVersions || !rp.InitiateFileDownload: // TODO add explicit permission in the CS3 api? - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return nil, nil, errtypes.PermissionDenied(f) - } - return nil, nil, errtypes.NotFound(f) - } - - contentPath := fs.lu.InternalPath(spaceID, revisionKey) - - blobid, blobsize, err := fs.lu.ReadBlobIDAndSizeAttr(ctx, contentPath, nil) - if err != nil { - return nil, nil, errors.Wrapf(err, "Decomposedfs: could not read blob id and size for revision '%s' of node '%s'", kp[1], n.ID) - } - - revisionNode := node.Node{SpaceID: spaceID, BlobID: blobid, Blobsize: blobsize} // blobsize is needed for the s3ng blobstore - - ri, err := n.AsResourceInfo(ctx, rp, nil, []string{"size", "mimetype", "etag"}, true) - if err != nil { - return nil, nil, err - } - - // update resource info with revision data - mtime, err := time.Parse(time.RFC3339Nano, kp[1]) - if err != nil { - return nil, nil, errors.Wrapf(err, "Decomposedfs: could not parse mtime for revision '%s' of node '%s'", kp[1], n.ID) - } - ri.Size = uint64(blobsize) - ri.Mtime = utils.TimeToTS(mtime) - ri.Etag, err = node.CalculateEtag(n.ID, mtime) - if err != nil { - return nil, nil, errors.Wrapf(err, "error calculating etag for revision '%s' of node '%s'", kp[1], n.ID) - } - - var reader io.ReadCloser - if openReaderFunc(ri) { - reader, err = fs.tp.ReadBlob(&revisionNode) - if err != nil { - return nil, nil, errors.Wrapf(err, "Decomposedfs: could not download blob of revision '%s' for node '%s'", n.ID, revisionKey) - } - } - return ri, reader, nil -} - -// RestoreRevision restores the specified revision of the resource -func (fs *Decomposedfs) RestoreRevision(ctx context.Context, ref *provider.Reference, revisionKey string) (returnErr error) { - _, span := tracer.Start(ctx, "RestoreRevision") - defer span.End() - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") - return errtypes.NotFound(revisionKey) - } - - spaceID := ref.ResourceId.SpaceId - // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) - if err != nil { - return err - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return err - } - - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return err - case !rp.RestoreFileVersion: - f, _ := storagespace.FormatReference(ref) - if rp.Stat { - return errtypes.PermissionDenied(f) - } - return errtypes.NotFound(f) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - // check lock - if err := n.CheckLock(ctx); err != nil { - return err - } - - // check processing - if n.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - - // write lock node before copying metadata - f, err := lockedfile.OpenFile(fs.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return err - } - defer func() { - _ = f.Close() - _ = os.Remove(fs.lu.MetadataBackend().LockfilePath(n.InternalPath())) - }() - - // move current version to new revision - nodePath := fs.lu.InternalPath(spaceID, kp[0]) - mtime, err := n.GetMTime(ctx) - if err != nil { - log.Error().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("cannot read mtime") - return err - } - - // revisions are stored alongside the actual file, so a rename can be efficient and does not cross storage / partition boundaries - newRevisionPath := fs.lu.InternalPath(spaceID, kp[0]+node.RevisionIDDelimiter+mtime.UTC().Format(time.RFC3339Nano)) - - // touch new revision - if _, err := os.Create(newRevisionPath); err != nil { - return err - } - defer func() { - if returnErr != nil { - if err := os.Remove(newRevisionPath); err != nil { - log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") - } - if err := fs.lu.MetadataBackend().Purge(ctx, newRevisionPath); err != nil { - log.Error().Err(err).Str("revision", filepath.Base(newRevisionPath)).Msg("could not clean up revision node") - } - } - }() - - // copy blob metadata from node to new revision node - err = fs.lu.CopyMetadataWithSourceLock(ctx, nodePath, newRevisionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || // for checksums - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr // FIXME somewhere I mix up the revision time and the mtime, causing the restore to overwrite the other existing revisien - }, f, true) - if err != nil { - return errtypes.InternalError("failed to copy blob xattrs to version node: " + err.Error()) - } - - // remember mtime from node as new revision mtime - if err = os.Chtimes(newRevisionPath, mtime, mtime); err != nil { - return errtypes.InternalError("failed to change mtime of version node") - } - - // update blob id in node - - // copy blob metadata from restored revision to node - restoredRevisionPath := fs.lu.InternalPath(spaceID, revisionKey) - err = fs.lu.CopyMetadata(ctx, restoredRevisionPath, nodePath, func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr - }, false) - if err != nil { - return errtypes.InternalError("failed to copy blob xattrs to old revision to node: " + err.Error()) - } - // always set the node mtime to the current time - err = fs.lu.MetadataBackend().SetMultiple(ctx, nodePath, - map[string][]byte{ - prefixes.MTimeAttr: []byte(time.Now().UTC().Format(time.RFC3339Nano)), - }, - false) - if err != nil { - return errtypes.InternalError("failed to set mtime attribute on node: " + err.Error()) - } - - revisionSize, err := fs.lu.MetadataBackend().GetInt64(ctx, restoredRevisionPath, prefixes.BlobsizeAttr) - if err != nil { - return errtypes.InternalError("failed to read blob size xattr from old revision") - } - - // drop old revision - if err := os.Remove(restoredRevisionPath); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision, continuing") - } - if err := os.Remove(fs.lu.MetadataBackend().MetadataPath(restoredRevisionPath)); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision metadata, continuing") - } - if err := os.Remove(fs.lu.MetadataBackend().LockfilePath(restoredRevisionPath)); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not delete old revision metadata lockfile, continuing") - } - if err := fs.lu.MetadataBackend().Purge(ctx, restoredRevisionPath); err != nil { - log.Warn().Err(err).Interface("ref", ref).Str("originalnode", kp[0]).Str("revisionKey", revisionKey).Msg("could not purge old revision from cache, continuing") - } - - // revision 5, current 10 (restore a smaller blob) -> 5-10 = -5 - // revision 10, current 5 (restore a bigger blob) -> 10-5 = +5 - sizeDiff := revisionSize - n.Blobsize - - return fs.tp.Propagate(ctx, n, sizeDiff) -} - -// DeleteRevision deletes the specified revision of the resource -func (fs *Decomposedfs) DeleteRevision(ctx context.Context, ref *provider.Reference, revisionKey string) error { - _, span := tracer.Start(ctx, "DeleteRevision") - defer span.End() - n, err := fs.getRevisionNode(ctx, ref, revisionKey, func(rp *provider.ResourcePermissions) bool { - return rp.RestoreFileVersion - }) - if err != nil { - return err - } - - return fs.deleteRevisionFile(ctx, n, revisionKey) -} - -// deleteRevisionFile deletes the revision node identified by revisionKey -// (nodeID + RevisionIDDelimiter + timestamp) together with its metadata -// sidecars and its own blob. It is a no-op if the revision does not exist. -func (fs *Decomposedfs) deleteRevisionFile(ctx context.Context, n *node.Node, revisionKey string) error { - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - return errtypes.NotFound(revisionKey) - } - - blobID, err := n.DeleteRevision(ctx, kp[1]) - if err != nil { - return err - } - if blobID == "" { - // no blob to delete (0-byte file or current-revision revert) - return nil - } - - return fs.tp.DeleteBlob(&node.Node{SpaceID: n.SpaceID, BlobID: blobID}) -} - -func (fs *Decomposedfs) getRevisionNode(ctx context.Context, ref *provider.Reference, revisionKey string, hasPermission func(*provider.ResourcePermissions) bool) (*node.Node, error) { - _, span := tracer.Start(ctx, "getRevisionNode") - defer span.End() - log := appctx.GetLogger(ctx) - - // verify revision key format - kp := strings.SplitN(revisionKey, node.RevisionIDDelimiter, 2) - if len(kp) != 2 { - log.Error().Str("revisionKey", revisionKey).Msg("malformed revisionKey") - return nil, errtypes.NotFound(revisionKey) - } - log.Debug().Str("revisionKey", revisionKey).Msg("DownloadRevision") - - spaceID := ref.ResourceId.SpaceId - // check if the node is available and has not been deleted - n, err := node.ReadNode(ctx, fs.lu, spaceID, kp[0], false, nil, false) - if err != nil { - return nil, err - } - if !n.Exists { - err = errtypes.NotFound(filepath.Join(n.ParentID, n.Name)) - return nil, err - } - - p, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, err - case !hasPermission(p): - return nil, errtypes.PermissionDenied(filepath.Join(n.ParentID, n.Name)) - } - - // Set space owner in context - storagespace.ContextSendSpaceOwnerID(ctx, n.SpaceOwnerOrManager(ctx)) - - return n, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go deleted file mode 100644 index 14ec16a32f..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex/spaceidindex.go +++ /dev/null @@ -1,159 +0,0 @@ -package spaceidindex - -import ( - "io" - "os" - "path/filepath" - "time" - - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/shamaton/msgpack/v2" -) - -// Index holds space id indexes -type Index struct { - root string - name string - cache mtimesyncedcache.Cache[string, map[string]string] -} - -type readWriteCloseSeekTruncater interface { - io.ReadWriteCloser - io.Seeker - Truncate(int64) error -} - -// New returns a new index instance -func New(root, name string) *Index { - return &Index{ - root: root, - name: name, - } -} - -// Init initializes the index and makes sure it can be used -func (i *Index) Init() error { - // Make sure to work on an existing tree - return os.MkdirAll(filepath.Join(i.root, i.name), 0700) -} - -// Load returns the content of an index -func (i *Index) Load(index string) (map[string]string, error) { - indexPath := filepath.Join(i.root, i.name, index+".mpk") - fi, err := os.Stat(indexPath) - if err != nil { - return nil, err - } - return i.readSpaceIndex(indexPath, i.name+":"+index, fi.ModTime()) -} - -// Add adds an entry to an index -// Consider calling AddAll() when trying to add multiple entries as every Add call has to lock the index -func (i *Index) Add(index, key string, value string) error { - return i.updateIndex(index, map[string]string{key: value}, []string{}) -} - -// AddAll adds multiple entries to the index -func (i *Index) AddAll(index string, m map[string]string) error { - return i.updateIndex(index, m, []string{}) -} - -// Remove removes an entry from the index -func (i *Index) Remove(index, key string) error { - return i.updateIndex(index, map[string]string{}, []string{key}) -} - -func (i *Index) updateIndex(index string, addLinks map[string]string, removeLinks []string) error { - indexPath := filepath.Join(i.root, i.name, index+".mpk") - - var err error - // acquire writelock - var f readWriteCloseSeekTruncater - f, err = lockedfile.OpenFile(indexPath, os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return errors.Wrap(err, "unable to lock index to write") - } - defer func() { - rerr := f.Close() - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - // Read current state - msgBytes, err := io.ReadAll(f) - if err != nil { - return err - } - links := map[string]string{} - if len(msgBytes) > 0 { - err = msgpack.Unmarshal(msgBytes, &links) - if err != nil { - return err - } - } - - // set new metadata - for key, val := range addLinks { - links[key] = val - } - for _, key := range removeLinks { - delete(links, key) - } - // Truncate file - _, err = f.Seek(0, io.SeekStart) - if err != nil { - return err - } - err = f.Truncate(0) - if err != nil { - return err - } - - // Write new metadata to file - d, err := msgpack.Marshal(links) - if err != nil { - return errors.Wrap(err, "unable to marshal index") - } - _, err = f.Write(d) - if err != nil { - return errors.Wrap(err, "unable to write index") - } - return nil -} - -func (i *Index) readSpaceIndex(indexPath, cacheKey string, mtime time.Time) (map[string]string, error) { - return i.cache.LoadOrStore(cacheKey, mtime, func() (map[string]string, error) { - // Acquire a read log on the index file - f, err := lockedfile.Open(indexPath) - if err != nil { - return nil, errors.Wrap(err, "unable to lock index to read") - } - defer func() { - rerr := f.Close() - - // if err is non nil we do not overwrite that - if err == nil { - err = rerr - } - }() - - // Read current state - msgBytes, err := io.ReadAll(f) - if err != nil { - return nil, errors.Wrap(err, "unable to read index") - } - links := map[string]string{} - if len(msgBytes) > 0 { - err = msgpack.Unmarshal(msgBytes, &links) - if err != nil { - return nil, errors.Wrap(err, "unable to parse index") - } - } - return links, nil - }) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaces.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaces.go deleted file mode 100644 index ad40ab37af..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaces.go +++ /dev/null @@ -1,1141 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "encoding/json" - "fmt" - "math" - "os" - "path/filepath" - "strconv" - "strings" - "sync/atomic" - "time" - - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/opencloud-eu/reva/v2/internal/grpc/services/storageprovider" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ocsconv "github.com/opencloud-eu/reva/v2/pkg/conversions" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/status" - "github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool" - sdk "github.com/opencloud-eu/reva/v2/pkg/sdk/common" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/templates" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" - "github.com/shamaton/msgpack/v2" - "golang.org/x/sync/errgroup" -) - -const ( - _spaceTypePersonal = "personal" - _spaceTypeProject = "project" - spaceTypeShare = "share" - spaceTypeAny = "*" - spaceIDAny = "*" - - quotaUnrestricted = 0 -) - -// CreateStorageSpace creates a storage space -func (fs *Decomposedfs) CreateStorageSpace(ctx context.Context, req *provider.CreateStorageSpaceRequest) (*provider.CreateStorageSpaceResponse, error) { - ctx = storageprovider.WithSpaceType(ctx, "") - u := ctxpkg.ContextMustGetUser(ctx) - - // "everything is a resource" this is the unique ID for the Space resource. - spaceID, err := fs.lu.GenerateSpaceID(req.Type, req.GetOwner()) - if err != nil { - return nil, err - } - if reqSpaceID := utils.ReadPlainFromOpaque(req.Opaque, "spaceid"); reqSpaceID != "" { - spaceID = reqSpaceID - } - - // Check if space already exists - rootPath := "" - switch req.Type { - case _spaceTypePersonal: - if fs.o.PersonalSpacePathTemplate != "" { - rootPath = filepath.Join(fs.o.Root, templates.WithUser(u, fs.o.PersonalSpacePathTemplate)) - } - default: - if fs.o.GeneralSpacePathTemplate != "" { - rootPath = filepath.Join(fs.o.Root, templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.GeneralSpacePathTemplate)) - } - } - if rootPath != "" { - if _, err := os.Stat(rootPath); err == nil { - return nil, errtypes.AlreadyExists("decomposedfs: spaces: space already exists") - } - } - - description := utils.ReadPlainFromOpaque(req.Opaque, "description") - alias := utils.ReadPlainFromOpaque(req.Opaque, "spaceAlias") - if alias == "" { - alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.GeneralSpaceAliasTemplate) - } - if req.Type == _spaceTypePersonal { - alias = templates.WithSpacePropertiesAndUser(u, req.Type, req.Name, spaceID, fs.o.PersonalSpaceAliasTemplate) - } - - root, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, true, nil, false) // will fall into `Exists` case below - switch { - case err != nil: - return nil, err - case !fs.p.CreateSpace(ctx, spaceID): - return nil, errtypes.PermissionDenied(spaceID) - case root.Exists: - return nil, errtypes.AlreadyExists("decomposedfs: spaces: space already exists") - } - - // create a directory node - root.SetType(provider.ResourceType_RESOURCE_TYPE_CONTAINER) - if rootPath == "" { - rootPath = root.InternalPath() - } - - // set 755 permissions for the base dir - if err := os.MkdirAll(filepath.Dir(rootPath), 0755); err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Decomposedfs: error creating spaces base dir %s", filepath.Dir(rootPath))) - } - - // 770 permissions for the space - if err := os.MkdirAll(rootPath, 0770); err != nil { - return nil, errors.Wrap(err, fmt.Sprintf("Decomposedfs: error creating space %s", rootPath)) - } - - // Store id in cache - if c, ok := fs.lu.(node.IDCacher); ok { - if err := c.CacheID(ctx, spaceID, spaceID, rootPath); err != nil { - return nil, err - } - } - - if req.GetOwner() != nil && req.GetOwner().GetId() != nil { - root.SetOwner(req.GetOwner().GetId()) - } else { - root.SetOwner(&userv1beta1.UserId{OpaqueId: spaceID, Type: userv1beta1.UserType_USER_TYPE_SPACE_OWNER}) - } - - metadata := node.Attributes{} - metadata.SetString(prefixes.IDAttr, spaceID) - metadata.SetString(prefixes.SpaceIDAttr, spaceID) - metadata.SetString(prefixes.OwnerIDAttr, root.Owner().GetOpaqueId()) - metadata.SetString(prefixes.OwnerIDPAttr, root.Owner().GetIdp()) - metadata.SetString(prefixes.OwnerTypeAttr, utils.UserTypeToString(root.Owner().GetType())) - - // always mark the space root node as the end of propagation - metadata.SetString(prefixes.PropagationAttr, "1") - metadata.SetString(prefixes.NameAttr, req.Name) - metadata.SetString(prefixes.SpaceNameAttr, req.Name) - - // This space is empty so set initial treesize to 0 - metadata.SetUInt64(prefixes.TreesizeAttr, 0) - - if req.Type != "" { - metadata.SetString(prefixes.SpaceTypeAttr, req.Type) - } - - if q := req.GetQuota(); q != nil { - // set default space quota - if fs.o.MaxQuota != quotaUnrestricted && q.GetQuotaMaxBytes() > fs.o.MaxQuota { - return nil, errtypes.BadRequest("decompsedFS: requested quota is higher than allowed") - } - metadata.SetInt64(prefixes.QuotaAttr, int64(q.QuotaMaxBytes)) - } else if fs.o.MaxQuota != quotaUnrestricted { - // If no quota was requested but a max quota was set then the the storage space has a quota - // of max quota. - metadata.SetInt64(prefixes.QuotaAttr, int64(fs.o.MaxQuota)) - } - - if description != "" { - metadata.SetString(prefixes.SpaceDescriptionAttr, description) - } - - if alias != "" { - metadata.SetString(prefixes.SpaceAliasAttr, alias) - } - - if err := root.SetXattrsWithContext(ctx, metadata, true); err != nil { - return nil, err - } - - // Write index - err = fs.updateIndexes(ctx, &provider.Grantee{ - Type: provider.GranteeType_GRANTEE_TYPE_USER, - Id: &provider.Grantee_UserId{UserId: req.GetOwner().GetId()}, - }, req.Type, root.ID, root.ID) - if err != nil { - return nil, err - } - - ctx = storageprovider.WithSpaceType(ctx, req.Type) - - if req.Type != _spaceTypePersonal { - if err := fs.AddGrant(ctx, &provider.Reference{ - ResourceId: &provider.ResourceId{ - SpaceId: spaceID, - OpaqueId: spaceID, - }, - }, &provider.Grant{ - Grantee: &provider.Grantee{ - Type: provider.GranteeType_GRANTEE_TYPE_USER, - Id: &provider.Grantee_UserId{ - UserId: u.Id, - }, - }, - Permissions: ocsconv.NewManagerRole().CS3ResourcePermissions(), - }); err != nil { - return nil, err - } - } - - space, err := fs.StorageSpaceFromNode(ctx, root, true) - if err != nil { - return nil, err - } - - resp := &provider.CreateStorageSpaceResponse{ - Status: &v1beta11.Status{ - Code: v1beta11.Code_CODE_OK, - }, - StorageSpace: space, - } - return resp, nil -} - -// ListStorageSpaces returns a list of StorageSpaces. -// The list can be filtered by space type or space id. -// Spaces are persisted with symlinks in /spaces// pointing to ../../nodes/, the root node of the space -// The spaceid is a concatenation of storageid + "!" + nodeid -func (fs *Decomposedfs) ListStorageSpaces(ctx context.Context, filter []*provider.ListStorageSpacesRequest_Filter, unrestricted bool) ([]*provider.StorageSpace, error) { - // TODO check filters - - // TODO when a space symlink is broken delete the space for cleanup - // read permissions are deduced from the node? - - // TODO for absolute references this actually requires us to move all user homes into a subfolder of /nodes/root, - // e.g. /nodes/root/ otherwise storage space names might collide even though they are of different types - // /nodes/root/personal/foo and /nodes/root/shares/foo might be two very different spaces, a /nodes/root/foo is not expressive enough - // we would not need /nodes/root if access always happened via spaceid+relative path - - var ( - spaceID = spaceIDAny - nodeID = spaceIDAny - requestedUserID *userv1beta1.UserId - ) - - spaceTypes := map[string]struct{}{} - - for i := range filter { - switch filter[i].Type { - case provider.ListStorageSpacesRequest_Filter_TYPE_SPACE_TYPE: - switch filter[i].GetSpaceType() { - case "+mountpoint": - // TODO include mount poits - case "+grant": - // TODO include grants - default: - spaceTypes[filter[i].GetSpaceType()] = struct{}{} - } - case provider.ListStorageSpacesRequest_Filter_TYPE_ID: - _, spaceID, nodeID, _ = storagespace.SplitID(filter[i].GetId().OpaqueId) - if strings.Contains(nodeID, "/") { - return []*provider.StorageSpace{}, nil - } - case provider.ListStorageSpacesRequest_Filter_TYPE_USER: - // TODO: refactor this to GetUserId() in cs3 - requestedUserID = filter[i].GetUser() - case provider.ListStorageSpacesRequest_Filter_TYPE_OWNER: - // TODO: improve further by not evaluating shares - requestedUserID = filter[i].GetOwner() - } - } - if len(spaceTypes) == 0 { - spaceTypes[spaceTypeAny] = struct{}{} - } - - authenticatedUserID := ctxpkg.ContextMustGetUser(ctx).GetId().GetOpaqueId() - - if !fs.p.ListSpacesOfUser(ctx, requestedUserID) { - return nil, errtypes.PermissionDenied(fmt.Sprintf("user %s is not allowed to list spaces of other users", authenticatedUserID)) - } - - checkNodePermissions := fs.MustCheckNodePermissions(ctx, unrestricted) - - spaces := []*provider.StorageSpace{} - // build the glob path, eg. - // /path/to/root/spaces/{spaceType}/{spaceId} - // /path/to/root/spaces/personal/nodeid - // /path/to/root/spaces/shared/nodeid - - if spaceID != spaceIDAny && nodeID != spaceIDAny { - // try directly reading the node - n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, true, nil, false) // permission to read disabled space is checked later - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not read node") - return nil, err - } - if !n.Exists { - // return empty list - return spaces, nil - } - space, err := fs.StorageSpaceFromNode(ctx, n, checkNodePermissions) - if err != nil { - return nil, err - } - // filter space types - _, ok1 := spaceTypes[spaceTypeAny] - _, ok2 := spaceTypes[space.SpaceType] - if ok1 || ok2 { - spaces = append(spaces, space) - } - // TODO: filter user id - return spaces, nil - } - - matches := map[string]string{} - var allMatches map[string]string - var err error - - if requestedUserID != nil { - allMatches, err = fs.userSpaceIndex.Load(requestedUserID.GetOpaqueId()) - // do not return an error if the user has no spaces - if err != nil && !os.IsNotExist(err) { - return nil, errors.Wrap(err, "error reading user index") - } - - if nodeID == spaceIDAny { - for spaceID, nodeID := range allMatches { - matches[spaceID] = nodeID - } - } else { - matches[allMatches[nodeID]] = allMatches[nodeID] - } - - // get Groups for userid - user := ctxpkg.ContextMustGetUser(ctx) - // TODO the user from context may not have groups populated - if !utils.UserIDEqual(user.GetId(), requestedUserID) { - user, err = fs.UserIDToUserAndGroups(ctx, requestedUserID) - if err != nil { - return nil, err // TODO log and continue? - } - } - - for _, group := range user.Groups { - allMatches, err = fs.groupSpaceIndex.Load(group) - if err != nil { - if os.IsNotExist(err) { - continue // no spaces for this group - } - return nil, errors.Wrap(err, "error reading group index") - } - - if nodeID == spaceIDAny { - for spaceID, nodeID := range allMatches { - matches[spaceID] = nodeID - } - } else { - matches[allMatches[nodeID]] = allMatches[nodeID] - } - } - - } - - if requestedUserID == nil { - if _, ok := spaceTypes[spaceTypeAny]; ok { - // TODO do not hardcode dirs - spaceTypes = map[string]struct{}{ - "personal": {}, - "project": {}, - "share": {}, - } - } - - for spaceType := range spaceTypes { - allMatches, err = fs.spaceTypeIndex.Load(spaceType) - if err != nil { - if os.IsNotExist(err) { - continue // no spaces for this space type - } - return nil, errors.Wrap(err, "error reading type index") - } - - if nodeID == spaceIDAny { - for spaceID, nodeID := range allMatches { - matches[spaceID] = nodeID - } - } else { - matches[allMatches[nodeID]] = allMatches[nodeID] - } - } - } - - // FIXME if the space does not exist try a node as the space root. - - // But then the whole /spaces/{spaceType}/{spaceid} becomes obsolete - // we can alway just look up by nodeid - // -> no. The /spaces folder is used for efficient lookup by type, otherwise we would have - // to iterate over all nodes and read the type from extended attributes - // -> but for lookup by id we can use the node directly. - // But what about sharding nodes by space? - // an efficient lookup would be possible if we received a spaceid&opaqueid in the request - // the personal spaces must also use the nodeid and not the name - numShares := atomic.Int64{} - errg, ctx := errgroup.WithContext(ctx) - work := make(chan []string, len(matches)) - results := make(chan *provider.StorageSpace, len(matches)) - - // Distribute work - errg.Go(func() error { - defer close(work) - for spaceID, nodeID := range matches { - select { - case work <- []string{spaceID, nodeID}: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - - // Spawn workers that'll concurrently work the queue - numWorkers := 20 - if len(matches) < numWorkers { - numWorkers = len(matches) - } - for i := 0; i < numWorkers; i++ { - errg.Go(func() error { - for match := range work { - spaceID, nodeID, err := fs.tp.ResolveSpaceIDIndexEntry(match[0], match[1]) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("resolve space id index entry, skipping") - continue - } - - n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, true, nil, true) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not read node, skipping") - continue - } - - if !n.Exists { - continue - } - - space, err := fs.StorageSpaceFromNode(ctx, n, checkNodePermissions) - if err != nil { - switch err.(type) { - case errtypes.IsPermissionDenied: - // ok - case errtypes.NotFound: - // ok - default: - appctx.GetLogger(ctx).Error().Err(err).Str("id", nodeID).Msg("could not convert to storage space") - } - continue - } - - // FIXME type share evolved to grant on the edge branch ... make it configurable if the driver should support them or not for now ... ignore type share - if space.SpaceType == spaceTypeShare { - numShares.Add(1) - // do not list shares as spaces for the owner - continue - } - - // TODO apply more filters - _, ok1 := spaceTypes[spaceTypeAny] - _, ok2 := spaceTypes[space.SpaceType] - if ok1 || ok2 { - select { - case results <- space: - case <-ctx.Done(): - return ctx.Err() - } - } - } - return nil - }) - } - - // Wait for things to settle down, then close results chan - go func() { - _ = errg.Wait() // error is checked later - close(results) - }() - - for r := range results { - r.HasTrashedItems = false // FIXME: implement me - spaces = append(spaces, r) - } - - // if there are no matches (or they happened to be spaces for the owner) and the node is a child return a space - if int64(len(matches)) <= numShares.Load() && nodeID != spaceID { - // try node id - n, err := node.ReadNode(ctx, fs.lu, spaceID, nodeID, true, nil, false) // permission to read disabled space is checked in storageSpaceFromNode - if err != nil { - return nil, err - } - if n.Exists { - space, err := fs.StorageSpaceFromNode(ctx, n, checkNodePermissions) - if err != nil { - return nil, err - } - spaces = append(spaces, space) - } - } - - return spaces, nil -} - -// UserIDToUserAndGroups converts a user ID to a user with groups -func (fs *Decomposedfs) UserIDToUserAndGroups(ctx context.Context, userid *userv1beta1.UserId) (*userv1beta1.User, error) { - user, err := fs.UserCache.Get(userid.GetOpaqueId()) - if err == nil { - return user.(*userv1beta1.User), nil - } - - gwConn, err := pool.GetGatewayServiceClient(fs.o.GatewayAddr) - if err != nil { - return nil, err - } - getUserResponse, err := gwConn.GetUser(ctx, &userv1beta1.GetUserRequest{ - UserId: userid, - SkipFetchingUserGroups: false, - }) - if err != nil { - return nil, err - } - if getUserResponse.Status.Code != v1beta11.Code_CODE_OK { - return nil, status.NewErrorFromCode(getUserResponse.Status.Code, "gateway") - } - _ = fs.UserCache.Set(userid.GetOpaqueId(), getUserResponse.GetUser()) - return getUserResponse.GetUser(), nil -} - -// MustCheckNodePermissions checks if permission checks are needed to be performed when user requests spaces -func (fs *Decomposedfs) MustCheckNodePermissions(ctx context.Context, unrestricted bool) bool { - // canListAllSpaces indicates if the user has the permission from the global user role - canListAllSpaces := fs.p.ListAllSpaces(ctx) - // unrestricted is the param which indicates if the user wants to list all spaces or only the spaces he is part of - // if a user lists all spaces unrestricted and doesn't have the permissions from the role, we need to check - // the nodePermissions and this will return a spaces list where the user has access to - // we can only skip the NodePermissions check if both values are true - if canListAllSpaces && unrestricted { - return false - } - return true -} - -// UpdateStorageSpace updates a storage space -func (fs *Decomposedfs) UpdateStorageSpace(ctx context.Context, req *provider.UpdateStorageSpaceRequest) (*provider.UpdateStorageSpaceResponse, error) { - var restore bool - if req.Opaque != nil { - _, restore = req.Opaque.Map["restore"] - } - - space := req.StorageSpace - _, spaceID, _, _ := storagespace.SplitID(space.Id.OpaqueId) - - metadata := make(node.Attributes, 5) - if space.Name != "" { - metadata.SetString(prefixes.NameAttr, space.Name) - metadata.SetString(prefixes.SpaceNameAttr, space.Name) - } - - if space.Quota != nil { - if fs.o.MaxQuota != quotaUnrestricted && fs.o.MaxQuota < space.Quota.QuotaMaxBytes { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_INVALID_ARGUMENT, Message: "decompsedFS: requested quota is higher than allowed"}, - }, nil - } else if fs.o.MaxQuota != quotaUnrestricted && space.Quota.QuotaMaxBytes == quotaUnrestricted { - // If the caller wants to unrestrict the space we give it the maximum allowed quota. - space.Quota.QuotaMaxBytes = fs.o.MaxQuota - } - metadata.SetInt64(prefixes.QuotaAttr, int64(space.Quota.QuotaMaxBytes)) - } - - // TODO also return values which are not in the request - if space.Opaque != nil { - if description, ok := space.Opaque.Map["description"]; ok { - metadata[prefixes.SpaceDescriptionAttr] = description.Value - } - if alias := utils.ReadPlainFromOpaque(space.Opaque, "spaceAlias"); alias != "" { - metadata.SetString(prefixes.SpaceAliasAttr, alias) - } - if image := utils.ReadPlainFromOpaque(space.Opaque, "image"); image != "" { - imageID, err := storagespace.ParseID(image) - if err != nil { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND, Message: "decomposedFS: space image resource not found"}, - }, nil - } - metadata.SetString(prefixes.SpaceImageAttr, imageID.OpaqueId) - } - if readme := utils.ReadPlainFromOpaque(space.Opaque, "readme"); readme != "" { - readmeID, err := storagespace.ParseID(readme) - if err != nil { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND, Message: "decomposedFS: space readme resource not found"}, - }, nil - } - metadata.SetString(prefixes.SpaceReadmeAttr, readmeID.OpaqueId) - } - } - - // check which permissions are needed - spaceNode, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, true, nil, false) - if err != nil { - return nil, err - } - - if !spaceNode.Exists { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND}, - }, nil - } - - sp, err := fs.p.AssemblePermissions(ctx, spaceNode) - if err != nil { - return &provider.UpdateStorageSpaceResponse{ - Status: status.NewStatusFromErrType(ctx, "assembling permissions failed", err), - }, nil - - } - - if !restore && len(metadata) == 0 && !permissions.IsViewer(sp) { - // you may land here when making an update request without changes - // check if user has access to the drive before continuing - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND}, - }, nil - } - - if !permissions.IsManager(sp) { - // We are not a space manager. We need to check for additional permissions. - k := []string{prefixes.NameAttr, prefixes.SpaceDescriptionAttr} - if !permissions.IsEditor(sp) { - k = append(k, prefixes.SpaceReadmeAttr, prefixes.SpaceAliasAttr, prefixes.SpaceImageAttr) - } - - if mapHasKey(metadata, k...) && !fs.p.ManageSpaceProperties(ctx, spaceID) { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_PERMISSION_DENIED}, - }, nil - } - - if restore && !fs.p.SpaceAbility(ctx, spaceID) { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_NOT_FOUND}, - }, nil - } - } - - if mapHasKey(metadata, prefixes.QuotaAttr) { - typ, err := spaceNode.SpaceRoot.Xattr(ctx, prefixes.SpaceTypeAttr) - if err != nil { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{ - Code: v1beta11.Code_CODE_INTERNAL, - Message: "space has no type", - }, - }, nil - } - - if !fs.p.SetSpaceQuota(ctx, spaceID, string(typ)) { - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_PERMISSION_DENIED}, - }, nil - } - } - metadata[prefixes.TreeMTimeAttr] = []byte(time.Now().UTC().Format(time.RFC3339Nano)) - - err = spaceNode.SetXattrsWithContext(ctx, metadata, true) - if err != nil { - return nil, err - } - - if restore { - if err := spaceNode.SetDTime(ctx, nil); err != nil { - return nil, err - } - } - - // send back the updated data from the storage - updatedSpace, err := fs.StorageSpaceFromNode(ctx, spaceNode, false) - if err != nil { - return nil, err - } - - return &provider.UpdateStorageSpaceResponse{ - Status: &v1beta11.Status{Code: v1beta11.Code_CODE_OK}, - StorageSpace: updatedSpace, - }, nil -} - -// DeleteStorageSpace deletes a storage space -func (fs *Decomposedfs) DeleteStorageSpace(ctx context.Context, req *provider.DeleteStorageSpaceRequest) error { - opaque := req.Opaque - var purge bool - if opaque != nil { - _, purge = opaque.Map["purge"] - } - - _, spaceID, _, err := storagespace.SplitID(req.Id.GetOpaqueId()) - if err != nil { - return err - } - - n, err := node.ReadNode(ctx, fs.lu, spaceID, spaceID, true, nil, false) // permission to read disabled space is checked later - if err != nil { - return err - } - - st, err := n.SpaceRoot.XattrString(ctx, prefixes.SpaceTypeAttr) - if err != nil { - return errtypes.InternalError(fmt.Sprintf("space %s does not have a spacetype, possible corrupt decompsedfs", n.ID)) - } - - if err := canDeleteSpace(ctx, spaceID, st, purge, n, fs.p); err != nil { - return err - } - if purge { - if !n.IsDisabled(ctx) { - return errtypes.NewErrtypeFromStatus(status.NewInvalid(ctx, "can't purge enabled space")) - } - - // TODO invalidate ALL indexes in msgpack, not only by type - spaceType, err := n.XattrString(ctx, prefixes.SpaceTypeAttr) - if err != nil { - return err - } - if err := fs.spaceTypeIndex.Remove(spaceType, spaceID); err != nil { - return err - } - - // invalidate cache - if err := fs.lu.MetadataBackend().Purge(ctx, n.InternalPath()); err != nil { - return err - } - - root := fs.getSpaceRoot(spaceID) - - // walkfn will delete the blob if the node has one - walkfn := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if filepath.Ext(path) != ".mpk" { - return nil - } - - b, err := os.ReadFile(path) - if err != nil { - return err - } - - m := map[string][]byte{} - if err := msgpack.Unmarshal(b, &m); err != nil { - return err - } - - bid := m["user.ocis.blobid"] - if string(bid) == "" { - return nil - } - - if err := fs.tp.DeleteBlob(&node.Node{ - BlobID: string(bid), - SpaceID: spaceID, - }); err != nil { - return err - } - - // remove .mpk file so subsequent attempts will not try to delete the blob again - return os.Remove(path) - } - - // This is deletes all blobs of the space - // NOTE: This isn't needed when no s3 is used, but we can't differentiate that here... - if err := filepath.Walk(root, walkfn); err != nil { - return err - } - - // remove space metadata - if err := os.RemoveAll(root); err != nil { - return err - } - - // try removing the space root node - // Note that this will fail when there are other spaceids starting with the same two digits. - _ = os.Remove(filepath.Dir(root)) - - return nil - } - - // mark as disabled by writing a dtime attribute - dtime := time.Now() - return n.SetDTime(ctx, &dtime) -} - -// the value of `target` depends on the implementation: -// - for ocis/s3ng it is the relative link to the space root -// - for the posixfs it is the node id -func (fs *Decomposedfs) updateIndexes(ctx context.Context, grantee *provider.Grantee, spaceType, spaceID, nodeID string) error { - target := fs.tp.BuildSpaceIDIndexEntry(spaceID, nodeID) - err := fs.linkStorageSpaceType(ctx, spaceType, spaceID, target) - if err != nil { - return err - } - if isShareGrant(ctx) { - // FIXME we should count the references for the by-type index currently removing the second share from the same - // space cannot determine if the by-type should be deletet, which is why we never delete them ... - return nil - } - - // create space grant index - switch grantee.Type { - case provider.GranteeType_GRANTEE_TYPE_USER: - return fs.linkSpaceByUser(ctx, grantee.GetUserId().GetOpaqueId(), spaceID, target) - case provider.GranteeType_GRANTEE_TYPE_GROUP: - return fs.linkSpaceByGroup(ctx, grantee.GetGroupId().GetOpaqueId(), spaceID, target) - default: - return errtypes.BadRequest("invalid grantee type: " + grantee.GetType().String()) - } -} - -func (fs *Decomposedfs) linkSpaceByUser(ctx context.Context, userID, spaceID, target string) error { - return fs.userSpaceIndex.Add(userID, spaceID, target) -} - -func (fs *Decomposedfs) linkSpaceByGroup(ctx context.Context, groupID, spaceID, target string) error { - return fs.groupSpaceIndex.Add(groupID, spaceID, target) -} - -func (fs *Decomposedfs) linkStorageSpaceType(ctx context.Context, spaceType, spaceID, target string) error { - return fs.spaceTypeIndex.Add(spaceType, spaceID, target) -} - -func (fs *Decomposedfs) StorageSpaceFromNode(ctx context.Context, n *node.Node, checkPermissions bool) (*provider.StorageSpace, error) { - user := ctxpkg.ContextMustGetUser(ctx) - if checkPermissions { - rp, err := fs.p.AssemblePermissions(ctx, n) - switch { - case err != nil: - return nil, err - case !rp.Stat: - return nil, errtypes.NotFound(fmt.Sprintf("space %s not found", n.ID)) - } - - if n.SpaceRoot.IsDisabled(ctx) { - rp, err := fs.p.AssemblePermissions(ctx, n) - if err != nil || !permissions.IsManager(rp) { - return nil, errtypes.PermissionDenied(fmt.Sprintf("user %s is not allowed to list deleted spaces %s", user.Username, n.ID)) - } - } - } - - sublog := appctx.GetLogger(ctx).With().Str("spaceid", n.SpaceID).Logger() - - var err error - // TODO apply more filters - var sname string - if sname, err = n.SpaceRoot.XattrString(ctx, prefixes.SpaceNameAttr); err != nil { - // FIXME: Is that a severe problem? - sublog.Debug().Err(err).Msg("space does not have a name attribute") - } - - /* - if err := n.FindStorageSpaceRoot(); err != nil { - return nil, err - } - */ - - // read the grants from the current node, not the root - grants, err := n.ListGrants(ctx) - if err != nil { - return nil, err - } - - grantMap := make(map[string]*provider.ResourcePermissions, len(grants)) - grantExpiration := make(map[string]*types.Timestamp) - groupMap := make(map[string]struct{}) - for _, g := range grants { - var id string - switch g.Grantee.Type { - case provider.GranteeType_GRANTEE_TYPE_GROUP: - id = g.Grantee.GetGroupId().OpaqueId - groupMap[id] = struct{}{} - case provider.GranteeType_GRANTEE_TYPE_USER: - id = g.Grantee.GetUserId().OpaqueId - default: - continue - } - - if g.Expiration != nil { - // We are doing this check here because we want to remove expired grants "on access". - // This way we don't have to have a cron job checking the grants in regular intervals. - // The tradeof obviously is that this code is here. - if isGrantExpired(g) { - if err := n.DeleteGrant(ctx, g, true); err != nil { - sublog.Error().Err(err).Str("grantee", id). - Msg("failed to delete expired space grant") - } - if n.IsSpaceRoot(ctx) { - // invalidate space grant - switch g.Grantee.Type { - case provider.GranteeType_GRANTEE_TYPE_USER: - // remove from user index - if err := fs.userSpaceIndex.Remove(g.Grantee.GetUserId().GetOpaqueId(), n.SpaceID); err != nil { - sublog.Error().Err(err).Str("grantee", id). - Msg("failed to delete expired user space index") - } - case provider.GranteeType_GRANTEE_TYPE_GROUP: - // remove from group index - if err := fs.groupSpaceIndex.Remove(g.Grantee.GetGroupId().GetOpaqueId(), n.SpaceID); err != nil { - sublog.Error().Err(err).Str("grantee", id). - Msg("failed to delete expired group space index") - } - } - } - - continue - } - grantExpiration[id] = g.Expiration - } - grantMap[id] = g.Permissions - } - - grantMapJSON, err := json.Marshal(grantMap) - if err != nil { - return nil, err - } - - grantExpirationMapJSON, err := json.Marshal(grantExpiration) - if err != nil { - return nil, err - } - - groupMapJSON, err := json.Marshal(groupMap) - if err != nil { - return nil, err - } - - ssID, err := storagespace.FormatReference( - &provider.Reference{ - ResourceId: &provider.ResourceId{ - SpaceId: n.SpaceRoot.SpaceID, - OpaqueId: n.SpaceRoot.ID}, - }, - ) - if err != nil { - return nil, err - } - space := &provider.StorageSpace{ - Opaque: &types.Opaque{ - Map: map[string]*types.OpaqueEntry{ - "grants": { - Decoder: "json", - Value: grantMapJSON, - }, - "grants_expirations": { - Decoder: "json", - Value: grantExpirationMapJSON, - }, - "groups": { - Decoder: "json", - Value: groupMapJSON, - }, - }, - }, - Id: &provider.StorageSpaceId{OpaqueId: ssID}, - Root: &provider.ResourceId{ - SpaceId: n.SpaceRoot.SpaceID, - OpaqueId: n.SpaceRoot.ID, - }, - Name: sname, - // SpaceType is read from xattr below - // Mtime is set either as node.tmtime or as fi.mtime below - } - - space.SpaceType, err = n.SpaceRoot.XattrString(ctx, prefixes.SpaceTypeAttr) - if err != nil { - appctx.GetLogger(ctx).Debug().Err(err).Msg("space does not have a type attribute") - } - - if n.SpaceRoot.IsDisabled(ctx) { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "trashed", "trashed") - } - - if n.Owner() != nil && n.Owner().OpaqueId != "" { - space.Owner = &userv1beta1.User{ // FIXME only return a UserID, not a full blown user object - Id: n.Owner(), - } - } - - // we set the space mtime to the root item mtime - // override the stat mtime with a tmtime if it is present - var tmtime time.Time - if tmt, err := n.GetTMTime(ctx); err == nil { - tmtime = tmt - un := tmt.UnixNano() - space.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - } else if fi, err := os.Stat(n.InternalPath()); err == nil { - // fall back to stat mtime - tmtime = fi.ModTime() - un := fi.ModTime().UnixNano() - space.Mtime = &types.Timestamp{ - Seconds: uint64(un / 1000000000), - Nanos: uint32(un % 1000000000), - } - } - - etag, err := node.CalculateEtag(n.ID, tmtime) - if err != nil { - return nil, err - } - space.Opaque.Map["etag"] = &types.OpaqueEntry{ - Decoder: "plain", - Value: []byte(etag), - } - - spaceAttributes, err := n.SpaceRoot.Xattrs(ctx) - if err != nil { - return nil, err - } - - // if quota is set try parsing it as int64, otherwise don't bother - if q, err := spaceAttributes.Int64(prefixes.QuotaAttr); err == nil && q >= 0 { - // make sure we have a proper signed int - // we use the same magic numbers to indicate: - // -1 = uncalculated - // -2 = unknown - // -3 = unlimited - space.Quota = &provider.Quota{ - QuotaMaxBytes: uint64(q), - QuotaMaxFiles: math.MaxUint64, // TODO MaxUInt64? = unlimited? why even max files? 0 = unlimited? - } - - } - if si := spaceAttributes.String(prefixes.SpaceImageAttr); si != "" { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "image", storagespace.FormatResourceID( - &provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: si}, - )) - } - if sd := spaceAttributes.String(prefixes.SpaceDescriptionAttr); sd != "" { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "description", sd) - } - if sr := spaceAttributes.String(prefixes.SpaceReadmeAttr); sr != "" { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "readme", storagespace.FormatResourceID( - &provider.ResourceId{StorageId: space.Root.StorageId, SpaceId: space.Root.SpaceId, OpaqueId: sr}, - )) - } - if sa := spaceAttributes.String(prefixes.SpaceAliasAttr); sa != "" { - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "spaceAlias", sa) - } - - // add rootinfo - ps, _ := n.SpaceRoot.PermissionSet(ctx) - space.RootInfo, _ = n.SpaceRoot.AsResourceInfo(ctx, ps, []string{"quota"}, nil, false) - - // we cannot put free, used and remaining into the quota, as quota, when set would always imply a quota limit - // for now we use opaque properties with a 'quota.' prefix - quotaStr := node.QuotaUnknown - if quotaInOpaque := sdk.DecodeOpaqueMap(space.RootInfo.Opaque)["quota"]; quotaInOpaque != "" { - quotaStr = quotaInOpaque - } - - total, used, remaining, err := fs.calculateTotalUsedRemaining(quotaStr, space.GetRootInfo().GetSize()) - if err != nil { - return nil, err - } - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "quota.total", strconv.FormatUint(total, 10)) - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "quota.used", strconv.FormatUint(used, 10)) - space.Opaque = utils.AppendPlainToOpaque(space.Opaque, "quota.remaining", strconv.FormatUint(remaining, 10)) - - return space, nil -} - -func mapHasKey(checkMap map[string][]byte, keys ...string) bool { - for _, key := range keys { - if _, hasKey := checkMap[key]; hasKey { - return true - } - } - return false -} - -func isGrantExpired(g *provider.Grant) bool { - if g.Expiration == nil { - return false - } - return time.Now().After(time.Unix(int64(g.Expiration.Seconds), int64(g.Expiration.Nanos))) -} - -func (fs *Decomposedfs) getSpaceRoot(spaceID string) string { - return filepath.Join(fs.o.Root, "spaces", lookup.Pathify(spaceID, 1, 2)) -} - -// Space deletion can be tricky as there are lots of different cases: -// - spaces of type personal can only be disabled and deleted by users with the "delete-all-home-spaces" permission -// - a user with the "delete-all-spaces" permission may delete but not enable/disable any project space -// - a user with the "Drive.ReadWriteEnabled" permission may enable/disable but not delete any project space -// - a project space can always be enabled/disabled/deleted by its manager (i.e. users have the "remove" grant) -func canDeleteSpace(ctx context.Context, spaceID string, typ string, purge bool, n *node.Node, p permissions.Permissions) error { - // delete-all-home spaces allows to disable and delete a personal space - if typ == "personal" { - if p.DeleteAllHomeSpaces(ctx) { - return nil - } - return errtypes.PermissionDenied("user is not allowed to delete a personal space") - } - - // space managers are allowed to disable and delete their project spaces - if rp, err := p.AssemblePermissions(ctx, n); err == nil && permissions.IsManager(rp) { - return nil - } - - // delete-all-spaces permissions allows to delete (purge, NOT disable) project spaces - if purge && p.DeleteAllSpaces(ctx) { - return nil - } - - // Drive.ReadWriteEnabled allows to disable a space - if !purge && p.SpaceAbility(ctx, spaceID) { - return nil - } - - return errtypes.PermissionDenied(fmt.Sprintf("user is not allowed to delete space %s", n.ID)) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/timemanager/timemanager.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/timemanager/timemanager.go deleted file mode 100644 index b478499cce..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/timemanager/timemanager.go +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2018-2024 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. -package timemanager - -import ( - "context" - "os" - "time" - - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" -) - -// Manager is responsible for managing time-related attributes of nodes in a decomposed file system. -type Manager struct { -} - -// OverrideMtime overrides the modification time (mtime) attribute of a node with the given time. -func (m *Manager) OverrideMtime(ctx context.Context, _ *node.Node, attrs *node.Attributes, mtime time.Time) error { - attrs.SetString(prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) - return nil -} - -// MTime retrieves the modification time (mtime) attribute of a node. -// If the attribute is not set, it falls back to the file's last modification time. -func (dtm *Manager) MTime(ctx context.Context, n *node.Node) (time.Time, error) { - b, err := n.XattrString(ctx, prefixes.MTimeAttr) - if err != nil { - fi, err := os.Lstat(n.InternalPath()) - if err != nil { - return time.Time{}, err - } - return fi.ModTime(), nil - } - return time.Parse(time.RFC3339Nano, b) -} - -// SetMTime sets the modification time (mtime) attribute of a node to the given time. -// If the time is nil, the attribute is removed. -func (dtm *Manager) SetMTime(ctx context.Context, n *node.Node, mtime *time.Time) error { - if mtime == nil { - return n.RemoveXattr(ctx, prefixes.MTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.MTimeAttr, mtime.UTC().Format(time.RFC3339Nano)) -} - -// TMTime retrieves the tree modification time (tmtime) attribute of a node. -// If the attribute is not set, it falls back to the node's modification time (mtime). -func (dtm *Manager) TMTime(ctx context.Context, n *node.Node) (time.Time, error) { - b, err := n.XattrString(ctx, prefixes.TreeMTimeAttr) - if err == nil { - return time.Parse(time.RFC3339Nano, b) - } - - // no tmtime, use mtime - return dtm.MTime(ctx, n) -} - -// SetTMTime sets the tree modification time (tmtime) attribute of a node to the given time. -// If the time is nil, the attribute is removed. -func (dtm *Manager) SetTMTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { - if tmtime == nil { - return n.RemoveXattr(ctx, prefixes.TreeMTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.TreeMTimeAttr, tmtime.UTC().Format(time.RFC3339Nano)) -} - -// CTime retrieves the creation time (ctime) attribute of a node. -// Since decomposedfs does not differentiate between ctime and mtime, it falls back to the node's modification time (mtime). -func (dtm *Manager) CTime(ctx context.Context, n *node.Node) (time.Time, error) { - // decomposedfs does not differentiate between ctime and mtime - return dtm.MTime(ctx, n) -} - -// SetCTime sets the creation time (ctime) attribute of a node to the given time. -// Since decomposedfs does not differentiate between ctime and mtime, it sets the modification time (mtime) instead. -func (dtm *Manager) SetCTime(ctx context.Context, n *node.Node, mtime *time.Time) error { - // decomposedfs does not differentiate between ctime and mtime - return dtm.SetMTime(ctx, n, mtime) -} - -// TCTime retrieves the tree creation time (tctime) attribute of a node. -// Since decomposedfs does not differentiate between ctime and mtime, it falls back to the tree modification time (tmtime). -func (dtm *Manager) TCTime(ctx context.Context, n *node.Node) (time.Time, error) { - // decomposedfs does not differentiate between ctime and mtime - return dtm.TMTime(ctx, n) -} - -// SetTCTime sets the tree creation time (tctime) attribute of a node to the given time. -// Since decomposedfs does not differentiate between ctime and mtime, it sets the tree modification time (tmtime) instead. -func (dtm *Manager) SetTCTime(ctx context.Context, n *node.Node, tmtime *time.Time) error { - // decomposedfs does not differentiate between ctime and mtime - return dtm.SetTMTime(ctx, n, tmtime) -} - -// DTime retrieves the deletion time (dtime) attribute of a node. -func (dtm *Manager) DTime(ctx context.Context, n *node.Node) (tmTime time.Time, err error) { - b, err := n.XattrString(ctx, prefixes.DTimeAttr) - if err != nil { - return time.Time{}, err - } - return time.Parse(time.RFC3339Nano, b) -} - -// SetDTime sets the deletion time (dtime) attribute of a node to the given time. -// If the time is nil, the attribute is removed. -func (dtm *Manager) SetDTime(ctx context.Context, n *node.Node, t *time.Time) (err error) { - if t == nil { - return n.RemoveXattr(ctx, prefixes.DTimeAttr, true) - } - return n.SetXattrString(ctx, prefixes.DTimeAttr, t.UTC().Format(time.RFC3339Nano)) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin/trashbin.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin/trashbin.go deleted file mode 100644 index 3f17f0b6b2..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin/trashbin.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2018-2024 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package trashbin - -import ( - "context" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/storage" -) - -type Trashbin interface { - Setup(storage.FS) error - - ListRecycle(ctx context.Context, ref *provider.Reference, key, relativePath string) ([]*provider.RecycleItem, error) - RestoreRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string, restoreRef *provider.Reference) error - PurgeRecycleItem(ctx context.Context, ref *provider.Reference, key, relativePath string) error - EmptyRecycle(ctx context.Context, ref *provider.Reference) error -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/async.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/async.go deleted file mode 100644 index 2e0920c1c6..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/async.go +++ /dev/null @@ -1,439 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package propagator - -import ( - "context" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/google/renameio/v2" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/rs/zerolog" - "github.com/shamaton/msgpack/v2" -) - -var _propagationGracePeriod = 3 * time.Minute - -// AsyncPropagator implements asynchronous treetime & treesize propagation -type AsyncPropagator struct { - treeSizeAccounting bool - treeTimeAccounting bool - propagationDelay time.Duration - lookup node.PathLookup - log *zerolog.Logger -} - -// Change represents a change to the tree -type Change struct { - SyncTime time.Time - SizeDiff int64 -} - -// NewAsyncPropagator returns a new AsyncPropagator instance -func NewAsyncPropagator(treeSizeAccounting, treeTimeAccounting bool, o options.AsyncPropagatorOptions, lookup node.PathLookup, log *zerolog.Logger) AsyncPropagator { - p := AsyncPropagator{ - treeSizeAccounting: treeSizeAccounting, - treeTimeAccounting: treeTimeAccounting, - propagationDelay: o.PropagationDelay, - lookup: lookup, - log: log, - } - - log.Info().Msg("async propagator starting up...") - - // spawn a goroutine that watches for stale .processing dirs and fixes them - go func() { - if !p.treeTimeAccounting && !p.treeSizeAccounting { - // no propagation enabled - log.Debug().Msg("propagation disabled or nothing to propagate") - return - } - - changesDirPath := filepath.Join(p.lookup.InternalRoot(), "changes") - doSleep := false // switch to not sleep on the first iteration - for { - if doSleep { - time.Sleep(5 * time.Minute) - } - doSleep = true - log.Debug().Msg("scanning for stale .processing dirs") - - entries, err := filepath.Glob(changesDirPath + "/**/*") - if err != nil { - log.Error().Err(err).Msg("failed to list changes") - continue - } - - for _, e := range entries { - changesDirPath := e - entry, err := os.Stat(changesDirPath) - if err != nil { - continue - } - // recover all dirs that seem to have been stuck - if !entry.IsDir() || time.Now().Before(entry.ModTime().Add(_propagationGracePeriod)) { - continue - } - - go func() { - if !strings.HasSuffix(changesDirPath, ".processing") { - // first rename the existing node dir - err = os.Rename(changesDirPath, changesDirPath+".processing") - if err != nil { - return - } - changesDirPath += ".processing" - } - - log.Debug().Str("dir", changesDirPath).Msg("propagating stale .processing dir") - parts := strings.SplitN(entry.Name(), ":", 2) - if len(parts) != 2 { - log.Error().Str("file", entry.Name()).Msg("encountered invalid .processing dir") - return - } - - now := time.Now() - _ = os.Chtimes(changesDirPath, now, now) - p.propagate(context.Background(), parts[0], strings.TrimSuffix(parts[1], ".processing"), true, *log) - }() - } - } - }() - - return p -} - -// Propagate triggers a propagation -func (p AsyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) error { - ctx, span := tracer.Start(ctx, "Propagate") - defer span.End() - log := appctx.GetLogger(ctx).With(). - Str("method", "async.Propagate"). - Str("spaceid", n.SpaceID). - Str("nodeid", n.ID). - Str("parentid", n.ParentID). - Int64("sizeDiff", sizeDiff). - Logger() - - if !p.treeTimeAccounting && (!p.treeSizeAccounting || sizeDiff == 0) { - // no propagation enabled - log.Debug().Msg("propagation disabled or nothing to propagate") - return nil - } - - // add a change to the parent node - c := Change{ - // use a sync time and don't rely on the mtime of the current node, as the stat might not change when a rename happened too quickly - SyncTime: time.Now().UTC(), - SizeDiff: sizeDiff, - } - go p.queuePropagation(ctx, n.SpaceID, n.ParentID, c, log) - - return nil -} - -func (p AsyncPropagator) queuePropagation(ctx context.Context, spaceID, nodeID string, change Change, log zerolog.Logger) { - // add a change to the parent node - changePath := p.changesPath(spaceID, nodeID, uuid.New().String()+".mpk") - - data, err := msgpack.Marshal(change) - if err != nil { - log.Error().Err(err).Msg("failed to marshal Change") - return - } - - _, subspan := tracer.Start(ctx, "write changes file") - ready := false - triggerPropagation := false - _ = os.MkdirAll(filepath.Dir(filepath.Dir(changePath)), 0700) - err = os.Mkdir(filepath.Dir(changePath), 0700) - triggerPropagation = err == nil || os.IsExist(err) // only the first goroutine, which succeeds to create the directory, is supposed to actually trigger the propagation - for retries := 0; retries <= 500; retries++ { - err := renameio.WriteFile(changePath, data, 0644) - if err == nil { - ready = true - break - } - log.Debug().Err(err).Msg("failed to write Change to disk (retrying)") - err = os.Mkdir(filepath.Dir(changePath), 0700) - triggerPropagation = err == nil || os.IsExist(err) // only the first goroutine, which succeeds to create the directory, is supposed to actually trigger the propagation - } - - if !ready { - log.Error().Err(err).Msg("failed to write Change to disk") - return - } - subspan.End() - - if !triggerPropagation { - return - } - - _, subspan = tracer.Start(ctx, "delay propagation") - time.Sleep(p.propagationDelay) // wait a moment before propagating - subspan.End() - - log.Debug().Msg("propagating") - // add a change to the parent node - changeDirPath := p.changesPath(spaceID, nodeID, "") - - // first rename the existing node dir - err = os.Rename(changeDirPath, changeDirPath+".processing") - if err != nil { - // This can fail in 2 ways - // 1. source does not exist anymore as it has already been propagated by another goroutine - // -> ignore, as the change is already being processed - // 2. target already exists because a previous propagation is still running - // -> ignore, the previous propagation will pick the new changes up - return - } - p.propagate(ctx, spaceID, nodeID, false, log) -} - -func (p AsyncPropagator) propagate(ctx context.Context, spaceID, nodeID string, recalculateTreeSize bool, log zerolog.Logger) { - changeDirPath := p.changesPath(spaceID, nodeID, "") - processingPath := changeDirPath + ".processing" - - cleanup := func() { - err := os.RemoveAll(processingPath) - if err != nil { - log.Error().Err(err).Msg("Could not remove .processing dir") - } - } - - _, subspan := tracer.Start(ctx, "list changes files") - d, err := os.Open(processingPath) - if err != nil { - log.Error().Err(err).Msg("Could not open change .processing dir") - cleanup() - return - } - defer d.Close() - names, err := d.Readdirnames(0) - if err != nil { - log.Error().Err(err).Msg("Could not read dirnames") - cleanup() - return - } - subspan.End() - - _, subspan = tracer.Start(ctx, "read changes files") - pc := Change{} - for _, name := range names { - if !strings.HasSuffix(name, ".mpk") { - continue - } - - b, err := os.ReadFile(filepath.Join(processingPath, name)) - if err != nil { - log.Error().Err(err).Msg("Could not read change") - cleanup() - return - } - c := Change{} - err = msgpack.Unmarshal(b, &c) - if err != nil { - log.Error().Err(err).Msg("Could not unmarshal change") - cleanup() - return - } - if c.SyncTime.After(pc.SyncTime) { - pc.SyncTime = c.SyncTime - } - pc.SizeDiff += c.SizeDiff - } - subspan.End() - - // TODO do we need to write an aggregated parentchange file? - - attrs := node.Attributes{} - - var f *lockedfile.File - // lock parent before reading treesize or tree time - nodePath := filepath.Join(p.lookup.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "nodes", lookup.Pathify(nodeID, 4, 2)) - - _, subspan = tracer.Start(ctx, "lockedfile.OpenFile") - lockFilepath := p.lookup.MetadataBackend().LockfilePath(nodePath) - f, err = lockedfile.OpenFile(lockFilepath, os.O_RDWR|os.O_CREATE, 0600) - subspan.End() - if err != nil { - log.Error().Err(err). - Str("lock filepath", lockFilepath). - Msg("Propagation failed. Could not open metadata for node with lock.") - cleanup() - return - } - // always log error if closing node fails - defer func() { - // ignore already closed error - cerr := f.Close() - if err == nil && cerr != nil && !errors.Is(cerr, os.ErrClosed) { - err = cerr // only overwrite err with en error from close if the former was nil - } - }() - - _, subspan = tracer.Start(ctx, "node.ReadNode") - var n *node.Node - if n, err = node.ReadNode(ctx, p.lookup, spaceID, nodeID, false, nil, false); err != nil { - log.Error().Err(err). - Msg("Propagation failed. Could not read node.") - cleanup() - return - } - subspan.End() - - if !n.Exists { - log.Debug().Str("attr", prefixes.PropagationAttr).Msg("node does not exist anymore, not propagating") - cleanup() - return - } - - if !n.HasPropagation(ctx) { - log.Debug().Str("attr", prefixes.PropagationAttr).Msg("propagation attribute not set or unreadable, not propagating") - cleanup() - return - } - - if p.treeTimeAccounting { - // update the parent tree time if it is older than the nodes mtime - updateSyncTime := false - - var tmTime time.Time - tmTime, err = n.GetTMTime(ctx) - switch { - case err != nil: - // missing attribute, or invalid format, overwrite - log.Debug().Err(err). - Msg("could not read tmtime attribute, overwriting") - updateSyncTime = true - case tmTime.Before(pc.SyncTime): - log.Debug(). - Time("tmtime", tmTime). - Time("stime", pc.SyncTime). - Msg("parent tmtime is older than node mtime, updating") - updateSyncTime = true - default: - log.Debug(). - Time("tmtime", tmTime). - Time("stime", pc.SyncTime). - Dur("delta", pc.SyncTime.Sub(tmTime)). - Msg("node tmtime is younger than stime, not updating") - } - - if updateSyncTime { - // update the tree time of the parent node - attrs.SetString(prefixes.TreeMTimeAttr, pc.SyncTime.UTC().Format(time.RFC3339Nano)) - } - - attrs.SetString(prefixes.TmpEtagAttr, "") - } - - // size accounting - if p.treeSizeAccounting && pc.SizeDiff != 0 { - var newSize uint64 - - // read treesize - treeSize, err := n.GetTreeSize(ctx) - switch { - case recalculateTreeSize || metadata.IsAttrUnset(err): - // fallback to calculating the treesize - log.Warn().Msg("treesize attribute unset, falling back to calculating the treesize") - newSize, err = calculateTreeSize(ctx, p.lookup, n.InternalPath()) - if err != nil { - log.Error().Err(err). - Msg("Error when calculating treesize of node.") // FIXME wat? - cleanup() - return - } - case err != nil: - log.Error().Err(err). - Msg("Failed to propagate treesize change. Error when reading the treesize attribute from node") - cleanup() - return - case pc.SizeDiff > 0: - newSize = treeSize + uint64(pc.SizeDiff) - case uint64(-pc.SizeDiff) > treeSize: - // The sizeDiff is larger than the current treesize. Which would result in - // a negative new treesize. Something must have gone wrong with the accounting. - // Reset the current treesize to 0. - log.Error().Uint64("treeSize", treeSize).Int64("sizeDiff", pc.SizeDiff). - Msg("Error when updating treesize of node. Updated treesize < 0. Resetting to 0") - newSize = 0 - default: - newSize = treeSize - uint64(-pc.SizeDiff) - } - - // update the tree size of the node - attrs.SetString(prefixes.TreesizeAttr, strconv.FormatUint(newSize, 10)) - log.Debug().Uint64("newSize", newSize).Msg("updated treesize of node") - } - - if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil { - log.Error().Err(err).Msg("Failed to update extend attributes of node") - cleanup() - return - } - - // Release node lock early, ignore already closed error - _, subspan = tracer.Start(ctx, "f.Close") - cerr := f.Close() - subspan.End() - if cerr != nil && !errors.Is(cerr, os.ErrClosed) { - log.Error().Err(cerr).Msg("Failed to close node and release lock") - } - - log.Info().Msg("Propagation done. cleaning up") - cleanup() - - if !n.IsSpaceRoot(ctx) { - p.queuePropagation(ctx, n.SpaceID, n.ParentID, pc, log) - } - - // Check for a changes dir that might have been added meanwhile and pick it up - if _, err = os.Open(changeDirPath); err == nil { - log.Info().Msg("Found a new changes dir. starting next propagation") - time.Sleep(p.propagationDelay) // wait a moment before propagating - err = os.Rename(changeDirPath, processingPath) - if err != nil { - // This can fail in 2 ways - // 1. source does not exist anymore as it has already been propagated by another goroutine - // -> ignore, as the change is already being processed - // 2. target already exists because a previous propagation is still running - // -> ignore, the previous propagation will pick the new changes up - return - } - p.propagate(ctx, spaceID, nodeID, false, log) - } -} - -func (p AsyncPropagator) changesPath(spaceID, nodeID, filename string) string { - return filepath.Join(p.lookup.InternalRoot(), "changes", spaceID[0:2], spaceID+":"+nodeID, filename) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/propagator.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/propagator.go deleted file mode 100644 index 18cf32a328..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/propagator.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package propagator - -import ( - "context" - "os" - "path/filepath" - "strconv" - - sprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -var tracer trace.Tracer - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree/propagator") -} - -type Propagator interface { - Propagate(ctx context.Context, node *node.Node, sizediff int64) error -} - -func New(lookup node.PathLookup, o *options.Options, log *zerolog.Logger) Propagator { - switch o.Propagator { - case "async": - return NewAsyncPropagator(o.TreeSizeAccounting, o.TreeTimeAccounting, o.AsyncPropagatorOptions, lookup, log) - default: - return NewSyncPropagator(o.TreeSizeAccounting, o.TreeTimeAccounting, lookup) - } -} - -func calculateTreeSize(ctx context.Context, lookup node.PathLookup, childrenPath string) (uint64, error) { - ctx, span := tracer.Start(ctx, "calculateTreeSize") - defer span.End() - var size uint64 - - f, err := os.Open(childrenPath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childrenPath", childrenPath).Msg("could not open dir") - return 0, err - } - defer f.Close() - - names, err := f.Readdirnames(0) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childrenPath", childrenPath).Msg("could not read dirnames") - return 0, err - } - for i := range names { - cPath := filepath.Join(childrenPath, names[i]) - resolvedPath, err := filepath.EvalSymlinks(cPath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not resolve child entry symlink") - continue // continue after an error - } - - // raw read of the attributes for performance reasons - attribs, err := lookup.MetadataBackend().All(ctx, resolvedPath) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("childpath", cPath).Msg("could not read attributes of child entry") - continue // continue after an error - } - sizeAttr := "" - if string(attribs[prefixes.TypeAttr]) == strconv.FormatUint(uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE), 10) { - sizeAttr = string(attribs[prefixes.BlobsizeAttr]) - } else { - sizeAttr = string(attribs[prefixes.TreesizeAttr]) - } - csize, err := strconv.ParseInt(sizeAttr, 10, 64) - if err != nil { - return 0, errors.Wrapf(err, "invalid blobsize xattr format") - } - size += uint64(csize) - } - return size, err -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/sync.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/sync.go deleted file mode 100644 index 4646c3358c..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator/sync.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package propagator - -import ( - "context" - "errors" - "os" - "strconv" - "time" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/rs/zerolog" -) - -// SyncPropagator implements synchronous treetime & treesize propagation -type SyncPropagator struct { - treeSizeAccounting bool - treeTimeAccounting bool - lookup node.PathLookup -} - -// NewSyncPropagator returns a new AsyncPropagator instance -func NewSyncPropagator(treeSizeAccounting, treeTimeAccounting bool, lookup node.PathLookup) SyncPropagator { - return SyncPropagator{ - treeSizeAccounting: treeSizeAccounting, - treeTimeAccounting: treeTimeAccounting, - lookup: lookup, - } -} - -// Propagate triggers a propagation -func (p SyncPropagator) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) error { - ctx, span := tracer.Start(ctx, "Propagate") - defer span.End() - sublog := appctx.GetLogger(ctx).With(). - Str("method", "sync.Propagate"). - Str("spaceid", n.SpaceID). - Str("nodeid", n.ID). - Int64("sizeDiff", sizeDiff). - Logger() - - if !p.treeTimeAccounting && (!p.treeSizeAccounting || sizeDiff == 0) { - // no propagation enabled - sublog.Debug().Msg("propagation disabled or nothing to propagate") - return nil - } - - // is propagation enabled for the parent node? - root := n.SpaceRoot - - // use a sync time and don't rely on the mtime of the current node, as the stat might not change when a rename happened too quickly - sTime := time.Now().UTC() - - // we loop until we reach the root - var ( - err error - stop bool - ) - - for err == nil && !stop && n.ID != root.ID { - n, stop, err = p.propagateItem(ctx, n, sTime, sizeDiff, sublog) - } - - if err != nil { - sublog.Error().Err(err).Msg("error propagating") - return err - } - return nil -} - -func (p SyncPropagator) propagateItem(ctx context.Context, n *node.Node, sTime time.Time, sizeDiff int64, log zerolog.Logger) (*node.Node, bool, error) { - log.Debug().Msg("propagating") - - attrs := node.Attributes{} - - var f *lockedfile.File - // lock parent before reading treesize or tree time - - _, subspan := tracer.Start(ctx, "lockedfile.OpenFile") - parentFilename := p.lookup.MetadataBackend().LockfilePath(n.ParentPath()) - f, err := lockedfile.OpenFile(parentFilename, os.O_RDWR|os.O_CREATE, 0600) - subspan.End() - if err != nil { - log.Error().Err(err). - Str("parent filename", parentFilename). - Msg("Propagation failed. Could not open metadata for parent with lock.") - return nil, true, err - } - // always log error if closing node fails - defer func() { - // ignore already closed error - cerr := f.Close() - if err == nil && cerr != nil && !errors.Is(cerr, os.ErrClosed) { - err = cerr // only overwrite err with en error from close if the former was nil - } - }() - - if n, err = n.Parent(ctx); err != nil { - log.Error().Err(err). - Msg("Propagation failed. Could not read parent node.") - return n, true, err - } - - if !n.HasPropagation(ctx) { - log.Debug().Str("attr", prefixes.PropagationAttr).Msg("propagation attribute not set or unreadable, not propagating") - // if the attribute is not set treat it as false / none / no propagation - return n, true, nil - } - - log = log.With().Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Logger() - - if p.treeTimeAccounting { - // update the parent tree time if it is older than the nodes mtime - updateSyncTime := false - - var tmTime time.Time - tmTime, err = n.GetTMTime(ctx) - switch { - case err != nil: - // missing attribute, or invalid format, overwrite - log.Debug().Err(err). - Msg("could not read tmtime attribute, overwriting") - updateSyncTime = true - case tmTime.Before(sTime): - log.Debug(). - Time("tmtime", tmTime). - Time("stime", sTime). - Msg("parent tmtime is older than node mtime, updating") - updateSyncTime = true - default: - log.Debug(). - Time("tmtime", tmTime). - Time("stime", sTime). - Dur("delta", sTime.Sub(tmTime)). - Msg("parent tmtime is younger than node mtime, not updating") - } - - if updateSyncTime { - // update the tree time of the parent node - attrs.SetString(prefixes.TreeMTimeAttr, sTime.UTC().Format(time.RFC3339Nano)) - } - - attrs.SetString(prefixes.TmpEtagAttr, "") - } - - // size accounting - if p.treeSizeAccounting && sizeDiff != 0 { - var newSize uint64 - - // read treesize - treeSize, err := n.GetTreeSize(ctx) - switch { - case metadata.IsAttrUnset(err): - // fallback to calculating the treesize - log.Warn().Msg("treesize attribute unset, falling back to calculating the treesize") - newSize, err = calculateTreeSize(ctx, p.lookup, n.InternalPath()) - if err != nil { - return n, true, err - } - case err != nil: - log.Error().Err(err). - Msg("Faild to propagate treesize change. Error when reading the treesize attribute from parent") - return n, true, err - case sizeDiff > 0: - newSize = treeSize + uint64(sizeDiff) - case uint64(-sizeDiff) > treeSize: - // The sizeDiff is larger than the current treesize. Which would result in - // a negative new treesize. Something must have gone wrong with the accounting. - // Reset the current treesize to 0. - log.Error().Uint64("treeSize", treeSize).Int64("sizeDiff", sizeDiff). - Msg("Error when updating treesize of parent node. Updated treesize < 0. Reestting to 0") - newSize = 0 - default: - newSize = treeSize - uint64(-sizeDiff) - } - - // update the tree size of the node - attrs.SetString(prefixes.TreesizeAttr, strconv.FormatUint(newSize, 10)) - log.Debug().Uint64("newSize", newSize).Msg("updated treesize of parent node") - } - - if err = n.SetXattrsWithContext(ctx, attrs, false); err != nil { - log.Error().Err(err).Msg("Failed to update extend attributes of parent node") - return n, true, err - } - - return n, false, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go deleted file mode 100644 index ef2a0ddf5e..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/tree.go +++ /dev/null @@ -1,982 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package tree - -import ( - "bytes" - "context" - "fmt" - "io" - "io/fs" - "os" - "path/filepath" - "regexp" - "strings" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" - "github.com/rs/zerolog" - "go-micro.dev/v4/store" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" -) - -var tracer trace.Tracer - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/tree") -} - -// Blobstore defines an interface for storing blobs in a blobstore -type Blobstore interface { - // TODO(OCISDEV-901): remove Upload once FinishUploadDecomposed is replaced by the upload coordinator (OCISDEV-900). - Upload(node *node.Node, source string) error - UploadFromReader(node *node.Node, r io.Reader, size int64) error - Download(node *node.Node) (io.ReadCloser, error) - Delete(node *node.Node) error -} - -// Tree manages a hierarchical tree -type Tree struct { - lookup node.PathLookup - blobstore Blobstore - propagator propagator.Propagator - - options *options.Options - - idCache store.Store -} - -// PermissionCheckFunc defined a function used to check resource permissions -type PermissionCheckFunc func(rp *provider.ResourcePermissions) bool - -// New returns a new instance of Tree -func New(lu node.PathLookup, bs Blobstore, o *options.Options, cache store.Store, log *zerolog.Logger) *Tree { - return &Tree{ - lookup: lu, - blobstore: bs, - options: o, - idCache: cache, - propagator: propagator.New(lu, o, log), - } -} - -// Setup prepares the tree structure -func (t *Tree) Setup() error { - // create data paths for internal layout - dataPaths := []string{ - filepath.Join(t.options.Root, "spaces"), - // notes contain symlinks from nodes//uploads/ to ../../uploads/ - // better to keep uploads on a fast / volatile storage before a workflow finally moves them to the nodes dir - filepath.Join(t.options.Root, "uploads"), - } - for _, v := range dataPaths { - err := os.MkdirAll(v, 0700) - if err != nil { - return err - } - } - return nil -} - -// GetMD returns the metadata of a node in the tree -func (t *Tree) GetMD(ctx context.Context, n *node.Node) (os.FileInfo, error) { - _, span := tracer.Start(ctx, "GetMD") - defer span.End() - md, err := os.Stat(n.InternalPath()) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, errtypes.NotFound(n.ID) - } - return nil, errors.Wrap(err, "tree: error stating "+n.ID) - } - - return md, nil -} - -// TouchFile creates a new empty file -func (t *Tree) TouchFile(ctx context.Context, n *node.Node, markprocessing bool, mtime string) error { - _, span := tracer.Start(ctx, "TouchFile") - defer span.End() - if n.Exists { - if markprocessing { - return n.SetXattr(ctx, prefixes.StatusPrefix, []byte(node.ProcessingStatus)) - } - - return errtypes.AlreadyExists(n.ID) - } - - if n.ID == "" { - n.ID = uuid.New().String() - } - n.SetType(provider.ResourceType_RESOURCE_TYPE_FILE) - - nodePath := n.InternalPath() - if err := os.MkdirAll(filepath.Dir(nodePath), 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: error creating node") - } - _, err := os.Create(nodePath) - if err != nil { - return errors.Wrap(err, "Decomposedfs: error creating node") - } - - attributes := n.NodeMetadata(ctx) - if markprocessing { - attributes[prefixes.StatusPrefix] = []byte(node.ProcessingStatus) - } - if mtime != "" { - if err := n.SetMtimeString(ctx, mtime); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set mtime") - } - } else { - now := time.Now() - if err := n.SetMtime(ctx, &now); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set mtime") - } - } - err = n.SetXattrsWithContext(ctx, attributes, true) - if err != nil { - return err - } - - // link child name to parent if it is new - childNameLink := filepath.Join(n.ParentPath(), n.Name) - var link string - link, err = os.Readlink(childNameLink) - if err == nil && link != "../"+n.ID { - if err = os.Remove(childNameLink); err != nil { - return errors.Wrap(err, "Decomposedfs: could not remove symlink child entry") - } - } - if errors.Is(err, fs.ErrNotExist) || link != "../"+n.ID { - relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - if err = os.Symlink(relativeNodePath, childNameLink); err != nil { - return errors.Wrap(err, "Decomposedfs: could not symlink child entry") - } - } - - return t.Propagate(ctx, n, 0) -} - -// CreateDir creates a new directory entry in the tree -func (t *Tree) CreateDir(ctx context.Context, n *node.Node) (err error) { - ctx, span := tracer.Start(ctx, "CreateDir") - defer span.End() - if n.Exists { - return errtypes.AlreadyExists(n.ID) // path? - } - - // create a directory node - n.SetType(provider.ResourceType_RESOURCE_TYPE_CONTAINER) - if n.ID == "" { - n.ID = uuid.New().String() - } - - err = t.createDirNode(ctx, n) - if err != nil { - return - } - - // make child appear in listings - relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - ctx, subspan := tracer.Start(ctx, "os.Symlink") - err = os.Symlink(relativeNodePath, filepath.Join(n.ParentPath(), n.Name)) - subspan.End() - if err != nil { - // no better way to check unfortunately - if !strings.Contains(err.Error(), "file exists") { - return - } - - // try to remove the node - ctx, subspan = tracer.Start(ctx, "os.RemoveAll") - e := os.RemoveAll(n.InternalPath()) - subspan.End() - if e != nil { - appctx.GetLogger(ctx).Debug().Err(e).Msg("cannot delete node") - } - return errtypes.AlreadyExists(err.Error()) - } - return t.Propagate(ctx, n, 0) -} - -// Move replaces the target with the source -func (t *Tree) Move(ctx context.Context, oldNode *node.Node, newNode *node.Node) (err error) { - _, span := tracer.Start(ctx, "Move") - defer span.End() - if oldNode.SpaceID != newNode.SpaceID { - // WebDAV RFC https://www.rfc-editor.org/rfc/rfc4918#section-9.9.4 says to use - // > 502 (Bad Gateway) - This may occur when the destination is on another - // > server and the destination server refuses to accept the resource. - // > This could also occur when the destination is on another sub-section - // > of the same server namespace. - // but we only have a not supported error - return errtypes.NotSupported("cannot move across spaces") - } - // if target exists delete it without trashing it - if newNode.Exists { - // TODO make sure all children are deleted - if err := os.RemoveAll(newNode.InternalPath()); err != nil { - return errors.Wrap(err, "Decomposedfs: Move: error deleting target node "+newNode.ID) - } - } - - // remove cache entry in any case to avoid inconsistencies - defer func() { _ = t.idCache.Delete(filepath.Join(oldNode.ParentPath(), oldNode.Name)) }() - - // Always target the old node ID for xattr updates. - // The new node id is empty if the target does not exist - // and we need to overwrite the new one when overwriting an existing path. - // are we just renaming (parent stays the same)? - if oldNode.ParentID == newNode.ParentID { - - // parentPath := t.lookup.InternalPath(oldNode.SpaceID, oldNode.ParentID) - parentPath := oldNode.ParentPath() - - // rename child - err = os.Rename( - filepath.Join(parentPath, oldNode.Name), - filepath.Join(parentPath, newNode.Name), - ) - if err != nil { - return errors.Wrap(err, "Decomposedfs: could not rename child") - } - - // update name attribute - if err := oldNode.SetXattrString(ctx, prefixes.NameAttr, newNode.Name); err != nil { - return errors.Wrap(err, "Decomposedfs: could not set name attribute") - } - - return t.Propagate(ctx, newNode, 0) - } - - // we are moving the node to a new parent, any target has been removed - // bring old node to the new parent - - // rename child - err = os.Rename( - filepath.Join(oldNode.ParentPath(), oldNode.Name), - filepath.Join(newNode.ParentPath(), newNode.Name), - ) - if err != nil { - return errors.Wrap(err, "Decomposedfs: could not move child") - } - - // update target parentid and name - attribs := node.Attributes{} - attribs.SetString(prefixes.ParentidAttr, newNode.ParentID) - attribs.SetString(prefixes.NameAttr, newNode.Name) - if err := oldNode.SetXattrsWithContext(ctx, attribs, true); err != nil { - return errors.Wrap(err, "Decomposedfs: could not update old node attributes") - } - - // the size diff is the current treesize or blobsize of the old/source node - var sizeDiff int64 - if oldNode.IsDir(ctx) { - treeSize, err := oldNode.GetTreeSize(ctx) - if err != nil { - return err - } - sizeDiff = int64(treeSize) - } else { - sizeDiff = oldNode.Blobsize - } - - // TODO inefficient because we might update several nodes twice, only propagate unchanged nodes? - // collect in a list, then only stat each node once - // also do this in a go routine ... webdav should check the etag async - - err = t.Propagate(ctx, oldNode, -sizeDiff) - if err != nil { - return errors.Wrap(err, "Decomposedfs: Move: could not propagate old node") - } - err = t.Propagate(ctx, newNode, sizeDiff) - if err != nil { - return errors.Wrap(err, "Decomposedfs: Move: could not propagate new node") - } - return nil -} - -func readChildNodeFromLink(ctx context.Context, path string) (string, error) { - _, span := tracer.Start(ctx, "readChildNodeFromLink") - defer span.End() - link, err := os.Readlink(path) - if err != nil { - return "", err - } - nodeID := strings.TrimLeft(link, "/.") - nodeID = strings.ReplaceAll(nodeID, "/", "") - return nodeID, nil -} - -// ListFolder lists the content of a folder node -func (t *Tree) ListFolder(ctx context.Context, n *node.Node) ([]*node.Node, error) { - ctx, span := tracer.Start(ctx, "ListFolder") - defer span.End() - dir := n.InternalPath() - - _, subspan := tracer.Start(ctx, "os.Open") - f, err := os.Open(dir) - subspan.End() - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, errtypes.NotFound(dir) - } - return nil, errors.Wrap(err, "tree: error listing "+dir) - } - defer f.Close() - - _, subspan = tracer.Start(ctx, "f.Readdirnames") - names, err := f.Readdirnames(0) - subspan.End() - if err != nil { - return nil, err - } - - numWorkers := t.options.MaxConcurrency - if len(names) < numWorkers { - numWorkers = len(names) - } - work := make(chan string) - results := make(chan *node.Node) - - g, ctx := errgroup.WithContext(ctx) - - // Distribute work - g.Go(func() error { - defer close(work) - for _, name := range names { - select { - case work <- name: - case <-ctx.Done(): - return ctx.Err() - } - } - return nil - }) - - // Spawn workers that'll concurrently work the queue - for i := 0; i < numWorkers; i++ { - g.Go(func() error { - var err error - for name := range work { - path := filepath.Join(dir, name) - nodeID := getNodeIDFromCache(ctx, path, t.idCache) - if nodeID == "" { - nodeID, err = readChildNodeFromLink(ctx, path) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("path", path).Msg("cannot read node from link") - continue - } - err = storeNodeIDInCache(ctx, path, nodeID, t.idCache) - if err != nil { - appctx.GetLogger(ctx).Warn().Err(err).Str("node", nodeID).Msg("cannot store node in the cache") - // failing to write in the cache is fine because we can keep going - } - } - - child, err := node.ReadNode(ctx, t.lookup, n.SpaceID, nodeID, false, n.SpaceRoot, true) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("node", nodeID).Msg("cannot read node") - continue - } - - // prevent listing denied resources - if !child.IsDenied(ctx) { - if child.SpaceRoot == nil { - child.SpaceRoot = n.SpaceRoot - } - select { - case results <- child: - case <-ctx.Done(): - return ctx.Err() - } - } - } - return nil - }) - } - // Wait for things to settle down, then close results chan - go func() { - _ = g.Wait() // error is checked later - close(results) - }() - - retNodes := []*node.Node{} - for n := range results { - retNodes = append(retNodes, n) - } - - if err := g.Wait(); err != nil { - return nil, err - } - - return retNodes, nil -} - -// Delete deletes a node in the tree by moving it to the trash -func (t *Tree) Delete(ctx context.Context, n *node.Node) (err error) { - _, span := tracer.Start(ctx, "Delete") - defer span.End() - path := filepath.Join(n.ParentPath(), n.Name) - // remove entry from cache immediately to avoid inconsistencies - defer func() { _ = t.idCache.Delete(path) }() - - // get the original path - origin, err := t.lookup.Path(ctx, n, node.NoCheck) - if err != nil { - return - } - - // set origin location in metadata - nodePath := n.InternalPath() - if err := n.SetXattrString(ctx, prefixes.TrashOriginAttr, origin); err != nil { - return err - } - - var sizeDiff int64 - if n.IsDir(ctx) { - treesize, err := n.GetTreeSize(ctx) - if err != nil { - return err // TODO calculate treesize if it is not set - } - sizeDiff = -int64(treesize) - } else { - sizeDiff = -n.Blobsize - } - - deletionTime := time.Now().UTC().Format(time.RFC3339Nano) - - // Prepare the trash - trashLink := filepath.Join(t.options.Root, "spaces", lookup.Pathify(n.SpaceRoot.ID, 1, 2), "trash", lookup.Pathify(n.ID, 4, 2)) - if err := os.MkdirAll(filepath.Dir(trashLink), 0700); err != nil { - // Roll back changes - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) - return err - } - - // FIXME can we just move the node into the trash dir? instead of adding another symlink and appending a trash timestamp? - // can we just use the mtime as the trash time? - // TODO store a trashed by userid - - // first make node appear in the space trash - // parent id and name are stored as extended attributes in the node itself - err = os.Symlink("../../../../../nodes/"+lookup.Pathify(n.ID, 4, 2)+node.TrashIDDelimiter+deletionTime, trashLink) - if err != nil { - // Roll back changes - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) - return - } - - // at this point we have a symlink pointing to a non existing destination, which is fine - - // rename the trashed node so it is not picked up when traversing up the tree and matches the symlink - trashPath := nodePath + node.TrashIDDelimiter + deletionTime - err = os.Rename(nodePath, trashPath) - if err != nil { - // To roll back changes - // TODO remove symlink - // Roll back changes - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) - return - } - err = t.lookup.MetadataBackend().Rename(nodePath, trashPath) - if err != nil { - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) - _ = os.Rename(trashPath, nodePath) - return - } - - // Remove lock file if it exists - _ = os.Remove(n.LockFilePath()) - - // finally remove the entry from the parent dir - if err = os.Remove(path); err != nil { - // To roll back changes - // TODO revert the rename - // TODO remove symlink - // Roll back changes - _ = n.RemoveXattr(ctx, prefixes.TrashOriginAttr, true) - return - } - - return t.Propagate(ctx, n, sizeDiff) -} - -// RestoreRecycleItemFunc returns a node and a function to restore it from the trash. -func (t *Tree) RestoreRecycleItemFunc(ctx context.Context, spaceid, key, trashPath string, targetNode *node.Node) (*node.Node, *node.Node, func() error, error) { - _, span := tracer.Start(ctx, "RestoreRecycleItemFunc") - defer span.End() - logger := appctx.GetLogger(ctx) - - recycleNode, trashItem, deletedNodePath, origin, err := t.readRecycleItem(ctx, spaceid, key, trashPath) - if err != nil { - return nil, nil, nil, err - } - - targetRef := &provider.Reference{ - ResourceId: &provider.ResourceId{SpaceId: spaceid, OpaqueId: spaceid}, - Path: utils.MakeRelativePath(origin), - } - - if targetNode == nil { - targetNode, err = t.lookup.NodeFromResource(ctx, targetRef) - if err != nil { - return nil, nil, nil, err - } - } - - if err := targetNode.CheckLock(ctx); err != nil { - return nil, nil, nil, err - } - - parent, err := targetNode.Parent(ctx) - if err != nil { - return nil, nil, nil, err - } - - fn := func() error { - if targetNode.Exists { - return errtypes.AlreadyExists("origin already exists") - } - - // add the entry for the parent dir - err = os.Symlink("../../../../../"+lookup.Pathify(recycleNode.ID, 4, 2), filepath.Join(targetNode.ParentPath(), targetNode.Name)) - if err != nil { - return err - } - - // rename to node only name, so it is picked up by id - nodePath := recycleNode.InternalPath() - - // attempt to rename only if we're not in a subfolder - if deletedNodePath != nodePath { - err = os.Rename(deletedNodePath, nodePath) - if err != nil { - return err - } - err = t.lookup.MetadataBackend().Rename(deletedNodePath, nodePath) - if err != nil { - return err - } - } - - targetNode.Exists = true - - attrs := node.Attributes{} - attrs.SetString(prefixes.NameAttr, targetNode.Name) - // set ParentidAttr to restorePath's node parent id - attrs.SetString(prefixes.ParentidAttr, targetNode.ParentID) - - if err = recycleNode.SetXattrsWithContext(ctx, attrs, true); err != nil { - return errors.Wrap(err, "Decomposedfs: could not update recycle node") - } - - // delete item link in trash - deletePath := trashItem - if trashPath != "" && trashPath != "/" { - resolvedTrashRoot, err := filepath.EvalSymlinks(trashItem) - if err != nil { - return errors.Wrap(err, "Decomposedfs: could not resolve trash root") - } - deletePath = filepath.Join(resolvedTrashRoot, trashPath) - if err = os.Remove(deletePath); err != nil { - logger.Error().Err(err).Str("trashItem", trashItem).Str("deletePath", deletePath).Str("trashPath", trashPath).Msg("error deleting trash item") - } - } else { - if err = utils.RemoveItem(deletePath); err != nil { - logger.Error().Err(err).Str("trashItem", trashItem).Str("deletePath", deletePath).Str("trashPath", trashPath).Msg("error recursively deleting trash item") - } - } - - var sizeDiff int64 - if recycleNode.IsDir(ctx) { - treeSize, err := recycleNode.GetTreeSize(ctx) - if err != nil { - return err - } - sizeDiff = int64(treeSize) - } else { - sizeDiff = recycleNode.Blobsize - } - return t.Propagate(ctx, targetNode, sizeDiff) - } - return recycleNode, parent, fn, nil -} - -// PurgeRecycleItemFunc returns a node and a function to purge it from the trash -func (t *Tree) PurgeRecycleItemFunc(ctx context.Context, spaceid, key string, path string) (*node.Node, func() error, error) { - _, span := tracer.Start(ctx, "PurgeRecycleItemFunc") - defer span.End() - logger := appctx.GetLogger(ctx) - - rn, trashItem, deletedNodePath, _, err := t.readRecycleItem(ctx, spaceid, key, path) - if err != nil { - return nil, nil, err - } - - ts := "" - timeSuffix := strings.SplitN(filepath.Base(deletedNodePath), node.TrashIDDelimiter, 2) - if len(timeSuffix) == 2 { - ts = timeSuffix[1] - } - - fn := func() error { - - if err := t.removeNode(ctx, deletedNodePath, ts, rn); err != nil { - return err - } - - // delete item link in trash - deletePath := trashItem - if path != "" && path != "/" { - resolvedTrashRoot, err := filepath.EvalSymlinks(trashItem) - if err != nil { - return errors.Wrap(err, "Decomposedfs: could not resolve trash root") - } - deletePath = filepath.Join(resolvedTrashRoot, path) - } - if err = utils.RemoveItem(deletePath); err != nil { - logger.Error().Err(err).Str("deletePath", deletePath).Msg("error deleting trash item") - return err - } - - return nil - } - - return rn, fn, nil -} - -// InitNewNode initializes a new node -func (t *Tree) InitNewNode(ctx context.Context, n *node.Node, fsize uint64) (metadata.UnlockFunc, error) { - _, span := tracer.Start(ctx, "InitNewNode") - defer span.End() - // create folder structure (if needed) - - _, subspan := tracer.Start(ctx, "os.MkdirAll") - err := os.MkdirAll(filepath.Dir(n.InternalPath()), 0700) - subspan.End() - if err != nil { - return nil, err - } - - // create and write lock new node metadata - _, subspan = tracer.Start(ctx, "metadata.Lock") - unlock, err := t.lookup.MetadataBackend().Lock(n.InternalPath()) - subspan.End() - if err != nil { - return nil, err - } - - // we also need to touch the actual node file here it stores the mtime of the resource - _, subspan = tracer.Start(ctx, "os.OpenFile") - h, err := os.OpenFile(n.InternalPath(), os.O_CREATE|os.O_EXCL, 0600) - subspan.End() - if err != nil { - return unlock, err - } - h.Close() - - _, subspan = tracer.Start(ctx, "node.CheckQuota") - _, err = node.CheckQuota(ctx, n.SpaceRoot, false, 0, fsize) - subspan.End() - if err != nil { - return unlock, err - } - - // link child name to parent if it is new - childNameLink := filepath.Join(n.ParentPath(), n.Name) - relativeNodePath := filepath.Join("../../../../../", lookup.Pathify(n.ID, 4, 2)) - log := appctx.GetLogger(ctx).With().Str("childNameLink", childNameLink).Str("relativeNodePath", relativeNodePath).Logger() - log.Info().Msg("initNewNode: creating symlink") - - _, subspan = tracer.Start(ctx, "os.Symlink") - err = os.Symlink(relativeNodePath, childNameLink) - subspan.End() - if err != nil { - log.Info().Err(err).Msg("initNewNode: symlink failed") - if errors.Is(err, fs.ErrExist) { - log.Info().Err(err).Msg("initNewNode: symlink already exists") - return unlock, errtypes.AlreadyExists(n.Name) - } - return unlock, errors.Wrap(err, "Decomposedfs: could not symlink child entry") - } - log.Info().Msg("initNewNode: symlink created") - - return unlock, nil -} - -func (t *Tree) removeNode(ctx context.Context, path, timeSuffix string, n *node.Node) error { - logger := appctx.GetLogger(ctx) - - if timeSuffix != "" { - n.ID = n.ID + node.TrashIDDelimiter + timeSuffix - } - - if n.IsDir(ctx) { - item, err := t.ListFolder(ctx, n) - if err != nil { - logger.Error().Err(err).Str("path", path).Msg("error listing folder") - } else { - for _, child := range item { - if err := t.removeNode(ctx, child.InternalPath(), "", child); err != nil { - return err - } - } - } - } - - // delete the actual node - if err := utils.RemoveItem(path); err != nil { - logger.Error().Err(err).Str("path", path).Msg("error purging node") - return err - } - - if err := t.lookup.MetadataBackend().Purge(ctx, path); err != nil { - logger.Error().Err(err).Str("path", t.lookup.MetadataBackend().MetadataPath(path)).Msg("error purging node metadata") - return err - } - - // delete blob from blobstore - if n.BlobID != "" { - if err := t.DeleteBlob(n); err != nil { - logger.Error().Err(err).Str("blobID", n.BlobID).Msg("error purging nodes blob") - return err - } - } - - // delete revisions - revs, err := filepath.Glob(n.InternalPath() + node.RevisionIDDelimiter + "*") - if err != nil { - logger.Error().Err(err).Str("path", n.InternalPath()+node.RevisionIDDelimiter+"*").Msg("glob failed badly") - return err - } - for _, rev := range revs { - if t.lookup.MetadataBackend().IsMetaFile(rev) { - continue - } - - bID, _, err := t.lookup.ReadBlobIDAndSizeAttr(ctx, rev, nil) - if err != nil { - logger.Error().Err(err).Str("revision", rev).Msg("error reading blobid attribute") - return err - } - - if err := utils.RemoveItem(rev); err != nil { - logger.Error().Err(err).Str("revision", rev).Msg("error removing revision node") - return err - } - - if bID != "" { - if err := t.DeleteBlob(&node.Node{SpaceID: n.SpaceID, BlobID: bID}); err != nil { - logger.Error().Err(err).Str("revision", rev).Str("blobID", bID).Msg("error removing revision node blob") - return err - } - } - - } - - return nil -} - -// Propagate propagates changes to the root of the tree -func (t *Tree) Propagate(ctx context.Context, n *node.Node, sizeDiff int64) (err error) { - return t.propagator.Propagate(ctx, n, sizeDiff) -} - -// WriteBlob writes a blob to the blobstore -// TODO(OCISDEV-901): remove once FinishUploadDecomposed is replaced by the upload coordinator (OCISDEV-900). -func (t *Tree) WriteBlob(node *node.Node, source string) error { - return t.blobstore.Upload(node, source) -} - -// WriteBlobFromReader writes a blob to the blobstore from a reader -func (t *Tree) WriteBlobFromReader(node *node.Node, r io.Reader, size int64) error { - return t.blobstore.UploadFromReader(node, r, size) -} - -// ReadBlob reads a blob from the blobstore -func (t *Tree) ReadBlob(node *node.Node) (io.ReadCloser, error) { - if node.BlobID == "" { - // there is no blob yet - we are dealing with a 0 byte file - return io.NopCloser(bytes.NewReader([]byte{})), nil - } - return t.blobstore.Download(node) -} - -// DeleteBlob deletes a blob from the blobstore -func (t *Tree) DeleteBlob(node *node.Node) error { - if node == nil { - return fmt.Errorf("could not delete blob, nil node was given") - } - if node.BlobID == "" { - return fmt.Errorf("could not delete blob, node with empty blob id was given") - } - - return t.blobstore.Delete(node) -} - -// BuildSpaceIDIndexEntry returns the entry for the space id index -func (t *Tree) BuildSpaceIDIndexEntry(spaceID, nodeID string) string { - return "../../../spaces/" + lookup.Pathify(spaceID, 1, 2) + "/nodes/" + lookup.Pathify(spaceID, 4, 2) -} - -// ResolveSpaceIDIndexEntry returns the node id for the space id index entry -func (t *Tree) ResolveSpaceIDIndexEntry(_, entry string) (string, string, error) { - return ReadSpaceAndNodeFromIndexLink(entry) -} - -// ReadSpaceAndNodeFromIndexLink reads a symlink and parses space and node id if the link has the correct format, eg: -// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51 -// ../../spaces/4c/510ada-c86b-4815-8820-42cdf82c3d51/nodes/4c/51/0a/da/-c86b-4815-8820-42cdf82c3d51.T.2022-02-24T12:35:18.196484592Z -func ReadSpaceAndNodeFromIndexLink(link string) (string, string, error) { - // ../../../spaces/sp/ace-id/nodes/sh/or/tn/od/eid - // 0 1 2 3 4 5 6 7 8 9 10 11 - parts := strings.Split(link, string(filepath.Separator)) - if len(parts) != 12 || parts[0] != ".." || parts[1] != ".." || parts[2] != ".." || parts[3] != "spaces" || parts[6] != "nodes" { - return "", "", errtypes.InternalError("malformed link") - } - return strings.Join(parts[4:6], ""), strings.Join(parts[7:12], ""), nil -} - -// TODO check if node exists? -func (t *Tree) createDirNode(ctx context.Context, n *node.Node) (err error) { - ctx, span := tracer.Start(ctx, "createDirNode") - defer span.End() - // create a directory node - nodePath := n.InternalPath() - if err := os.MkdirAll(nodePath, 0700); err != nil { - return errors.Wrap(err, "Decomposedfs: error creating node") - } - - attributes := n.NodeMetadata(ctx) - attributes[prefixes.SpaceIDAttr] = []byte(n.SpaceID) - attributes[prefixes.TreesizeAttr] = []byte("0") // initialize as empty, TODO why bother? if it is not set we could treat it as 0? - if t.options.TreeTimeAccounting || t.options.TreeSizeAccounting { - attributes[prefixes.PropagationAttr] = []byte("1") // mark the node for propagation - } - return n.SetXattrsWithContext(ctx, attributes, true) -} - -var nodeIDRegep = regexp.MustCompile(`.*/nodes/([^.]*).*`) - -// TODO refactor the returned params into Node properties? would make all the path transformations go away... -func (t *Tree) readRecycleItem(ctx context.Context, spaceID, key, path string) (recycleNode *node.Node, trashItem string, deletedNodePath string, origin string, err error) { - _, span := tracer.Start(ctx, "readRecycleItem") - defer span.End() - logger := appctx.GetLogger(ctx) - - if key == "" { - return nil, "", "", "", errtypes.InternalError("key is empty") - } - - backend := t.lookup.MetadataBackend() - var nodeID string - - trashItem = filepath.Join(t.lookup.InternalRoot(), "spaces", lookup.Pathify(spaceID, 1, 2), "trash", lookup.Pathify(key, 4, 2)) - resolvedTrashItem, err := filepath.EvalSymlinks(trashItem) - if err != nil { - return - } - deletedNodePath, err = filepath.EvalSymlinks(filepath.Join(resolvedTrashItem, path)) - if err != nil { - return - } - nodeID = nodeIDRegep.ReplaceAllString(deletedNodePath, "$1") - nodeID = strings.ReplaceAll(nodeID, "/", "") - - recycleNode = node.New(spaceID, nodeID, "", "", 0, "", provider.ResourceType_RESOURCE_TYPE_INVALID, nil, t.lookup) - recycleNode.SpaceRoot, err = node.ReadNode(ctx, t.lookup, spaceID, spaceID, false, nil, false) - if err != nil { - return - } - recycleNode.SetType(t.lookup.TypeFromPath(ctx, deletedNodePath)) - - var attrBytes []byte - if recycleNode.Type(ctx) == provider.ResourceType_RESOURCE_TYPE_FILE { - // lookup blobID in extended attributes - if attrBytes, err = backend.Get(ctx, deletedNodePath, prefixes.BlobIDAttr); err == nil { - recycleNode.BlobID = string(attrBytes) - } else { - return - } - - // lookup blobSize in extended attributes - if recycleNode.Blobsize, err = backend.GetInt64(ctx, deletedNodePath, prefixes.BlobsizeAttr); err != nil { - return - } - } - - // lookup parent id in extended attributes - if attrBytes, err = backend.Get(ctx, deletedNodePath, prefixes.ParentidAttr); err == nil { - recycleNode.ParentID = string(attrBytes) - } else { - return - } - - // lookup name in extended attributes - if attrBytes, err = backend.Get(ctx, deletedNodePath, prefixes.NameAttr); err == nil { - recycleNode.Name = string(attrBytes) - } else { - return - } - - // get origin node, is relative to space root - origin = "/" - - // lookup origin path in extended attributes - if attrBytes, err = backend.Get(ctx, resolvedTrashItem, prefixes.TrashOriginAttr); err == nil { - origin = filepath.Join(string(attrBytes), path) - } else { - logger.Error().Err(err).Str("trashItem", trashItem).Str("deletedNodePath", deletedNodePath).Msg("could not read origin path, restoring to /") - } - - return -} - -func getNodeIDFromCache(ctx context.Context, path string, cache store.Store) string { - _, span := tracer.Start(ctx, "getNodeIDFromCache") - defer span.End() - recs, err := cache.Read(path) - if err == nil && len(recs) > 0 { - return string(recs[0].Value) - } - return "" -} - -func storeNodeIDInCache(ctx context.Context, path string, nodeID string, cache store.Store) error { - _, span := tracer.Start(ctx, "storeNodeIDInCache") - defer span.End() - return cache.Write(&store.Record{ - Key: path, - Value: []byte(nodeID), - }) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload.go deleted file mode 100644 index e44c564a65..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload.go +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package decomposedfs - -import ( - "context" - "fmt" - "os" - "path/filepath" - "strings" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - tusd "github.com/tus/tusd/v2/pkg/handler" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload" - "github.com/opencloud-eu/reva/v2/pkg/storagespace" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -// Upload uploads data to the given resource -// TODO(OCISDEV-901): remove Upload once all drivers are migrated to CommitUpload and the coordinator (OCISDEV-900) is in place. -func (fs *Decomposedfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - _, span := tracer.Start(ctx, "Upload") - defer span.End() - up, err := fs.GetUpload(ctx, req.Ref.GetPath()) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error retrieving upload") - } - - session := up.(*upload.OcisSession) - - ctx = session.Context(ctx) - - if session.Chunk() != "" { // check chunking v1 - p, assembledFile, err := fs.chunkHandler.WriteChunk(session.Chunk(), req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } - if p == "" { - if err = session.Terminate(ctx); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error removing auxiliary files") - } - return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) - } - fd, err := os.Open(assembledFile) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error opening assembled file") - } - defer fd.Close() - defer os.RemoveAll(assembledFile) - req.Body = fd - - size, err := session.WriteChunk(ctx, 0, req.Body) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") - } - session.SetSize(size) - } else { - size, err := session.WriteChunk(ctx, 0, req.Body) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "Decomposedfs: error writing to binary file") - } - if size != req.Length { - return &provider.ResourceInfo{}, errtypes.PartialContent("Decomposedfs: unexpected end of stream") - } - } - - if err := session.FinishUploadDecomposed(ctx); err != nil { - return &provider.ResourceInfo{}, err - } - - if uff != nil { - uploadRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.SpaceID(), - }, - Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), - } - executant := session.Executant() - uff(session.SpaceOwner(), &executant, uploadRef) - } - - ri := &provider.ResourceInfo{ - // fill with at least fileid, mtime and etag - Id: &provider.ResourceId{ - StorageId: session.ProviderID(), - SpaceId: session.SpaceID(), - OpaqueId: session.NodeID(), - }, - } - - // add etag to metadata - ri.Etag, _ = node.CalculateEtag(session.NodeID(), session.MTime()) - - if !session.MTime().IsZero() { - ri.Mtime = utils.TimeToTS(session.MTime()) - } - - return ri, nil -} - -// InitiateUpload returns upload ids corresponding to different protocols it supports -// TODO(OCISDEV-901): remove InitiateUpload once all drivers are migrated to CommitUpload and the coordinator (OCISDEV-900) is in place. -func (fs *Decomposedfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - _, span := tracer.Start(ctx, "InitiateUpload") - defer span.End() - log := appctx.GetLogger(ctx) - - // remember the path from the reference - refpath := ref.GetPath() - var chunk *chunking.ChunkBLOBInfo - var err error - if chunking.IsChunked(refpath) { // check chunking v1 - chunk, err = chunking.GetChunkBLOBInfo(refpath) - if err != nil { - return nil, errtypes.BadRequest(err.Error()) - } - ref.Path = chunk.Path - } - n, err := fs.lu.NodeFromResource(ctx, ref) - switch err.(type) { - case nil: - // ok - case errtypes.IsNotFound: - return nil, errtypes.PreconditionFailed(err.Error()) - default: - return nil, err - } - - // permissions are checked in NewUpload below - - relative, err := fs.lu.Path(ctx, n, node.NoCheck) - // TODO why do we need the path here? - // jfd: it is used later when emitting the UploadReady event ... - // AAAND refPath might be . when accessing with an id / relative reference ... which causes NodeName to become . But then dir will also always be . - // That is why we still have to read the path here: so that the event we emit contains a relative reference with a path relative to the space root. WTF - if err != nil { - return nil, err - } - - lockID, _ := ctxpkg.ContextGetLockID(ctx) - - session := fs.sessionStore.New(ctx) - session.SetMetadata("filename", n.Name) - session.SetStorageValue("NodeName", n.Name) - if chunk != nil { - session.SetStorageValue("Chunk", filepath.Base(refpath)) - } - session.SetMetadata("dir", filepath.Dir(relative)) - session.SetStorageValue("Dir", filepath.Dir(relative)) - session.SetMetadata("lockid", lockID) - - session.SetSize(uploadLength) - session.SetStorageValue("SpaceRoot", n.SpaceRoot.ID) // TODO SpaceRoot -> SpaceID - session.SetStorageValue("SpaceOwnerOrManager", n.SpaceOwnerOrManager(ctx).GetOpaqueId()) // TODO needed for what? - - spaceGID, ok := ctx.Value(CtxKeySpaceGID).(uint32) - if ok { - session.SetStorageValue("SpaceGid", fmt.Sprintf("%d", spaceGID)) - } - - iid, _ := ctxpkg.ContextGetInitiator(ctx) - session.SetMetadata("initiatorid", iid) - - if metadata != nil { - session.SetMetadata("providerID", metadata["providerID"]) - if mtime, ok := metadata["mtime"]; ok { - if mtime != "null" { - session.SetMetadata("mtime", metadata["mtime"]) - } - } - if expiration, ok := metadata["expires"]; ok { - if expiration != "null" { - session.SetMetadata("expires", metadata["expires"]) - } - } - if _, ok := metadata["sizedeferred"]; ok { - session.SetSizeIsDeferred(true) - } - if checksum, ok := metadata["checksum"]; ok { - parts := strings.SplitN(checksum, " ", 2) - if len(parts) != 2 { - return nil, errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1", "md5", "adler32": - session.SetMetadata("checksum", checksum) - default: - return nil, errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - } - - // only check preconditions if they are not empty // TODO or is this a bad request? - if metadata["if-match"] != "" { - session.SetMetadata("if-match", metadata["if-match"]) - } - if metadata["if-none-match"] != "" { - session.SetMetadata("if-none-match", metadata["if-none-match"]) - } - if metadata["if-unmodified-since"] != "" { - session.SetMetadata("if-unmodified-since", metadata["if-unmodified-since"]) - } - } - - if session.MTime().IsZero() { - session.SetMetadata("mtime", utils.TimeToOCMtime(time.Now())) - } - - log.Debug().Str("uploadid", session.ID()).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Interface("metadata", metadata).Msg("Decomposedfs: resolved filename") - - _, err = node.CheckQuota(ctx, n.SpaceRoot, n.Exists, uint64(n.Blobsize), uint64(session.Size())) - if err != nil { - return nil, err - } - - if session.Filename() == "" { - return nil, errors.New("Decomposedfs: missing filename in metadata") - } - if session.Dir() == "" { - return nil, errors.New("Decomposedfs: missing dir in metadata") - } - - // the parent owner will become the new owner - parent, perr := n.Parent(ctx) - if perr != nil { - return nil, errors.Wrap(perr, "Decomposedfs: error getting parent "+n.ParentID) - } - - // check permissions - var ( - checkNode *node.Node - path string - ) - if n.Exists { - // check permissions of file to be overwritten - checkNode = n - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }}) - } else { - // check permissions of parent - checkNode = parent - path, _ = storagespace.FormatReference(&provider.Reference{ResourceId: &provider.ResourceId{ - SpaceId: checkNode.SpaceID, - OpaqueId: checkNode.ID, - }, Path: n.Name}) - } - rp, err := fs.p.AssemblePermissions(ctx, checkNode) - switch { - case err != nil: - return nil, err - case !rp.InitiateFileUpload: - return nil, errtypes.PermissionDenied(path) - } - - // are we trying to overwriting a folder with a file? - if n.Exists && n.IsDir(ctx) { - return nil, errtypes.PreconditionFailed("resource is not a file") - } - - // check lock - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - usr := ctxpkg.ContextMustGetUser(ctx) - - // fill future node info - if n.Exists { - if session.HeaderIfNoneMatch() == "*" { - return nil, errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", n.ParentID, n.Name, n.ID)) - } - session.SetStorageValue("NodeId", n.ID) - session.SetStorageValue("NodeExists", "true") - } else { - session.SetStorageValue("NodeId", uuid.New().String()) - } - session.SetStorageValue("NodeParentId", n.ParentID) - session.SetExecutant(usr) - session.SetStorageValue("LogLevel", log.GetLevel().String()) - - log.Debug().Interface("session", session).Msg("Decomposedfs: built session info") - - err = fs.um.RunInBaseScope(func() error { - // Create binary file in the upload folder with no content - // It will be used when determining the current offset of an upload - err := session.TouchBin() - if err != nil { - return err - } - - return session.Persist(ctx) - }) - if err != nil { - return nil, err - } - metrics.UploadSessionsInitiated.Inc() - - if uploadLength == 0 { - // Directly finish this upload - err = session.FinishUploadDecomposed(ctx) - if err != nil { - return nil, err - } - } - - return map[string]string{ - "simple": session.ID(), - "tus": session.ID(), - }, nil -} - -// MarkProcessing toggles a processing flag on the resource. -func (fs *Decomposedfs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - return err - } - if !n.Exists { - return errtypes.NotFound(ref.String()) - } - - // Early lock, so MarkProcessing is atomic. - f, err := lockedfile.OpenFile(fs.lu.MetadataBackend().LockfilePath(n.InternalPath()), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return err - } - defer func() { - if cerr := f.Close(); cerr != nil { - appctx.GetLogger(ctx).Error().Err(cerr).Str("nodeid", n.ID).Msg("could not close mark-processing lock") - } - }() - - // Evict the node's in-process xattr cache so IsProcessing reads from disk while we hold the lock. - n.ResetXattrsCache() - - if !processing { - if !n.IsProcessing(ctx) { - return nil - } - id, _ := n.ProcessingID(ctx) - if id != sessionID { - return nil // owned by a different session, do not clear - } - return n.RemoveXattr(ctx, prefixes.StatusPrefix, false) - } - - if n.IsProcessing(ctx) { - return errtypes.ResourceProcessing(ref.String()) - } - return n.SetXattrsWithContext(ctx, node.Attributes{ - prefixes.StatusPrefix: []byte(node.ProcessingStatus + sessionID), - }, false) // acquireLock=false, because outer lock already held -} - -// CommitUpload writes the staged bytes from source to the resource at ref. -// sessionID is used to identify the correct blob slot prepared before postprocessing. -// Caller owns source.Body and must close it after CommitUpload returns. -func (fs *Decomposedfs) CommitUpload(ctx context.Context, ref *provider.Reference, sessionID string, source storage.UploadSource) error { - if source.Body == nil { - return errtypes.BadRequest("Decomposedfs: source body is nil") - } - if sessionID == "" { - return errtypes.BadRequest("Decomposedfs: sessionID is empty") - } - - n, err := fs.lu.NodeFromResource(ctx, ref) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("CommitUpload: unexpected NodeFromResource failure") - return errtypes.InternalError("CommitUpload: node lookup failed unexpectedly") - } - if !n.Exists { - return errtypes.NotFound(ref.String()) - } - - blobNode := node.New(n.SpaceID, n.ID, "", "", source.Length, sessionID, - provider.ResourceType_RESOURCE_TYPE_FILE, nil, fs.lu) - - if err := fs.tp.WriteBlobFromReader(blobNode, source.Body, source.Length); err != nil { - if derr := fs.tp.DeleteBlob(blobNode); derr != nil { - appctx.GetLogger(ctx).Error().Err(derr).Str("nodeid", n.ID).Str("blobid", sessionID).Msg("could not clean up orphaned blob after failed write") - } - return errors.Wrap(err, "Decomposedfs: failed to write blob") - } - - now := time.Now() - if p, err := n.Parent(ctx); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not read parent for etag propagation") - } else { - _ = p.SetTMTime(ctx, &now) - if err := fs.tp.Propagate(ctx, p, 0); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Msg("could not propagate etag change") - } - } - - return nil -} - -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *Decomposedfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - composer.UseConcater(fs) - composer.UseLengthDeferrer(fs) -} - -// To implement the core tus.io protocol as specified in https://tus.io/protocols/resumable-upload.html#core-protocol -// - the storage needs to implement NewUpload and GetUpload -// - the upload needs to implement the tusd.Upload interface: WriteChunk, GetInfo, GetReader and FinishUpload - -// NewUpload returns a new tus Upload instance -func (fs *Decomposedfs) NewUpload(ctx context.Context, info tusd.FileInfo) (tusd.Upload, error) { - return nil, fmt.Errorf("not implemented, use InitiateUpload on the CS3 API to start a new upload") -} - -// GetUpload returns the Upload for the given upload id -func (fs *Decomposedfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - var ul tusd.Upload - var err error - _ = fs.um.RunInBaseScope(func() error { - ul, err = fs.sessionStore.Get(ctx, id) - return nil - }) - return ul, err -} - -// ListUploadSessions returns the upload sessions for the given filter -func (fs *Decomposedfs) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { - var sessions []*upload.OcisSession - if filter.ID != nil && *filter.ID != "" { - session, err := fs.sessionStore.Get(ctx, *filter.ID) - if err != nil { - return nil, err - } - sessions = []*upload.OcisSession{session} - } else { - var err error - sessions, err = fs.sessionStore.List(ctx) - if err != nil { - return nil, err - } - } - filteredSessions := []storage.UploadSession{} - now := time.Now() - for _, session := range sessions { - if filter.Processing != nil && *filter.Processing != session.IsProcessing() { - continue - } - if filter.Expired != nil { - if *filter.Expired { - if now.Before(session.Expires()) { - continue - } - } else { - if now.After(session.Expires()) { - continue - } - } - } - if filter.HasVirus != nil { - sr, _ := session.ScanData() - infected := sr != "" - if *filter.HasVirus != infected { - continue - } - } - filteredSessions = append(filteredSessions, session) - } - return filteredSessions, nil -} - -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (fs *Decomposedfs) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.OcisSession) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (fs *Decomposedfs) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.OcisSession) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (fs *Decomposedfs) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.OcisSession) -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go deleted file mode 100644 index 22ff4d8995..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/session.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2018-2023 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/json" - "os" - "path/filepath" - "strconv" - "time" - - "github.com/google/renameio/v2" - tusd "github.com/tus/tusd/v2/pkg/handler" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -// OcisSession extends tus upload lifecycle with postprocessing steps. -type OcisSession struct { - store OcisStore - // for now, we keep the json files in the uploads folder - info tusd.FileInfo -} - -// Context returns a context with the user, logger and lockid used when initiating the upload session -func (s *OcisSession) Context(ctx context.Context) context.Context { // restore logger from file info - sub := s.store.log.With().Int("pid", os.Getpid()).Logger() - ctx = appctx.WithLogger(ctx, &sub) - ctx = ctxpkg.ContextSetLockID(ctx, s.lockID()) - ctx = ctxpkg.ContextSetUser(ctx, s.executantUser()) - return ctxpkg.ContextSetInitiator(ctx, s.InitiatorID()) -} - -func (s *OcisSession) lockID() string { - return s.info.MetaData["lockid"] -} -func (s *OcisSession) executantUser() *userpb.User { - var o *typespb.Opaque - _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) - return &userpb.User{ - Id: &userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), - Idp: s.info.Storage["Idp"], - OpaqueId: s.info.Storage["UserId"], - }, - Username: s.info.Storage["UserName"], - DisplayName: s.info.Storage["UserDisplayName"], - Opaque: o, - } -} - -// Purge deletes the upload session metadata and written binary data -func (s *OcisSession) Purge(ctx context.Context) { - _, span := tracer.Start(ctx, "Purge") - defer span.End() - s.Cleanup(true, true, true, true) -} - -// TouchBin creates a file to contain the binary data. It's size will be used to keep track of the tus upload offset. -func (s *OcisSession) TouchBin() error { - file, err := os.OpenFile(s.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return err - } - return file.Close() -} - -// Persist writes the upload session metadata to disk -// events can update the scan outcome and the finished event might read an empty file because of race conditions -// so we need to lock the file while writing and use atomic writes -func (s *OcisSession) Persist(ctx context.Context) error { - _, span := tracer.Start(ctx, "Persist") - defer span.End() - infoPath := s.infoPath() - // create folder structure (if needed) - if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil { - return err - } - - var d []byte - d, err := json.Marshal(s.info) - if err != nil { - return err - } - return renameio.WriteFile(infoPath, d, 0600) -} - -// ToFileInfo returns tus compatible FileInfo so the tus handler can access the upload offset -func (s *OcisSession) ToFileInfo() tusd.FileInfo { - return s.info -} - -// ProviderID returns the provider id -func (s *OcisSession) ProviderID() string { - return s.info.MetaData["providerID"] -} - -// SpaceID returns the space id -func (s *OcisSession) SpaceID() string { - return s.info.Storage["SpaceRoot"] -} - -// NodeID returns the node id -func (s *OcisSession) NodeID() string { - return s.info.Storage["NodeId"] -} - -// NodeParentID returns the nodes parent id -func (s *OcisSession) NodeParentID() string { - return s.info.Storage["NodeParentId"] -} - -// NodeExists returns wether or not the node existed during InitiateUpload. -// FIXME If two requests try to write the same file they both will store a new -// random node id in the session and try to initialize a new node when -// finishing the upload. The second request will fail with an already exists -// error when trying to create the symlink for the node in the parent directory. -// A node should be created as part of InitiateUpload. When listing a directory -// we can decide if we want to skip the entry, or expose uploed progress -// information. But that is a bigger change and might involve client work. -func (s *OcisSession) NodeExists() bool { - return s.info.Storage["NodeExists"] == "true" -} - -// HeaderIfMatch returns the if-match header for the upload session -func (s *OcisSession) HeaderIfMatch() string { - return s.info.MetaData["if-match"] -} - -// HeaderIfNoneMatch returns the if-none-match header for the upload session -func (s *OcisSession) HeaderIfNoneMatch() string { - return s.info.MetaData["if-none-match"] -} - -// HeaderIfUnmodifiedSince returns the if-unmodified-since header for the upload session -func (s *OcisSession) HeaderIfUnmodifiedSince() string { - return s.info.MetaData["if-unmodified-since"] -} - -// Node returns the node for the session -func (s *OcisSession) Node(ctx context.Context) (*node.Node, error) { - return node.ReadNode(ctx, s.store.lu, s.SpaceID(), s.info.Storage["NodeId"], false, nil, true) -} - -// ID returns the upload session id -func (s *OcisSession) ID() string { - return s.info.ID -} - -// Filename returns the name of the node which is not the same as the name af the file being uploaded for legacy chunked uploads -func (s *OcisSession) Filename() string { - return s.info.Storage["NodeName"] -} - -// Chunk returns the chunk name when a legacy chunked upload was started -func (s *OcisSession) Chunk() string { - return s.info.Storage["Chunk"] -} - -// SetMetadata is used to fill the upload metadata that will be exposed to the end user -func (s *OcisSession) SetMetadata(key, value string) { - s.info.MetaData[key] = value -} - -// SetStorageValue is used to set metadata only relevant for the upload session implementation -func (s *OcisSession) SetStorageValue(key, value string) { - s.info.Storage[key] = value -} - -// SetSize will set the upload size of the underlying tus info. -func (s *OcisSession) SetSize(size int64) { - s.info.Size = size -} - -// SetSizeIsDeferred is uset to change the SizeIsDeferred property of the underlying tus info. -func (s *OcisSession) SetSizeIsDeferred(value bool) { - s.info.SizeIsDeferred = value -} - -// Dir returns the directory to which the upload is made -// TODO get rid of Dir(), whoever consumes the reference should be able to deal -// with a relative reference. -// Dir is only used to: -// - fill the Path property when emitting the UploadReady event after -// postprocessing finished. I wonder why the UploadReady contains a finished -// flag ... maybe multiple distinct events would make more sense. -// - build the reference that is passed to the FileUploaded event in the -// UploadFinishedFunc callback passed to the Upload call used for simple -// datatx put requests -// -// AFAICT only search and audit services consume the path. -// - search needs to index from the root anyway. And it only needs the most -// recent path to put it in the index. So it should already be able to deal -// with an id based reference. -// - audit on the other hand needs to log events with the path at the state of -// the event ... so it does need the full path. -// -// I think we can safely determine the path later, right before emitting the -// event. And maybe make it configurable, because only audit needs it, anyway. -func (s *OcisSession) Dir() string { - return s.info.Storage["Dir"] -} - -// Size returns the upload size -func (s *OcisSession) Size() int64 { - return s.info.Size -} - -// SizeDiff returns the size diff that was calculated after postprocessing -func (s *OcisSession) SizeDiff() int64 { - sizeDiff, _ := strconv.ParseInt(s.info.MetaData["sizeDiff"], 10, 64) - return sizeDiff -} - -// Reference returns a reference that can be used to access the uploaded resource -func (s *OcisSession) Reference() provider.Reference { - return provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: s.info.MetaData["providerID"], - SpaceId: s.info.Storage["SpaceRoot"], - OpaqueId: s.info.Storage["NodeId"], - }, - // Path is not used - } -} - -// Executant returns the id of the user that initiated the upload session -func (s *OcisSession) Executant() userpb.UserId { - return userpb.UserId{ - Type: userpb.UserType(userpb.UserType_value[s.info.Storage["UserType"]]), - Idp: s.info.Storage["Idp"], - OpaqueId: s.info.Storage["UserId"], - } -} - -// SetExecutant is used to remember the user that initiated the upload session -func (s *OcisSession) SetExecutant(u *userpb.User) { - s.info.Storage["Idp"] = u.GetId().GetIdp() - s.info.Storage["UserId"] = u.GetId().GetOpaqueId() - s.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type) - s.info.Storage["UserName"] = u.GetUsername() - s.info.Storage["UserDisplayName"] = u.GetDisplayName() - - b, _ := json.Marshal(u.GetOpaque()) - s.info.Storage["UserOpaque"] = string(b) -} - -// Offset returns the current upload offset -func (s *OcisSession) Offset() int64 { - return s.info.Offset -} - -// SpaceOwner returns the id of the space owner -func (s *OcisSession) SpaceOwner() *userpb.UserId { - return &userpb.UserId{ - // idp and type do not seem to be consumed and the node currently only stores the user id anyway - OpaqueId: s.info.Storage["SpaceOwnerOrManager"], - } -} - -// Expires returns the time the upload session expires -func (s *OcisSession) Expires() time.Time { - var t time.Time - if value, ok := s.info.MetaData["expires"]; ok { - t, _ = utils.MTimeToTime(value) - } - return t -} - -// MTime returns the mtime to use for the uploaded file -func (s *OcisSession) MTime() time.Time { - var t time.Time - if value, ok := s.info.MetaData["mtime"]; ok { - t, _ = utils.MTimeToTime(value) - } - return t -} - -// IsProcessing returns true if all bytes have been received. The session then has entered postprocessing state. -func (s *OcisSession) IsProcessing() bool { - // We might need a more sophisticated way to determine processing status soon - return s.info.Size == s.info.Offset && s.info.MetaData["scanResult"] == "" -} - -// binPath returns the path to the file storing the binary data. -func (s *OcisSession) binPath() string { - return filepath.Join(s.store.root, "uploads", s.info.ID) -} - -// infoPath returns the path to the .info file storing the file's info. -func (s *OcisSession) infoPath() string { - return sessionPath(s.store.root, s.info.ID) -} - -// InitiatorID returns the id of the initiating client -func (s *OcisSession) InitiatorID() string { - return s.info.MetaData["initiatorid"] -} - -// SetScanData sets virus scan data to the upload session -func (s *OcisSession) SetScanData(result string, date time.Time) { - s.info.MetaData["scanResult"] = result - s.info.MetaData["scanDate"] = date.Format(time.RFC3339) -} - -// ScanData returns the virus scan data -func (s *OcisSession) ScanData() (string, time.Time) { - date := s.info.MetaData["scanDate"] - if date == "" { - return "", time.Time{} - } - d, _ := time.Parse(time.RFC3339, date) - return s.info.MetaData["scanResult"], d -} - -func (s *OcisSession) SetStatus(status string) { - s.info.MetaData["status"] = status -} - -func (s *OcisSession) Status() string { - return s.info.MetaData["status"] -} - -func (s *OcisSession) SetStatusMessage(message string) { - s.info.MetaData["status_message"] = message -} - -func (s *OcisSession) StatusMessage() string { - return s.info.MetaData["status_message"] -} - -// sessionPath returns the path to the .info file storing the file's info. -func sessionPath(root, id string) string { - return filepath.Join(root, "uploads", id+".info") -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/store.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/store.go deleted file mode 100644 index 63b66ef076..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/store.go +++ /dev/null @@ -1,423 +0,0 @@ -// Copyright 2018-2022 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/json" - "fmt" - iofs "io/fs" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "syscall" - "time" - - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper" - "github.com/pkg/errors" - "github.com/rogpeppe/go-internal/lockedfile" - "github.com/rs/zerolog" - tusd "github.com/tus/tusd/v2/pkg/handler" -) - -var _idRegexp = regexp.MustCompile(".*/([^/]+).info") - -// PermissionsChecker defines an interface for checking permissions on a Node -type PermissionsChecker interface { - AssemblePermissions(ctx context.Context, n *node.Node) (ap provider.ResourcePermissions, err error) -} - -// OcisStore manages upload sessions -type OcisStore struct { - fs storage.FS - lu node.PathLookup - tp node.Tree - um usermapper.Mapper - root string - pub events.Publisher - async bool - tknopts options.TokenOptions - disableVersioning bool - log *zerolog.Logger -} - -// NewSessionStore returns a new OcisStore -func NewSessionStore(fs storage.FS, aspects aspects.Aspects, root string, async bool, tknopts options.TokenOptions, log *zerolog.Logger) *OcisStore { - return &OcisStore{ - fs: fs, - lu: aspects.Lookup, - tp: aspects.Tree, - root: root, - pub: aspects.EventStream, - async: async, - tknopts: tknopts, - disableVersioning: aspects.DisableVersioning, - um: aspects.UserMapper, - log: log, - } -} - -// New returns a new upload session -func (store OcisStore) New(ctx context.Context) *OcisSession { - return &OcisSession{ - store: store, - info: tusd.FileInfo{ - ID: uuid.New().String(), - Storage: map[string]string{ - "Type": "OCISStore", - }, - MetaData: tusd.MetaData{}, - }, - } -} - -// List lists all upload sessions -func (store OcisStore) List(ctx context.Context) ([]*OcisSession, error) { - uploads := []*OcisSession{} - infoFiles, err := filepath.Glob(filepath.Join(store.root, "uploads", "*.info")) - if err != nil { - return nil, err - } - - for _, info := range infoFiles { - id := strings.TrimSuffix(filepath.Base(info), filepath.Ext(info)) - progress, err := store.Get(ctx, id) - if err != nil { - appctx.GetLogger(ctx).Error().Interface("path", info).Msg("Decomposedfs: could not getUploadSession") - continue - } - - uploads = append(uploads, progress) - } - return uploads, nil -} - -// Get returns the upload session for the given upload id -func (store OcisStore) Get(ctx context.Context, id string) (*OcisSession, error) { - sessionPath := sessionPath(store.root, id) - match := _idRegexp.FindStringSubmatch(sessionPath) - if len(match) < 2 { - return nil, fmt.Errorf("invalid upload path") - } - - session := OcisSession{ - store: store, - info: tusd.FileInfo{}, - } - data, err := os.ReadFile(sessionPath) - if err != nil { - // handle stale NFS file handles that can occur when the file is deleted betwenn the ATTR and FOPEN call of os.ReadFile - if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE { - appctx.GetLogger(ctx).Info().Str("session", id).Err(err).Msg("treating stale file handle as not found") - err = tusd.ErrNotFound - } - if errors.Is(err, iofs.ErrNotExist) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - - if err := json.Unmarshal(data, &session.info); err != nil { - return nil, err - } - - stat, err := os.Stat(session.binPath()) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - - session.info.Offset = stat.Size() - - return &session, nil -} - -// Session is the interface used by the Cleanup call -type Session interface { - ID() string - Node(ctx context.Context) (*node.Node, error) - Context(ctx context.Context) context.Context - Cleanup(revertNodeMetadata, cleanBin, cleanInfo bool) -} - -// CreateNodeForUpload will create the target node for the Upload -// TODO move this to the node package as NodeFromUpload? -// should we in InitiateUpload create the node first? and then the upload? -func (store OcisStore) CreateNodeForUpload(ctx context.Context, session *OcisSession, initAttrs node.Attributes) (*node.Node, error) { - ctx, span := tracer.Start(session.Context(ctx), "CreateNodeForUpload") - defer span.End() - n := node.New( - session.SpaceID(), - session.NodeID(), - session.NodeParentID(), - session.Filename(), - session.Size(), - session.ID(), - provider.ResourceType_RESOURCE_TYPE_FILE, - nil, - store.lu, - ) - var err error - n.SpaceRoot, err = node.ReadNode(ctx, store.lu, session.SpaceID(), session.SpaceID(), false, nil, false) - if err != nil { - return nil, err - } - - // check lock - if err := n.CheckLock(ctx); err != nil { - return nil, err - } - - var unlock metadata.UnlockFunc - if session.NodeExists() { // TODO this is wrong. The node should be created when the upload starts, the revisions should be created independently of the node - // we do not need to propagate a change when a node is created, only when the upload is ready. - // that still creates problems for desktop clients because if another change causes propagation it will detects an empty file - // so the first upload has to point to the first revision with the expected size. The file cannot be downloaded, but it can be overwritten (which will create a new revision and make the node reflect the latest revision) - // any finished postprocessing will not affect the node metadata. - // *thinking* but then initializing an upload will lock the file until the upload has finished. That sucks. - // so we have to check if the node has been created meanwhile (well, only in case the upload does not know the nodeid ... or the NodeExists array that is checked by session.NodeExists()) - // FIXME look at the disk again to see if the file has been created in between, or just try initializing a new node and do the update existing node as a fallback. <- the latter! - - unlock, err = store.updateExistingNode(ctx, session, n, session.SpaceID(), uint64(session.Size())) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("failed to update existing node") - } - } else { - if c, ok := store.lu.(node.IDCacher); ok { - err := c.CacheID(ctx, n.SpaceID, n.ID, filepath.Join(n.ParentPath(), n.Name)) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Msg("failed to cache id") - } - } - - unlock, err = store.tp.InitNewNode(ctx, n, uint64(session.Size())) - if err != nil { - appctx.GetLogger(ctx).Error().Str("path", n.InternalPath()).Err(err).Msg("failed to init new node") - } - session.info.MetaData["sizeDiff"] = strconv.FormatInt(session.Size(), 10) - } - defer func() { - if unlock == nil { - appctx.GetLogger(ctx).Info().Msg("did not get a unlockfunc, not unlocking") - return - } - - if err := unlock(); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodeid", n.ID).Str("parentid", n.ParentID).Msg("could not close lock") - } - }() - if err != nil { - return nil, err - } - - // overwrite technical information - initAttrs.SetString(prefixes.IDAttr, n.ID) - initAttrs.SetInt64(prefixes.TypeAttr, int64(provider.ResourceType_RESOURCE_TYPE_FILE)) - initAttrs.SetString(prefixes.ParentidAttr, n.ParentID) - initAttrs.SetString(prefixes.NameAttr, n.Name) - initAttrs.SetString(prefixes.BlobIDAttr, n.BlobID) - initAttrs.SetInt64(prefixes.BlobsizeAttr, n.Blobsize) - initAttrs.SetString(prefixes.StatusPrefix, node.ProcessingStatus+session.ID()) - - // set mtime on the new node - mtime := time.Now() - if !session.MTime().IsZero() { - // overwrite mtime if requested - mtime = session.MTime() - } - err = store.lu.TimeManager().OverrideMtime(ctx, n, &initAttrs, mtime) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: failed to set the mtime") - } - - // update node metadata with new blobid etc - err = n.SetXattrsWithContext(ctx, initAttrs, false) - if err != nil { - return nil, errors.Wrap(err, "Decomposedfs: could not write metadata") - } - - err = store.um.RunInBaseScope(func() error { - return session.Persist(ctx) - }) - if err != nil { - return nil, err - } - - return n, nil -} - -func (store OcisStore) updateExistingNode(ctx context.Context, session *OcisSession, n *node.Node, spaceID string, fsize uint64) (metadata.UnlockFunc, error) { - _, span := tracer.Start(ctx, "updateExistingNode") - defer span.End() - targetPath := n.InternalPath() - - // write lock existing node before reading any metadata - f, err := lockedfile.OpenFile(store.lu.MetadataBackend().LockfilePath(targetPath), os.O_RDWR|os.O_CREATE, 0600) - if err != nil { - return nil, err - } - - unlock := func() error { - // NOTE: to prevent stale NFS file handles do not remove lock file! - return f.Close() - } - - old, _ := node.ReadNode(ctx, store.lu, spaceID, n.ID, false, nil, false) - if _, err := node.CheckQuota(ctx, n.SpaceRoot, true, uint64(old.Blobsize), fsize); err != nil { - return unlock, err - } - - oldNodeMtime, err := old.GetMTime(ctx) - if err != nil { - return unlock, err - } - oldNodeEtag, err := node.CalculateEtag(old.ID, oldNodeMtime) - if err != nil { - return unlock, err - } - - // When the if-match header was set we need to check if the - // etag still matches before finishing the upload. - if session.HeaderIfMatch() != "" && session.HeaderIfMatch() != oldNodeEtag { - return unlock, errtypes.Aborted("etag mismatch") - } - - // When the if-none-match header was set we need to check if any of the - // etags matches before finishing the upload. - if session.HeaderIfNoneMatch() != "" { - if session.HeaderIfNoneMatch() == "*" { - return unlock, errtypes.Aborted("etag mismatch, resource exists") - } - for _, ifNoneMatchTag := range strings.Split(session.HeaderIfNoneMatch(), ",") { - if ifNoneMatchTag == oldNodeEtag { - return unlock, errtypes.Aborted("etag mismatch") - } - } - } - - // When the if-unmodified-since header was set we need to check if the - // etag still matches before finishing the upload. - if session.HeaderIfUnmodifiedSince() != "" { - ifUnmodifiedSince, err := time.Parse(time.RFC3339Nano, session.HeaderIfUnmodifiedSince()) - if err != nil { - return unlock, errtypes.InternalError(fmt.Sprintf("failed to parse if-unmodified-since time: %s", err)) - } - - if oldNodeMtime.After(ifUnmodifiedSince) { - return unlock, errtypes.Aborted("if-unmodified-since mismatch") - } - } - - if !store.disableVersioning { - versionPath := session.store.lu.InternalPath(spaceID, n.ID+node.RevisionIDDelimiter+oldNodeMtime.UTC().Format(time.RFC3339Nano)) - - // create version node - _, err := os.OpenFile(versionPath, os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - if !errors.Is(err, os.ErrExist) { - return unlock, err - } - - // a revision with this mtime does already exist. - // If the blobs are the same we can just delete the old one - if err := validateChecksums(ctx, old, session, versionPath); err != nil { - return unlock, err - } - - // delete old blob - bID, _, err := session.store.lu.ReadBlobIDAndSizeAttr(ctx, versionPath, nil) - if err != nil { - return unlock, err - } - if err := session.store.tp.DeleteBlob(&node.Node{BlobID: bID, SpaceID: n.SpaceID}); err != nil { - return unlock, err - } - - // clean revision file - span.AddEvent("os.Create") - if _, err := os.Create(versionPath); err != nil { - return unlock, err - } - } - - // copy blob metadata to version node - if err := store.lu.CopyMetadataWithSourceLock(ctx, targetPath, versionPath, func(attributeName string, value []byte) (newValue []byte, copy bool) { - return value, strings.HasPrefix(attributeName, prefixes.ChecksumPrefix) || - attributeName == prefixes.TypeAttr || - attributeName == prefixes.BlobIDAttr || - attributeName == prefixes.BlobsizeAttr || - attributeName == prefixes.MTimeAttr - }, f, true); err != nil { - return unlock, err - } - session.info.MetaData["versionsPath"] = versionPath - // keep mtime from previous version - span.AddEvent("os.Chtimes") - if err := os.Chtimes(session.info.MetaData["versionsPath"], oldNodeMtime, oldNodeMtime); err != nil { - return unlock, errtypes.InternalError(fmt.Sprintf("failed to change mtime of version node: %s", err)) - } - } - - session.info.MetaData["sizeDiff"] = strconv.FormatInt((int64(fsize) - old.Blobsize), 10) - - return unlock, nil -} - -func validateChecksums(ctx context.Context, n *node.Node, session *OcisSession, versionPath string) error { - for _, t := range []string{"md5", "sha1", "adler32"} { - key := prefixes.ChecksumPrefix + t - - checksum, err := n.Xattr(ctx, key) - if err != nil { - return err - } - - revisionChecksum, err := session.store.lu.MetadataBackend().Get(ctx, versionPath, key) - if err != nil { - return err - } - - if string(checksum) == "" || string(revisionChecksum) == "" { - return errors.New("checksum not found") - } - - if string(checksum) != string(revisionChecksum) { - return errors.New("checksum mismatch") - } - } - - return nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go deleted file mode 100644 index b6eb62f71f..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload/upload.go +++ /dev/null @@ -1,427 +0,0 @@ -// Copyright 2018-2022 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package upload - -import ( - "context" - "encoding/hex" - "fmt" - "hash" - "io" - "io/fs" - "net/http" - "os" - "strconv" - "strings" - "time" - - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" - provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/golang-jwt/jwt/v5" - "github.com/pkg/errors" - tusd "github.com/tus/tusd/v2/pkg/handler" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" - - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" - "github.com/opencloud-eu/reva/v2/pkg/events" - "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node" - "github.com/opencloud-eu/reva/v2/pkg/utils" -) - -var ( - tracer trace.Tracer - ErrAlreadyExists = tusd.NewError("ERR_ALREADY_EXISTS", "file already exists", http.StatusConflict) - defaultFilePerm = os.FileMode(0664) -) - -func init() { - tracer = otel.Tracer("github.com/cs3org/reva/pkg/storage/utils/decomposedfs/upload") -} - -// WriteChunk writes the stream from the reader to the given offset of the upload -func (session *OcisSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - ctx, span := tracer.Start(session.Context(ctx), "WriteChunk") - defer span.End() - _, subspan := tracer.Start(ctx, "os.OpenFile") - file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - subspan.End() - - log := appctx.GetLogger(ctx) - if err != nil { - log.Error().Err(err).Msg("WriteChunk: error opening upload file") - return 0, err - } - defer file.Close() - - // calculate cheksum here? needed for the TUS checksum extension. https://tus.io/protocols/resumable-upload.html#checksum - // TODO but how do we get the `Upload-Checksum`? WriteChunk() only has a context, offset and the reader ... - // It is sent with the PATCH request, well or in the POST when the creation-with-upload extension is used - // but the tus handler uses a context.Background() so we cannot really check the header and put it in the context ... - _, subspan = tracer.Start(ctx, "io.Copy") - n, err := io.Copy(file, src) - subspan.End() - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the ocis driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil && err != io.ErrUnexpectedEOF { - log.Error().Err(err).Msg("WriteChunk: error copying data to upload file") - return n, err - } - - // update upload.Session.Offset so subsequent code flow can use it. - // No need to persist the session as the offset is determined by stating the blob in the GetUpload / ReadSession codepath. - // The session offset is written to disk in FinishUpload - session.info.Offset += n - return n, nil -} - -// GetInfo returns the FileInfo -func (session *OcisSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { - return session.ToFileInfo(), nil -} - -// GetReader returns an io.Reader for the upload -func (session *OcisSession) GetReader(ctx context.Context) (io.ReadCloser, error) { - _, span := tracer.Start(session.Context(ctx), "GetReader") - defer span.End() - return os.Open(session.binPath()) -} - -// FinishUpload finishes an upload and moves the file to the internal destination -// implements tusd.DataStore interface -// returns tusd errors -func (session *OcisSession) FinishUpload(ctx context.Context) error { - err := session.FinishUploadDecomposed(ctx) - - if err != nil { - // this is part of the tusd integration and we might be able to - // log the error in another place - log := appctx.GetLogger(ctx) - log.Error().Err(err).Msg("failed to finish upload") - } - - // we need to return a tusd error here to make the tusd handler return the correct status code - switch err.(type) { - case errtypes.AlreadyExists: - return tusd.NewError("ERR_ALREADY_EXISTS", err.Error(), http.StatusConflict) - case errtypes.Aborted: - return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed) - case errtypes.InsufficientStorage: - return tusd.NewError("ERR_INSUFFICIENT_STORAGE", err.Error(), http.StatusInsufficientStorage) - default: - return err - } -} - -// FinishUploadDecomposed finishes an upload and moves the file to the internal destination -// retures errtypes errors -func (session *OcisSession) FinishUploadDecomposed(ctx context.Context) error { - ctx, span := tracer.Start(session.Context(ctx), "FinishUpload") - defer span.End() - log := appctx.GetLogger(ctx) - - ctx = ctxpkg.ContextSetInitiator(ctx, session.InitiatorID()) - - sha1h, md5h, adler32h, err := node.CalculateChecksums(ctx, session.binPath()) - if err != nil { - return err - } - - // compare if they match the sent checksum - // TODO the tus checksum extension would do this on every chunk, but I currently don't see an easy way to pass in the requested checksum. for now we do it in FinishUpload which is also called for chunked uploads - if session.info.MetaData["checksum"] != "" { - var err error - parts := strings.SplitN(session.info.MetaData["checksum"], " ", 2) - if len(parts) != 2 { - return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") - } - switch parts[0] { - case "sha1": - err = checkHash(parts[1], sha1h) - case "md5": - err = checkHash(parts[1], md5h) - case "adler32": - err = checkHash(parts[1], adler32h) - default: - err = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) - } - if err != nil { - session.Cleanup(false, true, true, false) - return err - } - } - - // update checksums - attrs := node.Attributes{ - prefixes.ChecksumPrefix + "sha1": sha1h.Sum(nil), - prefixes.ChecksumPrefix + "md5": md5h.Sum(nil), - prefixes.ChecksumPrefix + "adler32": adler32h.Sum(nil), - } - - // At this point we scope by the space to create the final file in the final location - if session.store.um != nil && session.info.Storage["SpaceGid"] != "" { - gid, err := strconv.Atoi(session.info.Storage["SpaceGid"]) - if err != nil { - return errors.Wrap(err, "failed to parse space gid") - } - - unscope, err := session.store.um.ScopeUserByIds(-1, gid) - if err != nil { - return errors.Wrap(err, "failed to scope user") - } - if unscope != nil { - defer func() { _ = unscope() }() - } - } - - n, err := session.store.CreateNodeForUpload(ctx, session, attrs) - if err != nil { - return err - } - // increase the processing counter for every started processing - // will be decreased in Cleanup() - metrics.UploadProcessing.Inc() - metrics.UploadSessionsBytesReceived.Inc() - - if session.store.pub != nil && session.info.Size > 0 { - u, _ := ctxpkg.ContextGetUser(ctx) - s, err := session.URL(ctx) - if err != nil { - return err - } - - var iu *userpb.User - if utils.ExistsInOpaque(u.Opaque, "impersonating-user") { - iu = &userpb.User{} - if err := utils.ReadJSONFromOpaque(u.Opaque, "impersonating-user", iu); err != nil { - return err - } - } - - if err := events.Publish(ctx, session.store.pub, events.BytesReceived{ - UploadID: session.ID(), - URL: s, - SpaceOwner: n.SpaceOwnerOrManager(session.Context(ctx)), - ExecutingUser: u, - ResourceID: &provider.ResourceId{SpaceId: n.SpaceID, OpaqueId: n.ID}, - Filename: session.Filename(), - Filesize: uint64(session.Size()), - ImpersonatingUser: iu, - }); err != nil { - return err - } - } - - // if the upload is synchronous or the upload is empty, finalize it now - // for 0-byte uploads we take a shortcut and finalize isn't called elsewhere - if !session.store.async || session.info.Size == 0 { - // handle postprocessing synchronously - err = session.Finalize(ctx) - session.Cleanup(err != nil, true, true, true) - if err != nil { - log.Error().Err(err).Msg("failed to upload") - return err - } - metrics.UploadSessionsFinalized.Inc() - } - - return session.store.tp.Propagate(ctx, n, session.SizeDiff()) -} - -// Terminate terminates the upload -func (session *OcisSession) Terminate(_ context.Context) error { - session.Cleanup(true, true, true, true) - return nil -} - -// DeclareLength updates the upload length information -func (session *OcisSession) DeclareLength(ctx context.Context, length int64) error { - session.info.Size = length - session.info.SizeIsDeferred = false - return session.store.um.RunInBaseScope(func() error { - return session.Persist(session.Context(ctx)) - }) -} - -// ConcatUploads concatenates multiple uploads -func (session *OcisSession) ConcatUploads(_ context.Context, uploads []tusd.Upload) (err error) { - file, err := os.OpenFile(session.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return err - } - defer file.Close() - - for _, partialUpload := range uploads { - fileUpload := partialUpload.(*OcisSession) - - src, err := os.Open(fileUpload.binPath()) - if err != nil { - return err - } - defer src.Close() - - if _, err := io.Copy(file, src); err != nil { - return err - } - } - - return -} - -// Finalize finalizes the upload (eg moves the file to the internal destination) -func (session *OcisSession) Finalize(ctx context.Context) (err error) { - ctx, span := tracer.Start(session.Context(ctx), "Finalize") - defer span.End() - - revisionNode := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(), - provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu) - - // upload the data to the blobstore - _, subspan := tracer.Start(ctx, "WriteBlob") - err = session.store.tp.WriteBlob(revisionNode, session.binPath()) - subspan.End() - if err != nil { - return errors.Wrap(err, "failed to upload file to blobstore") - } - - return nil -} - -func checkHash(expected string, h hash.Hash) error { - hash := hex.EncodeToString(h.Sum(nil)) - if expected != hash { - return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %x", expected, hash)) - } - return nil -} - -// cleanup cleans up after the upload is finished -func (session *OcisSession) Cleanup(revertNodeMetadata, cleanBin, cleanInfo, unmarkPostprocessing bool) { - ctx := session.Context(context.Background()) - - if cleanBin { - if err := os.Remove(session.binPath()); err != nil && !errors.Is(err, fs.ErrNotExist) { - appctx.GetLogger(ctx).Error().Str("path", session.binPath()).Err(err).Msg("removing upload failed") - } - } - - if cleanInfo { - if err := os.Remove(session.infoPath()); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("session", session.ID()).Msg("removing upload info failed") - } - } - - if revertNodeMetadata { - n, err := session.Node(ctx) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("sessionid", session.ID()).Msg("reading node for session failed") - return - } - - versionID := strings.TrimPrefix(session.info.MetaData["versionsPath"], n.InternalPath()+node.RevisionIDDelimiter) - if session.NodeExists() && versionID != "" { - if err := n.RevertUpload(ctx, versionID); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", n.InternalPath()).Msg("reverting node metadata failed") - return - } - } else { - // if no other upload session is in progress (processing id != session id) or has finished (processing id == "") - latestSession, err := n.ProcessingID(ctx) - if err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("spaceid", n.SpaceID).Str("nodeid", n.ID).Str("uploadid", session.ID()).Msg("reading processingid for session failed") - } - if latestSession == session.ID() { - // actually delete the node - if err := n.Purge(ctx); err != nil { - appctx.GetLogger(ctx).Error().Err(err).Str("nodepath", n.InternalPath()).Msg("purging node failed") - return - } - } - // FIXME else if the upload has become a revision, delete the revision, or if it is the last one, delete the node - } - } - - if unmarkPostprocessing { - n, err := session.Node(ctx) - if err != nil { - appctx.GetLogger(ctx).Info().Str("session", session.ID()).Err(err).Msg("could not read node") - return - } - // FIXME: after cleanup the node might already be deleted ... - if n != nil { // node can be nil when there was an error before it was created (eg. checksum-mismatch) - if err := n.UnmarkProcessing(ctx, session.ID()); err != nil { - appctx.GetLogger(ctx).Info().Str("path", n.InternalPath()).Err(err).Msg("unmarking processing failed") - } - } - } -} - -// URL returns a url to download an upload -func (session *OcisSession) URL(_ context.Context) (string, error) { - u := joinurl(session.store.tknopts.DownloadEndpoint, "tus/", session.ID()) - if session.store.tknopts.DataGatewayEndpoint == "" { - return u, nil - } - - // we need to create a token - type transferClaims struct { - jwt.RegisteredClaims - Target string `json:"target"` - } - - ttl := time.Duration(session.store.tknopts.TransferExpires) * time.Second - claims := transferClaims{ - RegisteredClaims: jwt.RegisteredClaims{ - ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), - Audience: jwt.ClaimStrings{"reva"}, - IssuedAt: jwt.NewNumericDate(time.Now()), - }, - Target: u, - } - - t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) - - tkn, err := t.SignedString([]byte(session.store.tknopts.TransferSharedSecret)) - if err != nil { - return "", errors.Wrapf(err, "error signing token with claims %+v", claims) - } - - return joinurl(session.store.tknopts.DataGatewayEndpoint, tkn), nil -} - -// replace with url.JoinPath after switching to go1.19 -func joinurl(paths ...string) string { - var s strings.Builder - l := len(paths) - for i, p := range paths { - s.WriteString(p) - if !strings.HasSuffix(p, "/") && i != l-1 { - s.WriteString("/") - } - } - - return s.String() -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper.go deleted file mode 100644 index c5a51376d9..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package usermapper - -import ( - "context" -) - -// Mapper is the interface that wraps the basic mapping methods -type Mapper interface { - RunInBaseScope(f func() error) error - ScopeBase() (func() error, error) - ScopeUser(ctx context.Context) (func() error, error) - ScopeUserByIds(uid, gid int) (func() error, error) -} - -// UnscopeFunc is a function that unscopes the current user -type UnscopeFunc func() error - -// NullMapper is a user mapper that does nothing -type NullMapper struct{} - -// RunInBaseScope runs the given function in the scope of the base user -func (nm *NullMapper) RunInBaseScope(f func() error) error { - return f() -} - -// ScopeBase returns to the base uid and gid returning a function that can be used to restore the previous scope -func (nm *NullMapper) ScopeBase() (func() error, error) { - return func() error { return nil }, nil -} - -// ScopeUser returns to the base uid and gid returning a function that can be used to restore the previous scope -func (nm *NullMapper) ScopeUser(ctx context.Context) (func() error, error) { - return func() error { return nil }, nil -} - -func (nm *NullMapper) ScopeUserByIds(uid, gid int) (func() error, error) { - return func() error { return nil }, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper_linux.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper_linux.go deleted file mode 100644 index e806f6c324..0000000000 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper/usermapper_linux.go +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package usermapper - -import ( - "context" - "fmt" - "os/user" - "runtime" - "strconv" - - "golang.org/x/sys/unix" - - revactx "github.com/opencloud-eu/reva/v2/pkg/ctx" -) - -// UnixMapper is a user mapper that maps users to unix uids and gids -type UnixMapper struct { - baseUid int - baseGid int -} - -// New returns a new user mapper -func NewUnixMapper() *UnixMapper { - baseUid, _ := unix.SetfsuidRetUid(-1) - baseGid, _ := unix.SetfsgidRetGid(-1) - - return &UnixMapper{ - baseUid: baseUid, - baseGid: baseGid, - } -} - -// RunInUserScope runs the given function in the scope of the base user -func (um *UnixMapper) RunInBaseScope(f func() error) error { - unscope, err := um.ScopeBase() - if err != nil { - return err - } - defer func() { _ = unscope() }() - - return f() -} - -// ScopeBase returns to the base uid and gid returning a function that can be used to restore the previous scope -func (um *UnixMapper) ScopeBase() (func() error, error) { - return um.ScopeUserByIds(-1, um.baseGid) -} - -// ScopeUser returns to the base uid and gid returning a function that can be used to restore the previous scope -func (um *UnixMapper) ScopeUser(ctx context.Context) (func() error, error) { - u := revactx.ContextMustGetUser(ctx) - - uid, gid, err := um.mapUser(u.Username) - if err != nil { - return nil, err - } - return um.ScopeUserByIds(uid, gid) -} - -// ScopeUserByIds scopes the current user to the given uid and gid returning a function that can be used to restore the previous scope -func (um *UnixMapper) ScopeUserByIds(uid, gid int) (func() error, error) { - runtime.LockOSThread() // Lock this Goroutine to the current OS thread - - var err error - var prevUid int - var prevGid int - if uid >= 0 { - prevUid, err = unix.SetfsuidRetUid(uid) - if err != nil { - return nil, err - } - if testUid, _ := unix.SetfsuidRetUid(-1); testUid != uid { - return nil, fmt.Errorf("failed to setfsuid to %d", uid) - } - } - if gid >= 0 { - prevGid, err = unix.SetfsgidRetGid(gid) - if err != nil { - return nil, err - } - if testGid, _ := unix.SetfsgidRetGid(-1); testGid != gid { - return nil, fmt.Errorf("failed to setfsgid to %d", gid) - } - } - - return func() error { - if uid >= 0 { - _ = unix.Setfsuid(prevUid) - } - if gid >= 0 { - _ = unix.Setfsgid(prevGid) - } - runtime.UnlockOSThread() - return nil - }, nil -} - -func (u *UnixMapper) mapUser(username string) (int, int, error) { - userDetails, err := user.Lookup(username) - if err != nil { - return 0, 0, err - } - - uid, err := strconv.Atoi(userDetails.Uid) - if err != nil { - return 0, 0, err - } - gid, err := strconv.Atoi(userDetails.Gid) - if err != nil { - return 0, 0, err - } - - return uid, gid, nil -} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/eosfs/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/eosfs/upload.go index 2645f151b5..99d17114c6 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/eosfs/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/eosfs/upload.go @@ -20,85 +20,12 @@ package eosfs import ( "context" - "os" - "path" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/pkg/errors" ) -func (fs *eosfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - p, err := fs.resolve(ctx, req.Ref) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "eos: error resolving reference") - } - - if fs.isShareFolder(ctx, p) { - return &provider.ResourceInfo{}, errtypes.PermissionDenied("eos: cannot upload under the virtual share folder") - } - - if chunking.IsChunked(p) { - var assembledFile string - p, assembledFile, err = fs.chunkHandler.WriteChunk(p, req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } - if p == "" { - return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) - } - fd, err := os.Open(assembledFile) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "eos: error opening assembled file") - } - defer fd.Close() - defer os.RemoveAll(assembledFile) - req.Body = fd - } - - fn := fs.wrap(ctx, p) - - u, err := getUser(ctx) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "eos: no user in ctx") - } - - // We need the auth corresponding to the parent directory - // as the file might not exist at the moment - auth, err := fs.getUserAuth(ctx, u, path.Dir(fn)) - if err != nil { - return &provider.ResourceInfo{}, err - } - - if err := fs.c.Write(ctx, auth, fn, req.Body); err != nil { - return &provider.ResourceInfo{}, err - } - - eosFileInfo, err := fs.c.GetFileInfoByPath(ctx, auth, fn) - if err != nil { - return &provider.ResourceInfo{}, err - } - - ri, err := fs.convertToResourceInfo(ctx, eosFileInfo) - if err != nil { - return &provider.ResourceInfo{}, err - } - - return ri, nil -} - -func (fs *eosfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - p, err := fs.resolve(ctx, ref) - if err != nil { - return nil, err - } - return map[string]string{ - "simple": p, - }, nil -} - func (fs *eosfs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { return errtypes.NotSupported("op not supported") } @@ -106,3 +33,11 @@ func (fs *eosfs) MarkProcessing(ctx context.Context, ref *provider.Reference, pr func (fs *eosfs) CommitUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.UploadSource) error { return errtypes.NotSupported("op not supported") } + +func (fs *eosfs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil +} + +func (fs *eosfs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { + return nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/localfs/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/localfs/upload.go index 3ae7962824..c901ef5003 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/localfs/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/localfs/upload.go @@ -20,137 +20,12 @@ package localfs import ( "context" - "encoding/json" - "io" - "os" - "path/filepath" - userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - "github.com/google/uuid" - "github.com/opencloud-eu/reva/v2/pkg/appctx" - ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" - "github.com/opencloud-eu/reva/v2/pkg/utils" - "github.com/pkg/errors" - tusd "github.com/tus/tusd/v2/pkg/handler" ) -var defaultFilePerm = os.FileMode(0664) - -func (fs *localfs) Upload(ctx context.Context, req storage.UploadRequest, uff storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - upload, err := fs.GetUpload(ctx, req.Ref.GetPath()) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "localfs: error retrieving upload") - } - - uploadInfo := upload.(*fileUpload) - - p := uploadInfo.info.Storage["InternalDestination"] - if chunking.IsChunked(p) { - var assembledFile string - p, assembledFile, err = fs.chunkHandler.WriteChunk(p, req.Body) - if err != nil { - return &provider.ResourceInfo{}, err - } - if p == "" { - if err = uploadInfo.Terminate(ctx); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "localfs: error removing auxiliary files") - } - return &provider.ResourceInfo{}, errtypes.PartialContent(req.Ref.String()) - } - uploadInfo.info.Storage["InternalDestination"] = p - fd, err := os.Open(assembledFile) - if err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "localfs: error opening assembled file") - } - defer fd.Close() - defer os.RemoveAll(assembledFile) - req.Body = fd - } - - if _, err := uploadInfo.WriteChunk(ctx, 0, req.Body); err != nil { - return &provider.ResourceInfo{}, errors.Wrap(err, "localfs: error writing to binary file") - } - - if err := uploadInfo.FinishUpload(ctx); err != nil { - return &provider.ResourceInfo{}, err - } - - if uff != nil { - info := uploadInfo.info - uploadRef := &provider.Reference{ - ResourceId: &provider.ResourceId{ - StorageId: info.MetaData["providerID"], - SpaceId: info.Storage["SpaceRoot"], - OpaqueId: info.Storage["SpaceRoot"], - }, - Path: utils.MakeRelativePath(filepath.Join(info.MetaData["dir"], info.MetaData["filename"])), - } - owner, ok := ctxpkg.ContextGetUser(uploadInfo.ctx) - if !ok { - return &provider.ResourceInfo{}, errtypes.PreconditionFailed("error getting user from uploadinfo context") - } - // spaces support in localfs needs to be revisited: - // * info.Storage["SpaceRoot"] is never set - // * there is no space owner or manager that could be passed to the UploadFinishedFunc - uff(owner.Id, owner.Id, uploadRef) - } - - // return id, etag and mtime - ri, err := fs.GetMD(ctx, req.Ref, []string{}, []string{"id", "etag", "mtime"}) - if err != nil { - return &provider.ResourceInfo{}, err - } - - return ri, nil -} - -// InitiateUpload returns upload ids corresponding to different protocols it supports -// It resolves the resource and then reuses the NewUpload function -// Currently requires the uploadLength to be set -// TODO to implement LengthDeferrerDataStore make size optional -// TODO read optional content for small files in this request -func (fs *localfs) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - - np, err := fs.resolve(ctx, ref) - if err != nil { - return nil, errors.Wrap(err, "localfs: error resolving reference") - } - - info := tusd.FileInfo{ - MetaData: tusd.MetaData{ - "filename": filepath.Base(np), - "dir": filepath.Dir(np), - }, - Size: uploadLength, - } - - if metadata != nil { - info.MetaData["providerID"] = metadata["providerID"] - if metadata["mtime"] != "" { - info.MetaData["mtime"] = metadata["mtime"] - } - if _, ok := metadata["sizedeferred"]; ok { - info.SizeIsDeferred = true - } - } - - upload, err := fs.NewUpload(ctx, info) - if err != nil { - return nil, err - } - - info, _ = upload.GetInfo(ctx) - - return map[string]string{ - "simple": info.ID, - "tus": info.ID, - }, nil -} - func (fs *localfs) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { return errtypes.NotSupported("op not supported") } @@ -159,246 +34,10 @@ func (fs *localfs) CommitUpload(_ context.Context, _ *provider.Reference, _ stri return errtypes.NotSupported("op not supported") } -// UseIn tells the tus upload middleware which extensions it supports. -func (fs *localfs) UseIn(composer *tusd.StoreComposer) { - composer.UseCore(fs) - composer.UseTerminater(fs) - // TODO composer.UseConcater(fs) - // TODO composer.UseLengthDeferrer(fs) +func (fs *localfs) PrepareUpload(_ context.Context, _ *provider.Reference, _ string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + return &storage.PrepareUploadResult{VersionCreated: info.NodeExisted}, nil } -// NewUpload creates a new upload using the size as the file's length. To determine where to write the binary data -// the Fileinfo metadata must contain a dir and a filename. -// returns a unique id which is used to identify the upload. The properties Size and MetaData will be filled. -func (fs *localfs) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { - - log := appctx.GetLogger(ctx) - log.Debug().Interface("info", info).Msg("localfs: NewUpload") - - fn := info.MetaData["filename"] - if fn == "" { - return nil, errors.New("localfs: missing filename in metadata") - } - info.MetaData["filename"] = filepath.Clean(info.MetaData["filename"]) - - dir := info.MetaData["dir"] - if dir == "" { - return nil, errors.New("localfs: missing dir in metadata") - } - info.MetaData["dir"] = filepath.Clean(info.MetaData["dir"]) - - np := fs.wrap(ctx, filepath.Join(info.MetaData["dir"], info.MetaData["filename"])) - - log.Debug().Interface("info", info).Msg("localfs: resolved filename") - - info.ID = uuid.New().String() - - binPath, err := fs.getUploadPath(ctx, info.ID) - if err != nil { - return nil, errors.Wrap(err, "localfs: error resolving upload path") - } - usr := ctxpkg.ContextMustGetUser(ctx) - info.Storage = map[string]string{ - "Type": "LocalStore", - "BinPath": binPath, - "InternalDestination": np, - - "Idp": usr.Id.Idp, - "UserId": usr.Id.OpaqueId, - "UserName": usr.Username, - "UserType": utils.UserTypeToString(usr.Id.Type), - - "LogLevel": log.GetLevel().String(), - } - // Create binary file with no content - file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm) - if err != nil { - return nil, err - } - defer file.Close() - - u := &fileUpload{ - info: info, - binPath: binPath, - infoPath: binPath + ".info", - fs: fs, - ctx: ctx, - } - - // writeInfo creates the file by itself if necessary - err = u.writeInfo() - if err != nil { - return nil, err - } - - return u, nil -} - -func (fs *localfs) getUploadPath(ctx context.Context, uploadID string) (string, error) { - return filepath.Join(fs.conf.Uploads, uploadID), nil -} - -// GetUpload returns the Upload for the given upload id -func (fs *localfs) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { - binPath, err := fs.getUploadPath(ctx, id) - if err != nil { - return nil, err - } - infoPath := binPath + ".info" - info := tusd.FileInfo{} - data, err := os.ReadFile(infoPath) - if err != nil { - if os.IsNotExist(err) { - // Interpret os.ErrNotExist as 404 Not Found - err = tusd.ErrNotFound - } - return nil, err - } - if err := json.Unmarshal(data, &info); err != nil { - return nil, err - } - - stat, err := os.Stat(binPath) - if err != nil { - return nil, err - } - - info.Offset = stat.Size() - - u := &userpb.User{ - Id: &userpb.UserId{ - Idp: info.Storage["Idp"], - OpaqueId: info.Storage["UserId"], - Type: utils.UserTypeMap(info.Storage["UserType"]), - }, - Username: info.Storage["UserName"], - } - - ctx = ctxpkg.ContextSetUser(ctx, u) - - return &fileUpload{ - info: info, - binPath: binPath, - infoPath: infoPath, - fs: fs, - ctx: ctx, - }, nil -} - -type fileUpload struct { - // info stores the current information about the upload - info tusd.FileInfo - // infoPath is the path to the .info file - infoPath string - // binPath is the path to the binary file (which has no extension) - binPath string - // only fs knows how to handle metadata and versions - fs *localfs - // a context with a user - ctx context.Context -} - -// GetInfo returns the FileInfo -func (upload *fileUpload) GetInfo(ctx context.Context) (tusd.FileInfo, error) { - return upload.info, nil -} - -// GetReader returns an io.Reader for the upload -func (upload *fileUpload) GetReader(ctx context.Context) (io.ReadCloser, error) { - return os.Open(upload.binPath) -} - -// WriteChunk writes the stream from the reader to the given offset of the upload -func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { - file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm) - if err != nil { - return 0, err - } - defer file.Close() - - n, err := io.Copy(file, src) - - // If the HTTP PATCH request gets interrupted in the middle (e.g. because - // the user wants to pause the upload), Go's net/http returns an io.ErrUnexpectedEOF. - // However, for the driver it's not important whether the stream has ended - // on purpose or accidentally. - if err != nil { - if err != io.ErrUnexpectedEOF { - return n, err - } - } - - upload.info.Offset += n - err = upload.writeInfo() - - return n, err -} - -// writeInfo updates the entire information. Everything will be overwritten. -func (upload *fileUpload) writeInfo() error { - data, err := json.Marshal(upload.info) - if err != nil { - return err - } - return os.WriteFile(upload.infoPath, data, defaultFilePerm) -} - -// FinishUpload finishes an upload and moves the file to the internal destination -func (upload *fileUpload) FinishUpload(ctx context.Context) error { - - np := upload.info.Storage["InternalDestination"] - - // TODO check etag with If-Match header - // if destination exists - // if _, err := os.Stat(np); err == nil { - // the local storage does not store metadata - // the fileid is based on the path, so no we do not need to copy it to the new file - // the local storage does not track revisions - //} - - // if destination exists - if _, err := os.Stat(np); err == nil { - // create revision - if err := upload.fs.archiveRevision(upload.ctx, np); err != nil { - return err - } - } - - err := os.Rename(upload.binPath, np) - if err != nil { - return err - } - - // only delete the upload if it was successfully written to the fs - if err := os.Remove(upload.infoPath); err != nil { - if !os.IsNotExist(err) { - log := appctx.GetLogger(ctx) - log.Err(err).Interface("info", upload.info).Msg("localfs: could not delete upload info") - } - } - - // TODO: set mtime if specified in metadata - - // metadata propagation is left to the storage implementation - return err -} - -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// - the storage needs to implement AsTerminatableUpload -// - the upload needs to implement Terminate - -// AsTerminatableUpload returns a a TerminatableUpload -func (fs *localfs) AsTerminatableUpload(upload tusd.Upload) tusd.TerminatableUpload { - return upload.(*fileUpload) -} - -// Terminate terminates the upload -func (upload *fileUpload) Terminate(ctx context.Context) error { - if err := os.Remove(upload.infoPath); err != nil { - return err - } - if err := os.Remove(upload.binPath); err != nil { - return err - } +func (fs *localfs) RollbackUpload(_ context.Context, _ *provider.Reference, _ string, _ storage.RollbackInfo) error { return nil } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware/middleware.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware/middleware.go index 05e8bec1b2..611126601f 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware/middleware.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/utils/middleware/middleware.go @@ -25,11 +25,8 @@ import ( userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" - tusd "github.com/tus/tusd/v2/pkg/handler" - "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/storage" - "github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload" "github.com/opencloud-eu/reva/v2/pkg/storagespace" ) @@ -59,58 +56,17 @@ func (fs *FS) WithDisabledSpaces() storage.FS { return &f } -// ListUploadSessions returns the upload sessions matching the given filter -func (f *FS) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { - return f.next.(storage.UploadSessionLister).ListUploadSessions(ctx, filter) +// IsOrphaned reports whether the referenced resource exists but its metadata is unreadable. +func (f *FS) IsOrphaned(ctx context.Context, ref *provider.Reference) bool { + return f.next.(storage.OrphanChecker).IsOrphaned(ctx, ref) } -// ResolveUpload resolves a finished upload to the resource it produced, read as the upload's -// executant. It is best effort: if the wrapped storage does not resolve uploads it returns -// NotSupported rather than panicking, so the tus finalize falls back to no headers. -func (f *FS) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) { - r, ok := f.next.(storage.UploadSessionResolver) - if !ok { - return nil, ctx, errtypes.NotSupported("storage does not resolve upload sessions") - } - return r.ResolveUpload(ctx, info) +// PrepareCreatesNode reports whether the wrapped driver's PrepareUpload creates a +// new file's node. +func (f *FS) PrepareCreatesNode() bool { + nc, ok := f.next.(storage.NodeCreator) + return ok && nc.PrepareCreatesNode() } - -// UseIn tells the tus upload middleware which extensions it supports. -func (f *FS) UseIn(composer *tusd.StoreComposer) { - f.next.(storage.ComposableFS).UseIn(composer) -} - -// NewUpload returns a new tus Upload instance -func (f *FS) NewUpload(ctx context.Context, info tusd.FileInfo) (upload tusd.Upload, err error) { - return f.next.(tusd.DataStore).NewUpload(ctx, info) -} - -// NewUpload returns a new tus Upload instance -func (f *FS) GetUpload(ctx context.Context, id string) (upload tusd.Upload, err error) { - return f.next.(tusd.DataStore).GetUpload(ctx, id) -} - -// AsTerminatableUpload returns a TerminatableUpload -// To implement the termination extension as specified in https://tus.io/protocols/resumable-upload.html#termination -// the storage needs to implement AsTerminatableUpload -func (f *FS) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { - return up.(*upload.OcisSession) -} - -// AsLengthDeclarableUpload returns a LengthDeclarableUpload -// To implement the creation-defer-length extension as specified in https://tus.io/protocols/resumable-upload.html#creation -// the storage needs to implement AsLengthDeclarableUpload -func (f *FS) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { - return up.(*upload.OcisSession) -} - -// AsConcatableUpload returns a ConcatableUpload -// To implement the concatenation extension as specified in https://tus.io/protocols/resumable-upload.html#concatenation -// the storage needs to implement AsConcatableUpload -func (f *FS) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { - return up.(*upload.OcisSession) -} - func (f *FS) GetHome(ctx context.Context) (string, error) { var ( err error @@ -327,60 +283,6 @@ func (f *FS) ListFolder(ctx context.Context, ref *provider.Reference, mdKeys, fi return res0, res1 } -func (f *FS) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { - var ( - err error - unhook UnHook - unhooks []UnHook - ) - for _, hook := range f.hooks { - ctx, unhook, err = hook("InitiateUpload", ctx, ref.GetResourceId().GetSpaceId()) - if err != nil { - return nil, err - } - if unhook != nil { - unhooks = append(unhooks, unhook) - } - } - - res0, res1 := f.next.InitiateUpload(ctx, ref, uploadLength, metadata) - - for _, unhook := range unhooks { - if err := unhook(); err != nil { - return nil, err - } - } - - return res0, res1 -} - -func (f *FS) Upload(ctx context.Context, req storage.UploadRequest, uploadFunc storage.UploadFinishedFunc) (*provider.ResourceInfo, error) { - var ( - err error - unhook UnHook - unhooks []UnHook - ) - for _, hook := range f.hooks { - ctx, unhook, err = hook("Upload", ctx, req.Ref.GetResourceId().GetSpaceId()) - if err != nil { - return &provider.ResourceInfo{}, err - } - if unhook != nil { - unhooks = append(unhooks, unhook) - } - } - - res0, res1 := f.next.Upload(ctx, req, uploadFunc) - - for _, unhook := range unhooks { - if err := unhook(); err != nil { - return &provider.ResourceInfo{}, err - } - } - - return res0, res1 -} - func (f *FS) MarkProcessing(ctx context.Context, ref *provider.Reference, processing bool, sessionID string) error { var ( err error @@ -435,6 +337,37 @@ func (f *FS) CommitUpload(ctx context.Context, ref *provider.Reference, sessionI return err } +func (f *FS) PrepareUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.UploadInfo) (*storage.PrepareUploadResult, error) { + var ( + err error + unhook UnHook + unhooks []UnHook + ) + for _, hook := range f.hooks { + ctx, unhook, err = hook("PrepareUpload", ctx, ref.GetResourceId().GetSpaceId()) + if err != nil { + return nil, err + } + if unhook != nil { + unhooks = append(unhooks, unhook) + } + } + + res, err := f.next.PrepareUpload(ctx, ref, sessionID, info) + + for _, unhook := range unhooks { + if uerr := unhook(); uerr != nil { + return nil, uerr + } + } + + return res, err +} + +func (f *FS) RollbackUpload(ctx context.Context, ref *provider.Reference, sessionID string, info storage.RollbackInfo) error { + return f.next.RollbackUpload(ctx, ref, sessionID, info) +} + func (f *FS) Download(ctx context.Context, ref *provider.Reference, openReaderFunc func(md *provider.ResourceInfo) bool) (*provider.ResourceInfo, io.ReadCloser, error) { var ( err error diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/tags/tags.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/tags/tags.go index 5113b3f8cf..7b0da1d3a5 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/tags/tags.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/tags/tags.go @@ -69,6 +69,7 @@ func (t *Tags) Remove(s ...string) bool { } delete(t.exists, tag) + t.numtags-- removed = true } } diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/asyncconf.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/asyncconf.go new file mode 100644 index 0000000000..4c4bf802be --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/asyncconf.go @@ -0,0 +1,59 @@ +package upload + +import ( + "time" + + "github.com/go-viper/mapstructure/v2" +) + +// AsyncConf is how a service asks for async uploads: whether they are enabled, +// and the consumer subscription to use if they are. +type AsyncConf struct { + Enabled bool + ConsumerGroup string + NumConsumers int + // MountID is the storage id this provider answers for, used to drop + // postprocessing events belonging to other storages. + MountID string + CommitMaxRetries int + CommitRetryBackoff time.Duration +} + +// AsyncConfFromDriverConf reads the postprocessing settings off the driver's own +// config keys, so the coordinator and the driver cannot disagree about them. +func AsyncConfFromDriverConf(driverConf map[string]interface{}) AsyncConf { + if driverConf == nil { + return AsyncConf{} + } + var ac struct { + AsyncFileUploads bool `mapstructure:"asyncfileuploads"` + MountID string `mapstructure:"mount_id"` + Events struct { + NumConsumers int `mapstructure:"numconsumers"` + ConsumerGroup string `mapstructure:"consumer_group"` + CommitMaxRetries int `mapstructure:"commit_max_retries"` + CommitRetryBackoff time.Duration `mapstructure:"commit_retry_backoff"` + } `mapstructure:"events"` + } + _ = mapstructure.Decode(driverConf, &ac) + group := ac.Events.ConsumerGroup + if group == "" { + group = "dcfs" + } + maxRetries := ac.Events.CommitMaxRetries + if maxRetries <= 0 { + maxRetries = 3 + } + retryBackoff := ac.Events.CommitRetryBackoff + if retryBackoff <= 0 { + retryBackoff = 5 * time.Second + } + return AsyncConf{ + Enabled: ac.AsyncFileUploads, + ConsumerGroup: group, + NumConsumers: ac.Events.NumConsumers, + MountID: ac.MountID, + CommitMaxRetries: maxRetries, + CommitRetryBackoff: retryBackoff, + } +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/coordinator.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/coordinator.go new file mode 100644 index 0000000000..dbc16d3468 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/coordinator.go @@ -0,0 +1,1014 @@ +package upload + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/google/uuid" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + "github.com/opencloud-eu/reva/v2/pkg/events" + "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking" + "github.com/opencloud-eu/reva/v2/pkg/utils" +) + +// Request carries the metadata of a non-resumable (PUT) upload. +type Request struct { + Ref *provider.Reference + Body io.ReadCloser + Length int64 +} + +// FinishedFunc is called once an upload has finished, so that the caller can +// publish a FileUploaded event through its own publisher. +type FinishedFunc func(spaceOwner, executant *user.UserId, ref *provider.Reference) + +// Coordinator owns the upload lifecycle: initiation, data transfer and listing. +type Coordinator interface { + // InitiateUpload returns the protocols and ids that bytes can be appended to. + InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) + // GetUpload returns the session with the given id as a tusd upload. + GetUpload(ctx context.Context, id string) (tusd.Upload, error) + // UseIn registers the coordinator as the tusd data store. + UseIn(composer *tusd.StoreComposer) + // ListUploadSessions returns the upload sessions matching the given filter. + ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) + // Upload writes the whole body of a non-resumable (PUT) upload and finishes it. + Upload(ctx context.Context, req Request, uff FinishedFunc) (*provider.ResourceInfo, error) + // StartPostprocessing subscribes to postprocessing results and enables async + // uploads. Call once, before serving requests. + StartPostprocessing(stream events.Consumer, ac AsyncConf) error + // ResolveUpload stats the resource a finished upload produced, as the upload's executant. + storage.UploadSessionResolver +} + +const maxCommitRetryBackoff = 2 * time.Minute + +// coordinator is the concrete implementation of Coordinator. +type coordinator struct { + fs storage.FS + store SessionStore + chunkHandler *chunking.ChunkHandler + pub events.Publisher + // async defers the commit to postprocessing. Only StartPostprocessing sets it. + async bool + // mountID is the storage this coordinator serves, used to drop postprocessing + // events belonging to another one. + mountID string + commitMaxRetries int + commitRetryBackoff time.Duration +} + +// NewCoordinator constructs a coordinator backed by the given driver and store. +func NewCoordinator(fs storage.FS, store SessionStore, chunkFolder string, pub events.Publisher) *coordinator { + c := &coordinator{fs: fs, store: store, pub: pub} + if chunkFolder != "" { + c.chunkHandler = chunking.NewChunkHandler(chunkFolder) + } + return c +} + +// ResolveUpload stats the resource produced by a finished upload, reading it as the upload's +// executant. The session is reconstructed from the given tus FileInfo rather than the session +// store, so it still resolves after a synchronous upload has cleaned up its session. It returns +// the resource info and the executant context, so the caller can derive identity dependent +// values such as the WebDAV permissions. https://github.com/opencloud-eu/opencloud/issues/2409 +func (c *coordinator) ResolveUpload(ctx context.Context, info tusd.FileInfo) (*provider.ResourceInfo, context.Context, error) { + session := c.store.FromInfo(info) + ctx = session.Context(ctx) + ref := session.Reference() + ri, err := c.fs.GetMD(ctx, &ref, nil, nil) + return ri, ctx, err +} + +// NewCoordinatorFromConfig sets up a coordinator and its file store. Pass +// withChunking=false when only initiating uploads: chunk assembly happens on the +// data path, so the storageprovider doesn't need it. +func NewCoordinatorFromConfig(uploadDir string, driverConf map[string]interface{}, fs storage.FS, pub events.Publisher, log *zerolog.Logger, withChunking bool) (Coordinator, error) { + store := NewFileStoreFromConfig(uploadDir, driverConf, log) + if store == nil { + return nil, fmt.Errorf("cannot determine the upload directory, set upload_directory") + } + if err := store.Setup(); err != nil { + return nil, fmt.Errorf("upload directory setup failed: %w", err) + } + chunkFolder := "" + if withChunking { + chunkFolder = store.UploadDir() + } + return NewCoordinator(fs, store, chunkFolder, pub), nil +} + +// InitiateUpload resolves the target, then creates and persists the session that +// bytes are appended to. +func (c *coordinator) InitiateUpload(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (map[string]string, error) { + t, err := c.resolveTarget(ctx, ref, uploadLength, metadata) + if err != nil { + return nil, err + } + + session := c.store.New(ctx) + if err := c.populateSession(ctx, session, t, uploadLength, metadata); err != nil { + return nil, err + } + + if err := session.TouchBin(); err != nil { + return nil, fmt.Errorf("coordinator: could not create bin file: %w", err) + } + if err := session.Persist(ctx); err != nil { + session.Cleanup(ctx, true, true) + return nil, fmt.Errorf("coordinator: could not persist session: %w", err) + } + + metrics.UploadSessionsInitiated.Inc() + + // A zero-length upload has no bytes to append, so finish it here. + if uploadLength == 0 { + if _, err := c.finishUpload(ctx, session); err != nil { + return nil, err + } + } + + return map[string]string{ + "simple": session.ID(), + "tus": session.ID(), + }, nil +} + +// populateSession writes the resolved target and the request's metadata into a +// new session. +func (c *coordinator) populateSession(ctx context.Context, session Session, t *uploadTarget, uploadLength int64, metadata map[string]string) error { + session.SetMetadata("filename", t.name) + session.SetStorageValue("NodeName", t.name) + session.SetMetadata("dir", t.dir) + session.SetStorageValue("Dir", t.dir) + session.SetStorageValue("SpaceRoot", t.spaceID) + session.SetStorageValue("NodeParentId", t.parentID) + + if t.exists { + session.SetStorageValue("NodeId", t.nodeID) + session.SetStorageValue("NodeExists", "true") + } else { + // A new file has no id yet, and an empty one would resolve to the space root + // (lookup.go:191), so mint a placeholder TouchFile replaces. + session.SetStorageValue("NodeId", uuid.New().String()) + } + + if t.spaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", t.spaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", t.spaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(t.spaceOwner.GetType())) + } + if t.chunkName != "" { + session.SetStorageValue("Chunk", t.chunkName) + } + + // The ctx is gone by the time the bytes arrive, so record the actor now. + session.SetExecutant(ctxpkg.ContextMustGetUser(ctx)) + lockID, _ := ctxpkg.ContextGetLockID(ctx) + session.SetMetadata("lockid", lockID) + initiatorID, _ := ctxpkg.ContextGetInitiator(ctx) + session.SetMetadata("initiatorid", initiatorID) + + session.SetSize(uploadLength) + + return c.applyRequestMetadata(session, metadata) +} + +// applyRequestMetadata stores the client's upload metadata, rejecting a malformed +// checksum. Conditional headers are only recorded, and evaluated at finish. +func (c *coordinator) applyRequestMetadata(session Session, metadata map[string]string) error { + mtime, ok := metadata["mtime"] + if !ok || mtime == "null" { + mtime = utils.TimeToOCMtime(time.Now()) + } + session.SetMetadata("mtime", mtime) + + session.SetMetadata("providerID", metadata["providerID"]) + if v, ok := metadata["expires"]; ok && v != "null" { + session.SetMetadata("expires", v) + } + if _, ok := metadata["sizedeferred"]; ok { + session.SetSizeIsDeferred(true) + } + for _, key := range []string{"if-match", "if-none-match", "if-unmodified-since"} { + if v := metadata[key]; v != "" { + session.SetMetadata(key, v) + } + } + + checksum, ok := metadata["checksum"] + if !ok { + return nil + } + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + switch parts[0] { + case "sha1", "md5", "adler32": + session.SetMetadata("checksum", checksum) + default: + return errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + return nil +} + +// Upload writes the whole body of a non-resumable (PUT) upload and finishes it. +// req.Ref.Path carries the session id minted by InitiateUpload. +func (c *coordinator) Upload(ctx context.Context, req Request, uff FinishedFunc) (*provider.ResourceInfo, error) { + // The request path arrives rooted, while session ids are stored unrooted. + session, err := c.store.Get(ctx, strings.TrimPrefix(req.Ref.GetPath(), "/")) + if err != nil { + return nil, err + } + // Its bytes already arrived, and an async finish keeps the session for + // postprocessing: a repeated PUT would append to them and finish it again. + if session.Offset() > 0 { + return nil, errtypes.Aborted("coordinator: upload " + session.ID() + " was already received") + } + + // Behind the data gateway the request carries a transfer token, not the user. + ctx = session.Context(ctx) + + if chunk := session.Chunk(); chunk != "" { // legacy chunking v1 + if c.chunkHandler == nil { + return nil, errtypes.NotSupported("coordinator: chunked uploads require a chunk folder") + } + assembled, assembledSize, done, aErr := c.chunkHandler.Assemble(chunk, req.Body) + if aErr != nil { + return nil, aErr + } + if !done { + // The bytes accumulate in the chunk folder, so this session holds nothing. + session.Cleanup(ctx, true, true) + return nil, errtypes.PartialContent(req.Ref.String()) + } + defer assembled.Close() + // The declared length covers only the final chunk. + req.Body, req.Length = assembled, assembledSize + session.SetSize(assembledSize) + } + + size, err := session.WriteChunk(ctx, 0, req.Body) + if err != nil { + return nil, err + } + if size != req.Length { + return nil, errtypes.PartialContent("coordinator: unexpected end of stream") + } + + ri, err := c.finishUpload(ctx, session) + if err != nil { + return nil, err + } + + if uff != nil { + executant := session.Executant() + uff(session.SpaceOwner(), &executant, c.uploadRef(session)) + } + return ri, nil +} + +// finishUpload creates the node, validates the staged bytes and commits them, or +// hands them to postprocessing. +func (c *coordinator) finishUpload(ctx context.Context, session Session) (*provider.ResourceInfo, error) { + // a client that goes away must not leave the upload half finished + ctx = context.WithoutCancel(ctx) + // Only the staged bytes are read, so a mismatch has no node to undo. + if err := verifyAndStoreChecksums(ctx, session); err != nil { + session.Cleanup(ctx, true, true) + return nil, err + } + // Read before the node exists, so bad request metadata has no node to undo either. + info, err := uploadInfo(session) + if err != nil { + session.Cleanup(ctx, true, true) + return nil, err + } + // A driver whose PrepareUpload creates a new file's node is told where it goes, + // under the id minted at initiate. Any other is handed one TouchFile created. + if !session.NodeExists() && c.prepareCreatesNode() { + info.ParentID, info.Name = session.NodeParentID(), session.Filename() + } else if err := c.touchNode(ctx, session); err != nil { + return nil, err + } + + metrics.UploadSessionsBytesReceived.Inc() + + if err := c.prepare(ctx, session, info); err != nil { + return nil, err + } + + // Without a publisher there is nothing to hand the bytes to, so commit inline. + if c.async && c.pub != nil && session.Size() > 0 { + if err := c.publishBytesReceived(ctx, session); err != nil { + c.rollbackPrepared(ctx, session, session.SizeDiff()) + return nil, err + } + // The node stays flagged and the staged bytes are kept: postprocessing needs + // both to finish later. + return c.uploadedResourceInfo(ctx, session), nil + } + + return c.finishSync(ctx, session) +} + +// publishBytesReceived hands the staged upload to postprocessing, which downloads +// the bytes from a signed URL to scan them before they are committed. +func (c *coordinator) publishBytesReceived(ctx context.Context, session Session) error { + url, err := session.URL(ctx) + if err != nil { + return err + } + + return events.Publish(ctx, c.pub, events.BytesReceived{ + UploadID: session.ID(), + URL: url, + SpaceOwner: session.SpaceOwner(), + ExecutingUser: session.ExecutantUser(), + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Filename: session.Filename(), + Filesize: uint64(session.Size()), + ImpersonatingUser: impersonatingUser(ctx), + }) +} + +// prepareCreatesNode reports whether the driver's PrepareUpload creates a new +// file's node, sparing the extra metadata write a TouchFile costs. +func (c *coordinator) prepareCreatesNode() bool { + nc, ok := c.fs.(storage.NodeCreator) + return ok && nc.PrepareCreatesNode() +} + +// touchNode creates the node a new file's upload writes to. +func (c *coordinator) touchNode(ctx context.Context, session Session) error { + if session.NodeExists() { + return nil + } + + // The node has no id yet, so address it as a named child of its parent. + pathRef := &provider.Reference{ + ResourceId: &provider.ResourceId{ + SpaceId: session.SpaceID(), + OpaqueId: session.NodeParentID(), + }, + Path: session.Filename(), + } + // PrepareUpload marks and propagates the node, so TouchFile need not. + err := c.fs.TouchFile(storage.ContextSkipTouchPropagation(ctx), pathRef, false, session.Metadata()["mtime"]) + if err == nil { + // TouchFile does not report the node it created, read it back. + var ri *provider.ResourceInfo + ri, err = c.fs.GetMD(ctx, pathRef, nil, nil) + if err == nil { + // Replaces the placeholder minted at initiate. + session.SetStorageValue("NodeId", ri.GetId().GetOpaqueId()) + session.SetStorageValue("SpaceRoot", ri.GetId().GetSpaceId()) + if ri.GetOwner() != nil { + if owner := c.spaceOwnerOrManager(ctx, ri.GetOwner(), ri.GetId().GetSpaceId()); owner != nil { + session.SetStorageValue("SpaceOwnerOrManager", owner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", owner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(owner.GetType())) + } + } + return nil + } + } + session.Cleanup(ctx, true, true) + if _, ok := err.(errtypes.IsNotFound); ok { + // The parent went away, or the share was revoked, while bytes were in flight. + return errtypes.PreconditionFailed(err.Error()) + } + return err +} + +// prepare has the driver write the node metadata, mark it as processing so clients +// do not read it before its bytes are committed, and snapshot the previous version, +// ahead of the commit that writes the bytes. +func (c *coordinator) prepare(ctx context.Context, session Session, info storage.UploadInfo) error { + ref := session.Reference() + + // A failed PrepareUpload has already undone its own writes, a new file's node + // included, and marks the node only on success: there is nothing to unmark. + prepared, err := c.fs.PrepareUpload(ctx, &ref, session.ID(), info) + if err != nil { + session.Cleanup(ctx, true, true) + if _, ok := err.(errtypes.IsNotFound); ok && info.ParentID != "" { + // The parent went away, or the share was revoked, while bytes were in flight. + return errtypes.PreconditionFailed(err.Error()) + } + return err + } + metrics.UploadProcessing.Inc() + + // Persisted, not just held: the commit may run in another process, which can + // only learn these, and the real node id from touchNode, by reading them back. + // Saved once, here: a failure now finds the node marked, so the rollback can + // purge it. + if prepared.SpaceOwner != nil { + session.SetStorageValue("SpaceOwnerOrManager", prepared.SpaceOwner.GetOpaqueId()) + session.SetStorageValue("SpaceOwnerIdp", prepared.SpaceOwner.GetIdp()) + session.SetStorageValue("SpaceOwnerType", utils.UserTypeToString(prepared.SpaceOwner.GetType())) + } + session.SetSizeDiff(prepared.SizeDiff) + session.SetVersionCreated(prepared.VersionCreated) + session.SetStatus(SessionStatusProcessing, "") + if err := session.Persist(ctx); err != nil { + c.rollbackPrepared(ctx, session, prepared.SizeDiff) + return err + } + return nil +} + +// rollbackInfo describes the upload to undo. The ids come from the session, which +// is the only place they survive a node whose own metadata has become unreadable. +func rollbackInfo(session Session, sizeDiff int64) storage.RollbackInfo { + return storage.RollbackInfo{ + NodeExisted: session.NodeExists(), + SizeDiff: sizeDiff, + NodeID: session.NodeID(), + ParentID: session.NodeParentID(), + Filename: session.Filename(), + Size: session.Size(), + } +} + +// rollbackPrepared undoes a finish that failed after PrepareUpload succeeded. +func (c *coordinator) rollbackPrepared(ctx context.Context, session Session, sizeDiff int64) { + ref := session.Reference() + // Before unmarking: RollbackUpload keys off the processing id to confirm the + // node is still this upload's. + if err := c.fs.RollbackUpload(ctx, &ref, session.ID(), rollbackInfo(session, sizeDiff)); err != nil { + // The session is the last record of what to undo, so keep it for a retry + // rather than leaving the quota consumed with nothing to reclaim it from. + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("could not roll back upload, keeping session") + return + } + c.unmarkProcessing(ctx, session, &ref) + metrics.UploadProcessing.Dec() + session.Cleanup(ctx, true, true) +} + +// unmarkProcessing clears the processing flag, tolerating a node the rollback +// already purged. +func (c *coordinator) unmarkProcessing(ctx context.Context, session Session, ref *provider.Reference) { + err := c.fs.MarkProcessing(ctx, ref, false, session.ID()) + log := appctx.GetLogger(ctx).With().Str("uploadid", session.ID()).Logger() + switch err.(type) { + case nil: + case errtypes.IsNotFound: + // RollbackUpload purged the node this upload created. + log.Debug().Err(err).Msg("could not unmark processing") + default: + log.Error().Err(err).Msg("could not unmark processing") + } +} + +// uploadInfo collects what the driver needs to write the node metadata. The +// preconditions are re-checked there, because the resource may have changed while +// the bytes were being uploaded. +func uploadInfo(session Session) (storage.UploadInfo, error) { + md := session.Metadata() + info := storage.UploadInfo{ + NodeExisted: session.NodeExists(), + Size: session.Size(), + Checksums: session.Checksums(), + IfMatch: md["if-match"], + IfNoneMatch: md["if-none-match"], + } + if v := md["mtime"]; v != "" { + mtime, err := utils.MTimeToTime(v) + if err != nil { + return info, errtypes.BadRequest("coordinator: invalid mtime: " + v) + } + info.MTime = mtime + } + if v := md["if-unmodified-since"]; v != "" { + ius, err := time.Parse(time.RFC3339Nano, v) + if err != nil { + return info, errtypes.BadRequest("coordinator: invalid if-unmodified-since: " + v) + } + info.IfUnmodifiedSince = ius + } + return info, nil +} + +// verifyAndStoreChecksums hashes the staged bytes, checks them against the +// checksum the client announced, and stores them for the commit. +func verifyAndStoreChecksums(ctx context.Context, session Session) error { + sha1h, md5h, adler32h, err := calculateChecksums(ctx, session.BinPath()) + if err != nil { + return err + } + + // The announced checksum is not in Metadata(), so read the raw info. + info, err := session.GetInfo(ctx) + if err != nil { + return err + } + if checksum := info.MetaData["checksum"]; checksum != "" { + parts := strings.SplitN(checksum, " ", 2) + if len(parts) != 2 { + return errtypes.BadRequest("invalid checksum format. must be '[algorithm] [checksum]'") + } + var checkErr error + switch parts[0] { + case "sha1": + checkErr = checkHash(parts[1], sha1h) + case "md5": + checkErr = checkHash(parts[1], md5h) + case "adler32": + checkErr = checkHash(parts[1], adler32h) + default: + checkErr = errtypes.BadRequest("unsupported checksum algorithm: " + parts[0]) + } + if checkErr != nil { + return checkErr + } + } + + session.SetChecksums(sha1h.Sum(nil), md5h.Sum(nil), adler32h.Sum(nil)) + return nil +} + +// finishSync commits the staged bytes for an inline upload, discarding the +// session if the commit fails: the client is still waiting and will retry, so +// keeping the bytes would only leak them. +func (c *coordinator) finishSync(ctx context.Context, session Session) (*provider.ResourceInfo, error) { + ri, err := c.commit(ctx, session) + if err != nil { + // Keep the staged bytes: they are the only copy of the upload, and the + // failure may be transient. + c.rollbackNode(ctx, session, "commit failed: "+err.Error()) + metrics.UploadProcessing.Dec() + return nil, err + } + return ri, nil +} + +// finishAsync commits the staged bytes for an upload postprocessing has cleared, +// retrying transient failures with capped exponential backoff. On exhaustion +// the caller is responsible for rolling back the node and publishing the failure. +func (c *coordinator) finishAsync(ctx context.Context, session Session) error { + maxAttempts := c.commitMaxRetries + 1 + backoff := c.commitRetryBackoff + var err error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if _, err = c.commit(ctx, session); err == nil { + return nil + } + if attempt == maxAttempts { + break + } + backoff = min(backoff, maxCommitRetryBackoff) + appctx.GetLogger(ctx).Warn().Err(err). + Int("attempt", attempt).Int("maxAttempts", maxAttempts). + Dur("backoff", backoff). + Str("uploadid", session.ID()). + Msg("blob commit failed, retrying after backoff") + t := time.NewTimer(backoff) + select { + case <-ctx.Done(): + t.Stop() + return ctx.Err() + case <-t.C: + } + backoff *= 2 + } + return err +} + +// commit writes the staged bytes through the driver seam and, on success, unmarks +// the node and retires the session. It leaves a failure untouched for the caller +// to handle, the two finish paths wanting opposite things. +func (c *coordinator) commit(ctx context.Context, session Session) (*provider.ResourceInfo, error) { + ref := session.Reference() + + f, err := os.Open(session.BinPath()) + if err != nil { + return nil, err + } + // The verdict rides along with the bytes, so a committed blob always carries + // the one it was cleared under. Empty on the inline path, which never scans. + scanResult, scanDate := session.ScanData() + + // CommitUpload does not own the body; we opened it, so we close it. The path + // lets a driver that can move the staged file into place skip the copy. + err = c.fs.CommitUpload(ctx, &ref, session.ID(), storage.UploadSource{ + Body: f, + Path: session.BinPath(), + Length: session.Size(), + ScanResult: scanResult, + ScanDate: scanDate, + }) + f.Close() + if err != nil { + return nil, err + } + + // Not unmarkProcessing: the commit succeeded, so a missing node is unexpected. + if err := c.fs.MarkProcessing(ctx, &ref, false, session.ID()); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("could not unmark processing") + } + metrics.UploadProcessing.Dec() + session.Cleanup(ctx, true, true) + metrics.UploadSessionsFinalized.Inc() + + ri := c.uploadedResourceInfo(ctx, session) + c.publishUploadReady(ctx, session, ri) + return ri, nil +} + +// uploadedResourceInfo describes the uploaded resource for the caller, which turns +// it into PUT response headers. +// +// TODO: the fallback carries no etag, which a write-only share used to get. +// Returning it through the seam is the proper fix. +func (c *coordinator) uploadedResourceInfo(ctx context.Context, session Session) *provider.ResourceInfo { + ref := session.Reference() + // The driver owns the resulting etag and mtime, so read them back. + ri, err := c.fs.GetMD(ctx, &ref, nil, nil) + if err == nil { + // Drivers do not know their own mount id and the dataprovider does not stamp + // it, so an unstamped id would not resolve for the client. + if ri.GetId().GetStorageId() == "" { + ri.Id.StorageId = session.ProviderID() + } + return ri + } + + // GetMD is permission-gated, so a write-only share cannot stat what it just + // uploaded. That must not fail the upload. + appctx.GetLogger(ctx).Debug().Err(err).Str("uploadid", session.ID()).Msg("could not stat uploaded resource") + fallback := &provider.ResourceInfo{Id: ref.GetResourceId(), Size: uint64(session.Size())} + if mtime, mErr := utils.MTimeToTime(session.Metadata()["mtime"]); mErr == nil { + fallback.Mtime = utils.TimeToTS(mtime) + } + return fallback +} + +// publishUploadReady announces that the file is available. Consumers such as the +// search indexer only listen for UploadReady, so without it they would never learn +// about an upload the coordinator committed inline. +func (c *coordinator) publishUploadReady(ctx context.Context, session Session, ri *provider.ResourceInfo) { + if c.pub == nil { + return + } + if err := events.Publish(ctx, c.pub, events.UploadReady{ + UploadID: session.ID(), + Filename: session.Filename(), + SpaceOwner: session.SpaceOwner(), + ExecutingUser: session.ExecutantUser(), + FileRef: c.uploadRef(session), + ResourceID: ri.GetId(), + ParentID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeParentID(), + }, + Timestamp: utils.TSNow(), + IsVersion: session.VersionCreated(), + ImpersonatingUser: impersonatingUser(ctx), + }); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("failed to publish UploadReady event") + } +} + +// uploadTarget is what an upload needs to know about its destination +type uploadTarget struct { + // exists distinguishes an overwrite from a new file. + exists bool + // chunkName is set for legacy chunking-v1 uploads only. + chunkName string + + nodeID string + spaceID string + parentID string + dir string + name string + spaceOwner *user.UserId +} + +// resolveTarget locates the file an upload writes to, and rejects an upload that +// may not proceed. +func (c *coordinator) resolveTarget(ctx context.Context, ref *provider.Reference, uploadLength int64, metadata map[string]string) (*uploadTarget, error) { + t := &uploadTarget{} + if chunking.IsChunked(ref.GetPath()) { // check legacy chunking v1 + var err error + ref, t.chunkName, err = rewriteChunkedRef(ref) + if err != nil { + return nil, err + } + } + + // TODO: GetMD reports a file hidden by a deny-grant as NotFound, so the upload + // fails late with 409 instead of 403. Needs a permission-free resolve. + existing, err := c.fs.GetMD(ctx, ref, []string{}, []string{}) + switch err.(type) { + case nil: + t.exists = true + case errtypes.IsNotFound: + t.exists = false + default: + return nil, err + } + + if err := c.checkQuota(ctx, ref, uploadLength, existing); err != nil { + return nil, err + } + + if t.exists { + if err := c.describeExisting(ctx, ref, existing, metadata, t); err != nil { + return nil, err + } + } else if err := c.describeNew(ctx, ref, t); err != nil { + return nil, err + } + + if t.name == "" { + return nil, errtypes.BadRequest("coordinator: missing filename in ref") + } + if t.dir == "" { + return nil, errtypes.BadRequest("coordinator: could not determine upload directory") + } + return t, nil +} + +// checkQuota rejects an upload that would not fit in the space, counting only +// the bytes it adds. +func (c *coordinator) checkQuota(ctx context.Context, ref *provider.Reference, uploadLength int64, existing *provider.ResourceInfo) error { + if uploadLength < 0 { + return nil + } + spaceRef := &provider.Reference{ResourceId: &provider.ResourceId{ + StorageId: ref.GetResourceId().GetStorageId(), + SpaceId: ref.GetResourceId().GetSpaceId(), + }} + _, _, remaining, err := c.fs.GetQuota(ctx, spaceRef) + if err != nil { + return nil + } + + netRequired := uint64(uploadLength) + if existingSize := existing.GetSize(); existingSize < netRequired { + netRequired -= existingSize + } else { + netRequired = 0 + } + if remaining < netRequired { + return errtypes.InsufficientStorage("quota exceeded") + } + return nil +} + +// describeExisting fills in the target from the file being overwritten. +func (c *coordinator) describeExisting(ctx context.Context, ref *provider.Reference, existing *provider.ResourceInfo, metadata map[string]string, t *uploadTarget) error { + if !existing.GetPermissionSet().GetInitiateFileUpload() { + return errtypes.PermissionDenied(ref.GetPath()) + } + if existing.GetType() == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + return errtypes.PreconditionFailed("resource is not a file") + } + + t.nodeID = existing.GetId().GetOpaqueId() + t.spaceID = existing.GetId().GetSpaceId() + t.parentID = existing.GetParentId().GetOpaqueId() + t.name = existing.GetName() + t.spaceOwner = c.spaceOwnerOrManager(ctx, existing.GetOwner(), t.spaceID) + + // GetMD returns only the basename for id-based refs, so ask for the full path. + relPath := existing.GetPath() + if utils.IsRelativeReference(ref) { + if full, err := c.fs.GetPathByID(ctx, existing.GetId()); err == nil { + relPath = full + } + } + t.dir = filepath.Dir(relPath) + + // Lock before precondition, the order main checks them in (upload.go:293-302). + if err := c.checkLock(ctx, ref); err != nil { + return err + } + if metadata["if-none-match"] == "*" { + return errtypes.Aborted(fmt.Sprintf("parent %s already has a child %s, id %s", t.parentID, t.name, t.nodeID)) + } + return nil +} + +// spaceOwnerOrManager resolves the space owner, falling back to a manager. +func (c *coordinator) spaceOwnerOrManager(ctx context.Context, owner *user.UserId, spaceID string) *user.UserId { + // A space with no personal owner stores a SPACE_OWNER placeholder (spaces.go:145). + if owner != nil && owner.GetType() != user.UserType_USER_TYPE_SPACE_OWNER { + return owner + } + + grants, err := c.fs.ListGrants(ctx, &provider.Reference{ResourceId: &provider.ResourceId{ + SpaceId: spaceID, + OpaqueId: spaceID, + }}) + if err != nil { + return nil + } + + // Grants come back in map order, so several managers resolve to an arbitrary one. + for _, g := range grants { + // Group grants carry no user id. + uid := g.GetGrantee().GetUserId() + if uid == nil { + continue + } + p := g.GetPermissions() + if p.GetStat() && p.GetListContainer() && p.GetInitiateFileDownload() { + return uid + } + } + return nil +} + +// checkLock rejects an upload that conflicts with a lock held on the file. +func (c *coordinator) checkLock(ctx context.Context, ref *provider.Reference) error { + // An unlocked file and a driver without locks both error, so only a lock we + // actually read counts as one. + diskLock, err := c.fs.GetLock(ctx, ref) + if err != nil { + diskLock = nil + } + contextLockID, _ := ctxpkg.ContextGetLockID(ctx) + + switch { + case diskLock == nil && contextLockID != "": + return errtypes.Aborted("not locked") + case diskLock == nil: + return nil + case contextLockID == "": + return errtypes.Locked(diskLock.LockId) + case contextLockID != diskLock.LockId: + return errtypes.Aborted("mismatching lock") + } + return nil +} + +// describeNew fills in the target for a file that does not exist yet, from its +// parent directory. +func (c *coordinator) describeNew(ctx context.Context, ref *provider.Reference, t *uploadTarget) error { + t.spaceID = ref.GetResourceId().GetSpaceId() + t.dir = filepath.Dir(ref.GetPath()) + t.name = filepath.Base(ref.GetPath()) + + parentRef := &provider.Reference{ResourceId: ref.GetResourceId(), Path: t.dir} + parentMD, err := c.fs.GetMD(ctx, parentRef, []string{}, []string{}) + switch err.(type) { + case nil: + case errtypes.IsNotFound: + return c.missingParentError(ctx, ref, t.dir, err) + default: + return err + } + + if !parentMD.GetPermissionSet().GetInitiateFileUpload() { + return errtypes.PermissionDenied(ref.GetPath()) + } + t.parentID = parentMD.GetId().GetOpaqueId() + t.spaceID = parentMD.GetId().GetSpaceId() + + // An id-based ref yields a relative dir ("."), so ask for the parent's full path. + if utils.IsRelativeReference(ref) { + if parentPath, pErr := c.fs.GetPathByID(ctx, parentMD.GetId()); pErr == nil { + t.dir = parentPath + } + } + return nil +} + +// missingParentError tells a missing directory apart from one that is only +// invisible to the caller: a visible ancestor means it is missing. +func (c *coordinator) missingParentError(ctx context.Context, ref *provider.Reference, dir string, notFound error) error { + for ancestor := dir; ancestor != "." && ancestor != "/"; { + ancestor = filepath.Dir(ancestor) + ancestorRef := &provider.Reference{ResourceId: ref.GetResourceId(), Path: ancestor} + if _, err := c.fs.GetMD(ctx, ancestorRef, []string{}, []string{}); err == nil { + return errtypes.PreconditionFailed(notFound.Error()) + } + } + return errtypes.PermissionDenied(ref.GetPath()) +} + +// rewriteChunkedRef turns a legacy chunking-v1 path into a reference to the real +// target file, plus the chunk name. +func rewriteChunkedRef(ref *provider.Reference) (*provider.Reference, string, error) { + ci, err := chunking.GetChunkBLOBInfo(ref.GetPath()) + if err != nil { + return nil, "", errtypes.BadRequest(err.Error()) + } + return &provider.Reference{ResourceId: ref.ResourceId, Path: ci.Path}, filepath.Base(ref.GetPath()), nil +} + +// ListUploadSessions returns the upload sessions matching the given filter. +func (c *coordinator) ListUploadSessions(ctx context.Context, filter storage.UploadSessionFilter) ([]storage.UploadSession, error) { + // Only the driver can resolve a session's node, so the orphaned filter needs a + // driver that answers the question. Assert once rather than per session. + var orphanChecker storage.OrphanChecker + if filter.Orphaned != nil { + var ok bool + if orphanChecker, ok = c.fs.(storage.OrphanChecker); !ok { + return nil, errtypes.NotSupported("coordinator: the orphaned filter is not supported") + } + } + + var sessions []Session + if filter.ID != nil && *filter.ID != "" { + session, err := c.store.Get(ctx, *filter.ID) + if err != nil { + return nil, err + } + sessions = []Session{session} + } else { + var err error + sessions, err = c.store.List(ctx) + if err != nil { + return nil, err + } + } + + filtered := []storage.UploadSession{} + now := time.Now() + for _, session := range sessions { + if filter.Processing != nil && *filter.Processing != session.IsProcessing() { + continue + } + if filter.Expired != nil { + if *filter.Expired { + if now.Before(session.Expires()) { + continue + } + } else { + if now.After(session.Expires()) { + continue + } + } + } + if filter.HasVirus != nil { + sr, _ := session.ScanData() + infected := sr != "" + if *filter.HasVirus != infected { + continue + } + } + // evaluated last: unlike the other filters this reads the node metadata + // from disk, so it is only done for sessions that passed all other filters + if filter.Orphaned != nil { + ref := session.Reference() + if *filter.Orphaned != orphanChecker.IsOrphaned(ctx, &ref) { + continue + } + } + filtered = append(filtered, session) + } + return filtered, nil +} + +// uploadRef builds the reference upload events carry: the space root, plus the +// file's path within it. +func (c *coordinator) uploadRef(session Session) *provider.Reference { + return &provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.SpaceID(), + }, + Path: utils.MakeRelativePath(filepath.Join(session.Dir(), session.Filename())), + } +} + +// impersonatingUser returns the real actor behind a borrowed identity, which +// public link and OCM auth record in the request user's opaque. +func impersonatingUser(ctx context.Context) *user.User { + u, ok := ctxpkg.ContextGetUser(ctx) + if !ok || !utils.ExistsInOpaque(u.GetOpaque(), "impersonating-user") { + return nil + } + impersonating := &user.User{} + if err := utils.ReadJSONFromOpaque(u.GetOpaque(), "impersonating-user", impersonating); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Msg("could not read impersonating user") + return nil + } + return impersonating +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/filestore.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/filestore.go new file mode 100644 index 0000000000..3898705952 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/filestore.go @@ -0,0 +1,210 @@ +package upload + +import ( + "context" + "encoding/json" + iofs "io/fs" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/go-viper/mapstructure/v2" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/rs/zerolog" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" +) + +// TokenOptions carries the JWT-signing configuration needed to produce transfer +// URLs for the postprocessing service. +type TokenOptions struct { + DownloadEndpoint string + DataGatewayEndpoint string + TransferSharedSecret string + TransferExpires int64 +} + +// SessionStore abstracts upload-session persistence for the Coordinator. +type SessionStore interface { + New(ctx context.Context) Session + Get(ctx context.Context, id string) (Session, error) + List(ctx context.Context) ([]Session, error) + // FromInfo wraps an upload's tus FileInfo without reading the store, so it also + // works after a synchronous upload has cleaned up its session. + FromInfo(info tusd.FileInfo) Session +} + +// FileStore is a filesystem-backed SessionStore. Sessions are stored as a pair +// of files in the upload directory: +// +// - .info — JSON-encoded tusd.FileInfo +// - — staged binary bytes +// +// This is the same on-disk format used by OcisStore so existing sessions +// survive a rolling deploy that switches to FileStore. +type FileStore struct { + uploadDir string + opts TokenOptions + log *zerolog.Logger +} + +// FileStoreFromDriverConf builds a FileStore from a reva driver config map. +// Returns nil if the config carries no upload path (driver does not support +// coordinated uploads). Each service that mounts the same driver calls this +// independently. +func FileStoreFromDriverConf(driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if driverConf == nil { + return nil + } + + // storage_root is the ocm driver's spelling of root; it stages uploads under + // the same /uploads/ layout the decomposedfs family uses. + type driverRootConf struct { + Root string `mapstructure:"root"` + UploadDirectory string `mapstructure:"upload_directory"` + StorageRoot string `mapstructure:"storage_root"` + } + var rc driverRootConf + _ = mapstructure.Decode(driverConf, &rc) + + // upload_directory already names the upload directory itself, the way + // decomposedfs reads it (options.go:172, posix tree.go:168). The other two are + // storage roots that stage uploads in a subdirectory, so only they get joined. + if rc.UploadDirectory != "" { + return newFileStoreWithTokens(rc.UploadDirectory, driverConf, log) + } + + root := rc.Root + if root == "" { + root = rc.StorageRoot + } + if root == "" { + return nil + } + + return newFileStoreWithTokens(filepath.Join(root, "uploads"), driverConf, log) +} + +// NewFileStoreFromConfig builds a FileStore staging uploads in uploadDir when +// set, falling back to the active driver config. This allows drivers that have +// no local root (e.g. KW) to still get a coordinator by setting +// upload_directory at the service level rather than inside the driver. +// Returns nil only when neither source resolves to a non-empty path. +func NewFileStoreFromConfig(uploadDir string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + if uploadDir != "" { + // Still take the tokens from the driver config: they sign the transfer URL + // that postprocessing downloads the staged bytes from, and a service-level + // upload directory says nothing about them. + return newFileStoreWithTokens(uploadDir, driverConf, log) + } + return FileStoreFromDriverConf(driverConf, log) +} + +func newFileStoreWithTokens(uploadDir string, driverConf map[string]interface{}, log *zerolog.Logger) *FileStore { + type tokenConf struct { + DownloadEndpoint string `mapstructure:"download_endpoint"` + DataGatewayEndpoint string `mapstructure:"datagateway_endpoint"` + TransferSharedSecret string `mapstructure:"transfer_shared_secret"` + TransferExpires int64 `mapstructure:"transfer_expires"` + } + var tc tokenConf + if tokens, ok := driverConf["tokens"]; ok { + _ = mapstructure.Decode(tokens, &tc) + } + return NewFileStore(uploadDir, TokenOptions(tc), log) +} + +// NewFileStore creates a FileStore staging uploads in uploadDir. +// uploadDir must be on a shared filesystem when multiple pods handle the same space. +func NewFileStore(uploadDir string, opts TokenOptions, log *zerolog.Logger) *FileStore { + return &FileStore{uploadDir: uploadDir, opts: opts, log: log} +} + +// UploadDir returns the directory this FileStore stages uploads in. +func (fs *FileStore) UploadDir() string { + return fs.uploadDir +} + +// Setup creates the upload directory eagerly so permission problems are caught +// at startup rather than on the first upload. +func (fs *FileStore) Setup() error { + return os.MkdirAll(fs.uploadDir, 0700) +} + +// New allocates a fresh session with a new UUID. +func (fs *FileStore) New(_ context.Context) Session { + return &FileSession{ + store: fs, + info: tusd.FileInfo{ + ID: uuid.New().String(), + Storage: map[string]string{ + "Type": "OCISStore", + }, + MetaData: tusd.MetaData{ + "status": SessionStatusUploading, + }, + }, + } +} + +// FromInfo wraps an upload's tus FileInfo into a session without reading the store. +func (fs *FileStore) FromInfo(info tusd.FileInfo) Session { + return &FileSession{store: fs, info: info} +} + +// Get loads the session with the given id from disk. +func (fs *FileStore) Get(ctx context.Context, id string) (Session, error) { + infoPath := fileSessionPath(fs.uploadDir, id) + + data, err := os.ReadFile(infoPath) + if err != nil { + if pathErr, ok := err.(*os.PathError); ok && pathErr.Err == syscall.ESTALE { + return nil, tusd.ErrNotFound + } + if errors.Is(err, iofs.ErrNotExist) { + return nil, tusd.ErrNotFound + } + return nil, err + } + + var info tusd.FileInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + session := &FileSession{store: fs, info: info} + + stat, err := os.Stat(session.binPath()) + if err != nil { + if os.IsNotExist(err) { + return nil, tusd.ErrNotFound + } + return nil, err + } + session.info.Offset = stat.Size() + + return session, nil +} + +// List returns all sessions found in the upload directory. +func (fs *FileStore) List(ctx context.Context) ([]Session, error) { + infoFiles, err := filepath.Glob(filepath.Join(fs.uploadDir, "*.info")) + if err != nil { + return nil, err + } + + sessions := make([]Session, 0, len(infoFiles)) + for _, path := range infoFiles { + id := strings.TrimSuffix(filepath.Base(path), ".info") + session, err := fs.Get(ctx, id) + if err != nil { + appctx.GetLogger(ctx).Error().Str("path", path).Err(err).Msg("filestore: could not load session") + continue + } + sessions = append(sessions, session) + } + return sessions, nil +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/postprocessing.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/postprocessing.go new file mode 100644 index 0000000000..852cce825d --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/postprocessing.go @@ -0,0 +1,261 @@ +// This file holds the postprocessing result collector: the second half of an +// async upload. finishUpload stages the bytes and publishes BytesReceived; the +// postprocessing service scans them and reports back here, and only then are the +// bytes committed through the driver seam. +// +// It is driver-agnostic on purpose: this logic used to live inside decomposedfs, +// which meant only that driver could offer async uploads. + +package upload + +import ( + "context" + "errors" + + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/rs/zerolog" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/opencloud-eu/reva/v2/pkg/events" + "github.com/opencloud-eu/reva/v2/pkg/rhttp/datatx/metrics" + "github.com/opencloud-eu/reva/v2/pkg/utils" +) + +// RegisteredEvents are the postprocessing events the coordinator consumes. +var RegisteredEvents = []events.Unmarshaller{ + events.PostprocessingFinished{}, + events.PostprocessingStepFinished{}, + events.RestartPostprocessing{}, + events.StartPostprocessingStep{}, + events.CleanUpload{}, +} + +// StartPostprocessing subscribes to postprocessing results and switches the +// coordinator over to async uploads: from here on finished uploads stage their +// bytes and wait for a scan verdict instead of committing inline. +// +// The two go together on purpose. Deferring a commit is only safe if something +// will arrive to finish it, so there is no way to enable async without a running +// consumer, and none to run a consumer that never receives work. +// +// Call once, before serving requests. Fails without a publisher: nothing would +// hand uploads to postprocessing, so every one of them would wait for a verdict +// that never comes. +func (c *coordinator) StartPostprocessing(stream events.Consumer, ac AsyncConf) error { + if c.pub == nil { + return errors.New("coordinator: async uploads need an event publisher") + } + ch, err := events.Consume(stream, ac.ConsumerGroup, RegisteredEvents...) + if err != nil { + return err + } + numConsumers := ac.NumConsumers + if numConsumers <= 0 { + numConsumers = 1 + } + c.mountID = ac.MountID + c.commitMaxRetries = ac.CommitMaxRetries + c.commitRetryBackoff = ac.CommitRetryBackoff + c.async = true + for i := 0; i < numConsumers; i++ { + go c.Postprocessing(ch) + } + return nil +} + +// Postprocessing consumes postprocessing results until ch is closed. Run it in +// its own goroutine, one per configured consumer. +func (c *coordinator) Postprocessing(ch <-chan events.Event) { + for event := range ch { + // derive a per-event context from the event's W3C trace context so the + // postprocessing work is linked to the originating request + ctx := event.GetTraceContext(context.Background()) + ctx = ctxpkg.ContextSetInitiator(ctx, event.InitiatorID) + c.processEvent(ctx, event) + } +} + +// servesStorage reports whether an event is for the storage this coordinator +// serves. Postprocessing runs as a separate service and broadcasts its results to +// every storage provider, so each has to recognise its own. Events that name no +// storage predate this and are accepted. +func (c *coordinator) servesStorage(id *provider.ResourceId) bool { + if c.mountID == "" || id.GetStorageId() == "" { + return true + } + return id.GetStorageId() == c.mountID +} + +func (c *coordinator) processEvent(ctx context.Context, event events.Event) { + log := appctx.GetLogger(ctx) + + switch ev := event.Event.(type) { + case events.PostprocessingFinished: + if !c.servesStorage(ev.ResourceID) { + return + } + c.onPostprocessingFinished(ctx, ev, log) + case events.RestartPostprocessing: + c.onRestartPostprocessing(ctx, ev, log) + case events.StartPostprocessingStep: + if ev.UploadID == "" { + return + } + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + // postprocessing broadcasts to every storage, the session may live elsewhere + log.Debug().Err(err).Str("uploadid", ev.UploadID).Msg("StartPostprocessingStep: could not load session") + return + } + session.SetStatus(SessionStatusProcessing, "started postprocessing step: "+string(ev.StepToStart)) + if err := session.Persist(ctx); err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("could not persist the session status") + } + case events.CleanUpload: + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("CleanUpload: could not load session") + return + } + if !ev.KeepUpload { + c.rollbackPrepared(ctx, session, session.SizeDiff()) + } else { + c.rollbackNode(ctx, session, "cleaned up, keeping the upload") + metrics.UploadProcessing.Dec() + } + case events.PostprocessingStepFinished: + if !c.servesStorage(ev.ResourceID) { + return + } + if ev.FinishedStep != events.PPStepAntivirus { + // only the antivirus result is recorded on the session + return + } + res, ok := ev.Result.(events.VirusscanResult) + if !ok || res.ErrorMsg != "" { + // the scan itself failed; PostprocessingFinished decides the outcome + return + } + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + // an empty upload id means an on-demand scan, which has no session + if ev.UploadID != "" { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("PostprocessingStepFinished: could not load session") + } + return + } + session.SetScanData(res.Description, res.Scandate) + switch ev.Outcome { + case events.PPOutcomeAbort, events.PPOutcomeDelete: + session.SetStatus(SessionStatusFailed, "finished postprocessing step: "+string(ev.FinishedStep)) + default: + session.SetStatus(SessionStatusProcessing, "finished postprocessing step: "+string(ev.FinishedStep)) + } + metrics.UploadSessionsScanned.Inc() + if err := session.Persist(ctx); err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("could not persist scan result") + } + } +} + +// onPostprocessingFinished completes or discards an upload according to the +// outcome postprocessing reported. +func (c *coordinator) onPostprocessingFinished(ctx context.Context, ev events.PostprocessingFinished, log *zerolog.Logger) { + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + // Without the session we cannot reach the staged bytes, so they are leaked + // here. Housekeeping cleans them up later. + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("PostprocessingFinished: could not load session") + return + } + ctx = session.Context(ctx) + log = appctx.GetLogger(ctx) + + switch ev.Outcome { + case events.PPOutcomeContinue: + if err := c.finishAsync(ctx, session); err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("blob commit failed after all retries, reverting node and keeping bytes") + metrics.UploadSessionsCommitFailed.Inc() + // Keep the staged bytes: the blobstore outage may be transient, so an + // admin can retry with RestartPostprocessing rather than lose the upload. + c.rollbackNode(ctx, session, "commit failed: "+err.Error()) + metrics.UploadProcessing.Dec() + c.publishUploadFailed(ctx, session, ev) + } + return + case events.PPOutcomeAbort: + metrics.UploadSessionsAborted.Inc() + // Keep the staged bytes: an abort is a transient failure and the upload can + // be restarted with RestartPostprocessing. + c.rollbackNode(ctx, session, "postprocessing aborted") + case events.PPOutcomeDelete: + metrics.UploadSessionsDeleted.Inc() + c.rollbackPrepared(ctx, session, session.SizeDiff()) + default: + log.Error().Str("outcome", string(ev.Outcome)).Str("uploadid", ev.UploadID).Msg("unknown postprocessing outcome, aborting") + metrics.UploadSessionsAborted.Inc() + c.rollbackNode(ctx, session, "unknown postprocessing outcome: "+string(ev.Outcome)) + } + + c.publishUploadFailed(ctx, session, ev) +} + +// onRestartPostprocessing re-publishes BytesReceived so a previously aborted +// upload gets another postprocessing run. +func (c *coordinator) onRestartPostprocessing(ctx context.Context, ev events.RestartPostprocessing, log *zerolog.Logger) { + session, err := c.store.Get(ctx, ev.UploadID) + if err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("RestartPostprocessing: could not load session") + return + } + metrics.UploadSessionsRestarted.Inc() + if err := c.publishBytesReceived(session.Context(ctx), session); err != nil { + log.Error().Err(err).Str("uploadid", ev.UploadID).Msg("could not restart postprocessing") + } +} + +// rollbackNode reverts the node to its pre-upload state but keeps the staged +// bytes and the session, so postprocessing can be restarted. +func (c *coordinator) rollbackNode(ctx context.Context, session Session, reason string) { + ref := session.Reference() + if err := c.fs.RollbackUpload(ctx, &ref, session.ID(), rollbackInfo(session, session.SizeDiff())); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("could not roll back upload, keeping session") + return + } + c.unmarkProcessing(ctx, session, &ref) + + // the size was given back, so a later cleanup of the kept session must not + // release it again + session.SetSizeDiff(0) + session.SetStatus(SessionStatusFailed, reason) + if err := session.Persist(ctx); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("could not persist the rolled back session") + } +} + +// publishUploadFailed tells consumers the upload will not become available. +// Clients wait on UploadReady, so staying silent would leave them hanging. +func (c *coordinator) publishUploadFailed(ctx context.Context, session Session, ev events.PostprocessingFinished) { + if c.pub == nil { + return + } + if err := events.Publish(ctx, c.pub, events.UploadReady{ + UploadID: session.ID(), + Failed: true, + Filename: session.Filename(), + SpaceOwner: session.SpaceOwner(), + ExecutingUser: ev.ExecutingUser, + FileRef: c.uploadRef(session), + ResourceID: &provider.ResourceId{ + StorageId: session.ProviderID(), + SpaceId: session.SpaceID(), + OpaqueId: session.NodeID(), + }, + Timestamp: utils.TSNow(), + IsVersion: session.VersionCreated(), + ImpersonatingUser: ev.ImpersonatingUser, + }); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", session.ID()).Msg("failed to publish UploadReady event") + } +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/session.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/session.go new file mode 100644 index 0000000000..8899d934d9 --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/session.go @@ -0,0 +1,529 @@ +package upload + +import ( + "context" + "crypto/md5" //nolint:gosec + "crypto/sha1" //nolint:gosec + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "hash/adler32" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + typespb "github.com/cs3org/go-cs3apis/cs3/types/v1beta1" + "github.com/golang-jwt/jwt/v5" + "github.com/google/renameio/v2" + "github.com/pkg/errors" + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + ctxpkg "github.com/opencloud-eu/reva/v2/pkg/ctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" + "github.com/opencloud-eu/reva/v2/pkg/storage" + "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/disk" + "github.com/opencloud-eu/reva/v2/pkg/utils" +) + +const defaultFilePerm = os.FileMode(0664) + +// Session status values, stored in the session's "status" metadata +const ( + SessionStatusUploading = "uploading" + SessionStatusProcessing = "processing" + SessionStatusFailed = "failed" +) + +// FileSession is the Session implementation for disk-backed uploads. While an +// upload is in progress, incoming bytes are staged in a .bin file and upload +// metadata (size, owner, checksums, etc.) is persisted in a .info file. Both +// survive process restarts, allowing TUS resumption. +// +// In scope: read/write the staged .bin file, persist/load upload metadata in the .info file. +// Out of scope: TUS protocol, checksums, event publishing, postprocessing — those live in coordinatedUpload and coordinator. +type FileSession struct { + store *FileStore + info tusd.FileInfo +} + +// Session is the driver-agnostic view of an upload session the Coordinator +// needs. Implementations must be pure state (CRUD): protocol orchestration +// belongs to coordinatedUpload or the coordinator itself. +type Session interface { + storage.UploadSession + + // Data access — delegated to by coordinatedUpload for TUS reads/writes. + GetInfo(ctx context.Context) (tusd.FileInfo, error) + GetReader(ctx context.Context) (io.ReadCloser, error) + WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) + + // Internal coordinator plumbing. + Chunk() string + BinPath() string + ExecutantUser() *userpb.User + ProviderID() string + SpaceID() string + NodeID() string + NodeParentID() string + NodeExists() bool + Dir() string + URL(ctx context.Context) (string, error) + SetScanData(result string, date time.Time) + SetStatus(status, msg string) + Checksums() storage.UploadChecksums + SetChecksums(sha1, md5, adler32 []byte) + SizeDiff() int64 + SetSizeDiff(d int64) + VersionCreated() bool + SetVersionCreated(v bool) + Metadata() map[string]string + Persist(ctx context.Context) error + Cleanup(ctx context.Context, cleanBin, cleanInfo bool) + Context(ctx context.Context) context.Context + + // Typed setters used by Coordinator.InitiateUpload to populate a new session + // without knowing internal storage key names. + SetStorageValue(key, value string) + SetMetadata(key, value string) + SetSize(size int64) + SetSizeIsDeferred(value bool) + SetExecutant(u *userpb.User) + TouchBin() error +} + +func (s *FileSession) GetInfo(_ context.Context) (tusd.FileInfo, error) { + return s.info, nil +} + +func (s *FileSession) GetReader(_ context.Context) (io.ReadCloser, error) { + return os.Open(s.binPath()) +} + +func (s *FileSession) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + log := appctx.GetLogger(ctx) + file, err := os.OpenFile(s.binPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + log.Error().Err(err).Str("uploadid", s.info.ID).Msg("WriteChunk: error opening upload file") + return 0, err + } + defer func() { + // sync each chunk right away: the upload can be resumed after a crash, and slow + // disks are spared one large flush at the end of the upload + _ = disk.Fdatasync(file) + _ = file.Close() + }() + + n, err := io.Copy(file, src) + if err != nil && err != io.ErrUnexpectedEOF { + log.Error().Err(err).Str("uploadid", s.info.ID).Msg("WriteChunk: error copying data to upload file") + return n, err + } + s.info.Offset += n + return n, nil +} + +// Purge removes all on-disk state for this session. +func (s *FileSession) Purge(ctx context.Context) { + s.Cleanup(ctx, true, true) +} + +// ScanData returns the AV scan result and scan date stored on the session. +func (s *FileSession) ScanData() (string, time.Time) { + date := s.info.MetaData["scanDate"] + if date == "" { + return "", time.Time{} + } + d, _ := time.Parse(time.RFC3339, date) + return s.info.MetaData["scanResult"], d +} + +// ID returns the upload session ID. +func (s *FileSession) ID() string { + return s.info.ID +} + +// Filename returns the filename stored in the session. +func (s *FileSession) Filename() string { + return s.info.Storage["NodeName"] +} + +// Size returns the declared upload size. +func (s *FileSession) Size() int64 { + return s.info.Size +} + +// Offset returns the current upload offset. +func (s *FileSession) Offset() int64 { + return s.info.Offset +} + +// Chunk returns the chunk basename stored in the session, or "" for non-chunked uploads. +func (s *FileSession) Chunk() string { + return s.info.Storage["Chunk"] +} + +// BinPath returns the path to the staged binary file. +func (s *FileSession) BinPath() string { + return s.binPath() +} + +// SpaceGid returns the numeric GID of the space owner, or "" if not set. +func (s *FileSession) SpaceGid() string { + return s.info.Storage["SpaceGid"] +} + +// ProviderID returns the storage provider ID stored in the session. +func (s *FileSession) ProviderID() string { + return s.info.MetaData["providerID"] +} + +// SpaceID returns the space (root) ID. +func (s *FileSession) SpaceID() string { + return s.info.Storage["SpaceRoot"] +} + +// NodeID returns the node ID for this upload. +func (s *FileSession) NodeID() string { + return s.info.Storage["NodeId"] +} + +// NodeParentID returns the parent node ID for this upload. +func (s *FileSession) NodeParentID() string { + return s.info.Storage["NodeParentId"] +} + +// NodeExists returns whether the target node existed when the upload was initiated. +func (s *FileSession) NodeExists() bool { + return s.info.Storage["NodeExists"] == "true" +} + +// Dir returns the directory portion of the upload path. +func (s *FileSession) Dir() string { + return s.info.Storage["Dir"] +} + +// IsProcessing returns true if the upload is in the processing state, meaning that the upload has finished and postprocessing is still running. +func (s *FileSession) IsProcessing() bool { + return s.info.MetaData["status"] == SessionStatusProcessing +} + +// SetStatus sets the status of the upload session +func (s *FileSession) SetStatus(status, msg string) { + s.info.MetaData["status"] = status + s.info.MetaData["statusMessage"] = msg +} + +// Status returns the status of the upload session +func (s *FileSession) Status() string { + return s.info.MetaData["status"] +} + +// StatusMessage returns the status message of the upload session +func (s *FileSession) StatusMessage() string { + return s.info.MetaData["statusMessage"] +} + +// SpaceOwner returns the space owner user ID. +func (s *FileSession) SpaceOwner() *userpb.UserId { + return &userpb.UserId{ + OpaqueId: s.info.Storage["SpaceOwnerOrManager"], + Idp: s.info.Storage["SpaceOwnerIdp"], + Type: userpb.UserType(userpb.UserType_value[s.info.Storage["SpaceOwnerType"]]), + } +} + +// Executant returns the user ID of the user who initiated this upload. +func (s *FileSession) Executant() userpb.UserId { + return userpb.UserId{ + Type: utils.UserTypeMap(s.info.Storage["UserType"]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + } +} + +// Expires returns the upload expiry time. +func (s *FileSession) Expires() time.Time { + var t time.Time + if value, ok := s.info.MetaData["expires"]; ok { + t, _ = utils.MTimeToTime(value) + } + return t +} + +// Reference returns a CS3 reference for the resource being uploaded. +func (s *FileSession) Reference() provider.Reference { + return provider.Reference{ + ResourceId: &provider.ResourceId{ + StorageId: s.info.MetaData["providerID"], + SpaceId: s.info.Storage["SpaceRoot"], + OpaqueId: s.info.Storage["NodeId"], + }, + } +} + +// Checksums returns the pre-computed checksums stored on the session. +func (s *FileSession) Checksums() storage.UploadChecksums { + decode := func(key string) []byte { + b, _ := hex.DecodeString(s.info.MetaData[key]) + return b + } + return storage.UploadChecksums{ + SHA1: decode("checksumSHA1"), + MD5: decode("checksumMD5"), + Adler32: decode("checksumAdler32"), + } +} + +// Metadata returns the upload metadata the coordinator passes to the driver. +func (s *FileSession) Metadata() map[string]string { + return map[string]string{ + "providerID": s.info.MetaData["providerID"], + "mtime": s.info.MetaData["mtime"], + "nodeExists": s.info.Storage["NodeExists"], + "sessionID": s.info.ID, + "if-match": s.info.MetaData["if-match"], + "if-none-match": s.info.MetaData["if-none-match"], + "if-unmodified-since": s.info.MetaData["if-unmodified-since"], + } +} + +// SizeDiff returns the tree size change PrepareUpload propagated optimistically. +// Rolling an upload back has to undo exactly that amount. +func (s *FileSession) SizeDiff() int64 { + d, _ := strconv.ParseInt(s.info.MetaData["sizeDiff"], 10, 64) + return d +} + +// SetSizeDiff records the size change PrepareUpload reported. It is persisted +// because the async path prepares and commits in different processes, so the +// value cannot be held in memory between the two. +func (s *FileSession) SetSizeDiff(d int64) { + s.info.MetaData["sizeDiff"] = strconv.FormatInt(d, 10) +} + +// VersionCreated reports whether this upload superseded existing content, which +// UploadReady consumers use to tell an overwrite from a new file. +func (s *FileSession) VersionCreated() bool { + return s.info.MetaData["versionCreated"] == "true" +} + +// SetVersionCreated records what PrepareUpload reported. Persisted for the same +// reason as the size diff: on the async path the commit runs in another process. +func (s *FileSession) SetVersionCreated(v bool) { + s.info.MetaData["versionCreated"] = strconv.FormatBool(v) +} + +// SetScanData stores AV scan results on the session. +func (s *FileSession) SetScanData(result string, date time.Time) { + s.info.MetaData["scanResult"] = result + s.info.MetaData["scanDate"] = date.Format(time.RFC3339) +} + +// SetChecksums stores pre-computed checksums so CommitUpload can use them without +// re-reading the binary file. +func (s *FileSession) SetChecksums(sha1Sum, md5Sum, adler32Sum []byte) { + s.info.MetaData["checksumSHA1"] = hex.EncodeToString(sha1Sum) + s.info.MetaData["checksumMD5"] = hex.EncodeToString(md5Sum) + s.info.MetaData["checksumAdler32"] = hex.EncodeToString(adler32Sum) +} + +// SetMetadata sets a user-visible upload metadata field. +func (s *FileSession) SetMetadata(key, value string) { + s.info.MetaData[key] = value +} + +// SetStorageValue sets an internal storage field on the session. +func (s *FileSession) SetStorageValue(key, value string) { + s.info.Storage[key] = value +} + +// SetSize updates the declared upload size. +func (s *FileSession) SetSize(size int64) { + s.info.Size = size +} + +// SetSizeIsDeferred marks the upload size as not yet known. +func (s *FileSession) SetSizeIsDeferred(value bool) { + s.info.SizeIsDeferred = value +} + +// SetExecutant stores the identity of the user who initiated the upload. +func (s *FileSession) SetExecutant(u *userpb.User) { + s.info.Storage["Idp"] = u.GetId().GetIdp() + s.info.Storage["UserId"] = u.GetId().GetOpaqueId() + s.info.Storage["UserType"] = utils.UserTypeToString(u.GetId().Type) + s.info.Storage["UserName"] = u.GetUsername() + s.info.Storage["UserDisplayName"] = u.GetDisplayName() + b, _ := json.Marshal(u.GetOpaque()) + s.info.Storage["UserOpaque"] = string(b) + g, _ := json.Marshal(u.GetGroups()) + s.info.Storage["UserGroups"] = string(g) +} + +// TouchBin creates the empty staging file. +func (s *FileSession) TouchBin() error { + f, err := os.OpenFile(s.binPath(), os.O_CREATE|os.O_WRONLY, defaultFilePerm) + if err != nil { + return err + } + return f.Close() +} + +// Persist writes the session metadata atomically to disk. +func (s *FileSession) Persist(ctx context.Context) error { + infoPath := s.infoPath() + if err := os.MkdirAll(filepath.Dir(infoPath), 0700); err != nil { + return err + } + d, err := json.Marshal(s.info) + if err != nil { + return err + } + return renameio.WriteFile(infoPath, d, 0600) +} + +// Cleanup removes the staged binary and/or info file. +// Node deletion and processing flag changes are the coordinator's responsibility. +func (s *FileSession) Cleanup(ctx context.Context, cleanBin, cleanInfo bool) { + if cleanBin { + if err := os.Remove(s.binPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + appctx.GetLogger(ctx).Error().Str("path", s.binPath()).Err(err).Msg("filestore: removing staged binary failed") + } + } + if cleanInfo { + if err := os.Remove(s.infoPath()); err != nil && !errors.Is(err, os.ErrNotExist) { + appctx.GetLogger(ctx).Error().Str("path", s.infoPath()).Err(err).Msg("filestore: removing session info failed") + } + } +} + +// Context reconstructs a context carrying the user, logger, lock ID, and +// initiator ID that were recorded when the upload was initiated. +func (s *FileSession) Context(ctx context.Context) context.Context { + sub := s.store.log.With().Int("pid", os.Getpid()).Logger() + ctx = appctx.WithLogger(ctx, &sub) + ctx = ctxpkg.ContextSetLockID(ctx, s.info.MetaData["lockid"]) + ctx = ctxpkg.ContextSetUser(ctx, s.ExecutantUser()) + return ctxpkg.ContextSetInitiator(ctx, s.info.MetaData["initiatorid"]) +} + +// URL returns the URL that the postprocessing service can use to download the +// staged binary: a signed JWT URL via the data gateway, or the data server's own +// URL when no data gateway is configured, because the proxy signs data URLs at +// the edge. +func (s *FileSession) URL(_ context.Context) (string, error) { + type transferClaims struct { + jwt.RegisteredClaims + Target string `json:"target"` + } + + target := joinURLParts(s.store.opts.DownloadEndpoint, "tus/", s.info.ID) + if s.store.opts.DataGatewayEndpoint == "" { + return target, nil + } + ttl := time.Duration(s.store.opts.TransferExpires) * time.Second + claims := transferClaims{ + RegisteredClaims: jwt.RegisteredClaims{ + ExpiresAt: jwt.NewNumericDate(time.Now().Add(ttl)), + Audience: jwt.ClaimStrings{"reva"}, + IssuedAt: jwt.NewNumericDate(time.Now()), + }, + Target: target, + } + t := jwt.NewWithClaims(jwt.GetSigningMethod("HS256"), claims) + tkn, err := t.SignedString([]byte(s.store.opts.TransferSharedSecret)) + if err != nil { + return "", errors.Wrapf(err, "filestore: error signing transfer token with claims %+v", claims) + } + return joinURLParts(s.store.opts.DataGatewayEndpoint, tkn), nil +} + +func (s *FileSession) ToFileInfo() tusd.FileInfo { + return s.info +} + +func (s *FileSession) InitiatorID() string { + return s.info.MetaData["initiatorid"] +} + +func (s *FileSession) binPath() string { + return filepath.Join(s.store.uploadDir, s.info.ID) +} + +func (s *FileSession) infoPath() string { + return fileSessionPath(s.store.uploadDir, s.info.ID) +} + +// ExecutantUser returns the full identity of the user who initiated the upload. +// Upload events must carry it rather than the bare id from Executant(): consumers +// read the display name straight off the event and do not look it up, so an +// id-only user reaches the activity feed with a blank name. +func (s *FileSession) ExecutantUser() *userpb.User { + var o *typespb.Opaque + _ = json.Unmarshal([]byte(s.info.Storage["UserOpaque"]), &o) + var groups []string + _ = json.Unmarshal([]byte(s.info.Storage["UserGroups"]), &groups) + return &userpb.User{ + Id: &userpb.UserId{ + Type: utils.UserTypeMap(s.info.Storage["UserType"]), + Idp: s.info.Storage["Idp"], + OpaqueId: s.info.Storage["UserId"], + }, + Username: s.info.Storage["UserName"], + DisplayName: s.info.Storage["UserDisplayName"], + Opaque: o, + Groups: groups, + } +} + +// fileSessionPath returns the path to the .info file for the given session ID. +func fileSessionPath(uploadDir, id string) string { + return filepath.Join(uploadDir, id+".info") +} + +// calculateChecksums computes sha1, md5, and adler32 in a single pass over path. +func calculateChecksums(_ context.Context, path string) (hash.Hash, hash.Hash, hash.Hash32, error) { + sha1h := sha1.New() //nolint:gosec + md5h := md5.New() //nolint:gosec + adler32h := adler32.New() + + f, err := os.Open(path) + if err != nil { + return nil, nil, nil, err + } + defer f.Close() + + r1 := io.TeeReader(f, sha1h) + r2 := io.TeeReader(r1, md5h) + if _, err = io.Copy(adler32h, r2); err != nil { + return nil, nil, nil, err + } + return sha1h, md5h, adler32h, nil +} + +func checkHash(expected string, h hash.Hash) error { + got := hex.EncodeToString(h.Sum(nil)) + if expected != got { + return errtypes.ChecksumMismatch(fmt.Sprintf("invalid checksum: expected %s got %s", expected, got)) + } + return nil +} + +// joinURLParts concatenates URL path segments, inserting "/" between them if needed. +func joinURLParts(parts ...string) string { + var b strings.Builder + for i, p := range parts { + b.WriteString(p) + if i < len(parts)-1 && !strings.HasSuffix(p, "/") { + b.WriteByte('/') + } + } + return b.String() +} diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/tus_adapter.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/tus_adapter.go new file mode 100644 index 0000000000..52c44e689c --- /dev/null +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/upload/tus_adapter.go @@ -0,0 +1,175 @@ +package upload + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + tusd "github.com/tus/tusd/v2/pkg/handler" + + "github.com/opencloud-eu/reva/v2/pkg/appctx" + "github.com/opencloud-eu/reva/v2/pkg/errtypes" +) + +var errNotImplemented = tusd.NewError("ERR_NOT_IMPLEMENTED", "use InitiateUpload on the CS3 API to start a new upload", http.StatusNotImplemented) + +// tusAdapter adapts a single upload session to tusd's Upload interfaces. +type tusAdapter struct { + session Session + coord *coordinator +} + +func (u *tusAdapter) GetInfo(ctx context.Context) (tusd.FileInfo, error) { + return u.session.GetInfo(ctx) +} + +func (u *tusAdapter) GetReader(ctx context.Context) (io.ReadCloser, error) { + return u.session.GetReader(ctx) +} + +// ServeContent serves the staged bytes, honouring a Range header: postprocessing reads +// only the first bytes of a file larger than it scans. +func (u *tusAdapter) ServeContent(_ context.Context, w http.ResponseWriter, r *http.Request) error { + f, err := os.Open(u.session.BinPath()) + if err != nil { + return err + } + defer f.Close() + http.ServeContent(w, r, "", time.Time{}, f) + return nil +} + +func (u *tusAdapter) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) { + return u.session.WriteChunk(ctx, offset, src) +} + +// FinishUpload runs the coordinator's finish path once all bytes have arrived. +func (u *tusAdapter) FinishUpload(ctx context.Context) error { + _, err := u.coord.finishUpload(u.session.Context(ctx), u.session) + if err != nil { + // tusd only answers the client, so log the cause here + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", u.session.ID()).Msg("failed to finish upload") + } + + // tusd answers an error type it does not recognise with a bare 500. + switch err.(type) { + case nil: + return nil + case errtypes.AlreadyExists: + return tusd.NewError("ERR_ALREADY_EXISTS", err.Error(), http.StatusConflict) + case errtypes.ResourceProcessing, errtypes.TooEarly: + return tusd.NewError("ERR_TOO_EARLY", err.Error(), http.StatusTooEarly) + case errtypes.Aborted: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusPreconditionFailed) + case errtypes.PreconditionFailed: + return tusd.NewError("ERR_PRECONDITION_FAILED", err.Error(), http.StatusMethodNotAllowed) + case errtypes.Locked: + return tusd.NewError("ERR_LOCKED", err.Error(), http.StatusLocked) + case errtypes.BadRequest: + return tusd.NewError("ERR_BAD_REQUEST", err.Error(), http.StatusBadRequest) + case errtypes.ChecksumMismatch: + return tusd.NewError("ERR_CHECKSUM_MISMATCH", err.Error(), errtypes.StatusChecksumMismatch) + case errtypes.IsPermissionDenied: + return tusd.NewError("ERR_PERMISSION_DENIED", err.Error(), http.StatusForbidden) + case errtypes.InsufficientStorage: + return tusd.NewError("ERR_INSUFFICIENT_STORAGE", err.Error(), http.StatusInsufficientStorage) + default: + return err + } +} + +// Terminate discards an upload, so a cancelled one leaves nothing behind. +func (u *tusAdapter) Terminate(ctx context.Context) error { + ref := u.session.Reference() + + // Rollback rather than Delete, which is permission-gated. It is a no-op unless + // this upload still owns the node. + if err := u.coord.fs.RollbackUpload(ctx, &ref, u.session.ID(), rollbackInfo(u.session, u.session.SizeDiff())); err != nil { + appctx.GetLogger(ctx).Error().Err(err).Str("uploadid", u.session.ID()).Msg("could not roll back terminated upload, keeping session") + return err + } + // A node that was never created cannot be unmarked, which is the normal case here. + if err := u.coord.fs.MarkProcessing(ctx, &ref, false, u.session.ID()); err != nil { + appctx.GetLogger(ctx).Debug().Err(err).Str("uploadid", u.session.ID()).Msg("could not unmark terminated upload") + } + + u.session.Cleanup(ctx, true, true) + return nil +} + +// DeclareLength records the total size for uploads initiated without one. +func (u *tusAdapter) DeclareLength(ctx context.Context, length int64) error { + u.session.SetSize(length) + u.session.SetSizeIsDeferred(false) + return u.session.Persist(ctx) +} + +// ConcatUploads appends the staged bytes of the partial uploads, in the order given. +func (u *tusAdapter) ConcatUploads(ctx context.Context, partials []tusd.Upload) error { + file, err := os.OpenFile(u.session.BinPath(), os.O_WRONLY|os.O_APPEND, defaultFilePerm) + if err != nil { + return err + } + defer file.Close() + + for _, partial := range partials { + p, ok := partial.(*tusAdapter) + if !ok { + return fmt.Errorf("coordinator: unexpected partial upload type %T", partial) + } + src, err := p.session.GetReader(ctx) + if err != nil { + return err + } + _, copyErr := io.Copy(file, src) + src.Close() + if copyErr != nil { + return copyErr + } + } + return nil +} + +// UseIn registers the coordinator as tusd's data store, in place of the driver's own. +func (c *coordinator) UseIn(composer *tusd.StoreComposer) { + composer.UseCore(c) + composer.UseTerminater(c) + composer.UseConcater(c) + composer.UseLengthDeferrer(c) + composer.UseContentServer(c) +} + +// NewUpload is unsupported: uploads always start through the CS3 InitiateUpload call. +func (c *coordinator) NewUpload(_ context.Context, _ tusd.FileInfo) (tusd.Upload, error) { + return nil, errNotImplemented +} + +// GetUpload wraps the session with the given id for the TUS data path. +func (c *coordinator) GetUpload(ctx context.Context, id string) (tusd.Upload, error) { + session, err := c.store.Get(ctx, id) + if err != nil { + return nil, err + } + return &tusAdapter{session: session, coord: c}, nil +} + +// The As* methods let tusd reach the extension interfaces on a plain tusd.Upload. + +func (c *coordinator) AsTerminatableUpload(up tusd.Upload) tusd.TerminatableUpload { + return up.(*tusAdapter) +} + +func (c *coordinator) AsLengthDeclarableUpload(up tusd.Upload) tusd.LengthDeclarableUpload { + return up.(*tusAdapter) +} + +func (c *coordinator) AsConcatableUpload(up tusd.Upload) tusd.ConcatableUpload { + return up.(*tusAdapter) +} + +func (c *coordinator) AsServableUpload(up tusd.Upload) tusd.ServableUpload { + return up.(*tusAdapter) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index ad6f2d9ad0..96f11619cd 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -596,7 +596,7 @@ github.com/go-micro/plugins/v4/wrapper/trace/opentelemetry ## explicit; go 1.12 github.com/go-ole/go-ole github.com/go-ole/go-ole/oleutil -# github.com/go-playground/locales v0.14.1 +# github.com/go-playground/locales v0.14.2 ## explicit; go 1.17 github.com/go-playground/locales github.com/go-playground/locales/currency @@ -604,8 +604,8 @@ github.com/go-playground/locales/en # github.com/go-playground/universal-translator v0.18.2 ## explicit; go 1.26 github.com/go-playground/universal-translator -# github.com/go-playground/validator/v10 v10.30.3 -## explicit; go 1.25.0 +# github.com/go-playground/validator/v10 v10.30.5 +## explicit; go 1.26.0 github.com/go-playground/validator/v10 github.com/go-playground/validator/v10/translations/en # github.com/go-redis/redis/v8 v8.11.5 @@ -889,8 +889,8 @@ github.com/kovidgoyal/imaging/prism/meta/webpmeta github.com/kovidgoyal/imaging/streams github.com/kovidgoyal/imaging/types github.com/kovidgoyal/imaging/webp -# github.com/leodido/go-urn v1.4.0 -## explicit; go 1.18 +# github.com/leodido/go-urn v1.5.0 +## explicit; go 1.24.0 github.com/leodido/go-urn github.com/leodido/go-urn/scim/schema # github.com/leonelquinteros/gotext v1.7.3-0.20260422134830-b012b4ccae69 @@ -1360,7 +1360,7 @@ github.com/opencloud-eu/icap-client # github.com/opencloud-eu/libre-graph-api-go v1.0.9-0.20261016073229-17ed3d5dd346 ## explicit; go 1.23 github.com/opencloud-eu/libre-graph-api-go -# github.com/opencloud-eu/reva/v2 v2.52.1 +# github.com/opencloud-eu/reva/v2 v2.52.2-0.20261106140807-2d4db7d275fc ## explicit; go 1.26.0 github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/runtime @@ -1620,8 +1620,6 @@ github.com/opencloud-eu/reva/v2/pkg/storage/fs/loader github.com/opencloud-eu/reva/v2/pkg/storage/fs/local github.com/opencloud-eu/reva/v2/pkg/storage/fs/localhome github.com/opencloud-eu/reva/v2/pkg/storage/fs/nextcloud -github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis -github.com/opencloud-eu/reva/v2/pkg/storage/fs/ocis/blobstore github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql github.com/opencloud-eu/reva/v2/pkg/storage/fs/owncloudsql/filecache github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix @@ -1637,8 +1635,6 @@ github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry -github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng -github.com/opencloud-eu/reva/v2/pkg/storage/fs/s3ng/blobstore github.com/opencloud-eu/reva/v2/pkg/storage/internal/goroutinelock github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/aspects @@ -1655,7 +1651,6 @@ github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/timemanager github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/trashbin github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/tree/propagator -github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/usermapper github.com/opencloud-eu/reva/v2/pkg/storage/registry/loader github.com/opencloud-eu/reva/v2/pkg/storage/registry/registry @@ -1664,23 +1659,6 @@ github.com/opencloud-eu/reva/v2/pkg/storage/registry/static github.com/opencloud-eu/reva/v2/pkg/storage/utils/ace github.com/opencloud-eu/reva/v2/pkg/storage/utils/acl github.com/opencloud-eu/reva/v2/pkg/storage/utils/chunking -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/aspects -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/lookup -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/metadata/prefixes -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/migrator -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/mtimesyncedcache -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/node -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/options -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/permissions -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/spaceidindex -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/timemanager -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/trashbin -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/tree/propagator -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/upload -github.com/opencloud-eu/reva/v2/pkg/storage/utils/decomposedfs/usermapper github.com/opencloud-eu/reva/v2/pkg/storage/utils/downloader github.com/opencloud-eu/reva/v2/pkg/storage/utils/eosfs github.com/opencloud-eu/reva/v2/pkg/storage/utils/filelocks @@ -1708,6 +1686,7 @@ github.com/opencloud-eu/reva/v2/pkg/token/manager/jwt github.com/opencloud-eu/reva/v2/pkg/token/manager/loader github.com/opencloud-eu/reva/v2/pkg/token/manager/registry github.com/opencloud-eu/reva/v2/pkg/trace +github.com/opencloud-eu/reva/v2/pkg/upload github.com/opencloud-eu/reva/v2/pkg/user github.com/opencloud-eu/reva/v2/pkg/user/manager/demo github.com/opencloud-eu/reva/v2/pkg/user/manager/json